mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
Merge origin/main into gatekeeper/fix-2325-1
This commit is contained in:
Gatefixer
@@ -0,0 +1,222 @@
|
||||
name: Check doc links
|
||||
|
||||
# Checking external links is inherently noisy: third-party sites rate-limit
|
||||
# automated clients, reject non-browser user agents, and go down temporarily.
|
||||
# Blocking pull requests on that trades a lot of false failures for very little
|
||||
# signal, so this runs on a schedule and reports findings in a single tracking
|
||||
# issue instead of failing anyone's build.
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 7 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
# The report lives in one repository-global issue, so runs must not overlap: a
|
||||
# lookup racing a create produces duplicate issues, and a healthy run closing
|
||||
# the issue while a failing run only rewrites its body would leave a broken
|
||||
# report closed. The group is deliberately ref-independent so that a manual
|
||||
# dispatch serializes against the scheduled run.
|
||||
concurrency:
|
||||
group: docs-link-check
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
env:
|
||||
REPORT_TITLE: "Docs link checker report"
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan links
|
||||
runs-on: ubuntu-24.04
|
||||
# lychee-action is pinned by SHA, but its wrapper downloads the lychee
|
||||
# release tarball at run time without verifying a digest, and hands the
|
||||
# resulting binary a GitHub token. Release assets remain replaceable, so
|
||||
# that binary is confined to a job whose token can only read public
|
||||
# content; everything that writes runs in the report job below.
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
exit_code: ${{ steps.lychee.outputs.exit_code }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
# workflow_dispatch can run from any ref, but the report is
|
||||
# repository-global. Always measure the default branch so a manual
|
||||
# run from a topic branch cannot close a report that main warrants,
|
||||
# or overwrite it with branch-only findings.
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check links
|
||||
id: lychee
|
||||
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
|
||||
with:
|
||||
# Restricted to http(s) on purpose. Much of docs/src is generated
|
||||
# API reference (the js/ tree comes from `npm run docs` in nodejs)
|
||||
# and the hand-written pages use mkdocstrings cross-references and
|
||||
# nav-relative paths that only resolve in the site mkdocs builds,
|
||||
# not in this checkout, so relative links would be reported as
|
||||
# broken on every run.
|
||||
args: >-
|
||||
--scheme https
|
||||
--scheme http
|
||||
--no-progress
|
||||
--max-retries 3
|
||||
--timeout 20
|
||||
'docs/src/**/*.md'
|
||||
format: json
|
||||
output: ./lychee/out.json
|
||||
jobSummary: false
|
||||
# The report, not a red build, is the signal for broken links. The
|
||||
# validation step below still fails the run if the check itself
|
||||
# breaks.
|
||||
fail: false
|
||||
|
||||
- name: Validate report
|
||||
# lychee does not reserve exit code 2 for broken links: its CLI
|
||||
# parser also exits 2 on an invalid option, before any link was
|
||||
# checked or any report written. Only a parseable report whose
|
||||
# counts agree with the exit code counts as a link verdict; anything
|
||||
# else fails here, and the report job below is skipped entirely, so
|
||||
# the tracking issue is never touched. Exit 2 covers timeouts as
|
||||
# well as errors, and a timed-out host is exactly the transient
|
||||
# unavailability this report exists to surface, so both count as
|
||||
# findings. Requiring total > 0 also catches a glob that silently
|
||||
# stopped matching any file.
|
||||
if: steps.lychee.outputs.exit_code == 0 || steps.lychee.outputs.exit_code == 2
|
||||
env:
|
||||
EXIT_CODE: ${{ steps.lychee.outputs.exit_code }}
|
||||
run: |
|
||||
jq -e --argjson code "$EXIT_CODE" '
|
||||
(.total > 0) and
|
||||
(if $code == 0
|
||||
then .errors == 0 and .timeouts == 0
|
||||
and (.error_map | length == 0) and (.timeout_map | length == 0)
|
||||
else (.errors + .timeouts) > 0
|
||||
and ((.error_map | length) + (.timeout_map | length)) > 0
|
||||
end)
|
||||
' ./lychee/out.json
|
||||
|
||||
- name: Upload report
|
||||
if: steps.lychee.outputs.exit_code == 2
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: link-report
|
||||
path: ./lychee/out.json
|
||||
retention-days: 7
|
||||
|
||||
report:
|
||||
name: Update report issue
|
||||
needs: scan
|
||||
runs-on: ubuntu-24.04
|
||||
# Deliberately no checkout: this job needs the report artifact and the
|
||||
# issues API, not the repository contents.
|
||||
permissions:
|
||||
issues: write
|
||||
env:
|
||||
EXIT_CODE: ${{ needs.scan.outputs.exit_code }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- name: Classify checker result
|
||||
# lychee exits 0 when every link resolves and 2 when links fail,
|
||||
# both already cross-checked against the report by the scan job's
|
||||
# validation step. Anything else (1 runtime, 3 bad config) means the
|
||||
# check never produced a link verdict, which must surface as a failed
|
||||
# run rather than be published as "broken documentation links".
|
||||
run: |
|
||||
case "$EXIT_CODE" in
|
||||
0|2)
|
||||
echo "lychee exit code $EXIT_CODE"
|
||||
;;
|
||||
*)
|
||||
echo "::error::lychee exited with '$EXIT_CODE': the link check did not complete. Leaving the report issue untouched."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Find existing report issue
|
||||
id: report
|
||||
# Matched on title alone, and through search rather than a listing:
|
||||
# the issue action applies labels in a separate call after creating the
|
||||
# issue, so a label filter misses a half-created report, and this
|
||||
# repository has far more open issues than one listing page holds.
|
||||
# Closed issues are included because a healthy run closes the report:
|
||||
# an open-only lookup would forget that identity and the next failing
|
||||
# run would open a duplicate. The oldest match stays the canonical
|
||||
# report and is reopened below when links break again.
|
||||
run: |
|
||||
match=$(gh issue list --repo "$GITHUB_REPOSITORY" --state all \
|
||||
--search "in:title \"$REPORT_TITLE\" author:app/github-actions" \
|
||||
--limit 50 --json number,title,state \
|
||||
--jq "[.[] | select(.title == \"$REPORT_TITLE\")] | sort_by(.number) | first // empty")
|
||||
echo "number=$(jq -r '.number // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
|
||||
echo "state=$(jq -r '.state // empty' <<<"$match")" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download report
|
||||
if: env.EXIT_CODE == 2
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: link-report
|
||||
path: ./lychee
|
||||
|
||||
- name: Compose report
|
||||
if: env.EXIT_CODE == 2
|
||||
run: |
|
||||
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
|
||||
{
|
||||
echo "Broken documentation links found by [\`$GITHUB_WORKFLOW\`]($run_url)."
|
||||
echo
|
||||
echo "This issue is rewritten by every scheduled run and closed automatically once all links resolve."
|
||||
echo
|
||||
echo "Entries can be false positives: some sites rate-limit or block automated clients while working fine in a browser. Confirm before editing the docs, and add persistent offenders to \`--exclude\` in \`.github/workflows/docs-link-check.yml\`."
|
||||
echo
|
||||
# Timeouts are reported alongside errors: entries land in
|
||||
# timeout_map with a status text instead of an HTTP code.
|
||||
jq -r '
|
||||
"\(.errors) of \(.total) links failed, \(.timeouts) timed out.",
|
||||
"",
|
||||
([(.error_map | to_entries[]), (.timeout_map | to_entries[])]
|
||||
| group_by(.key)[] |
|
||||
"### Errors in \(.[0].key)",
|
||||
"",
|
||||
(map(.value[])[] | "* [\(.status.code // .status.text // "ERR")] <\(.url)> — \(.status.details // .status.text // "unknown error")"),
|
||||
"")
|
||||
' ./lychee/out.json
|
||||
} > ./lychee/issue.md
|
||||
|
||||
- name: Reopen report issue
|
||||
# A healthy run closes the report, and the issue action below only
|
||||
# rewrites the body of whatever number it is given. Without an
|
||||
# explicit reopen, the 2 -> 0 -> 2 sequence would keep rewriting a
|
||||
# closed issue while links are broken. A CLOSED state implies the
|
||||
# lookup found a canonical issue, so no separate emptiness check.
|
||||
if: env.EXIT_CODE == 2 && steps.report.outputs.state == 'CLOSED'
|
||||
env:
|
||||
ISSUE_NUMBER: ${{ steps.report.outputs.number }}
|
||||
run: |
|
||||
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
|
||||
gh issue reopen "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
|
||||
--comment "Broken documentation links found again in [the latest run]($run_url)."
|
||||
|
||||
- name: Report broken links
|
||||
if: env.EXIT_CODE == 2
|
||||
uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0
|
||||
with:
|
||||
# Empty on the first failing run, which creates the issue; afterwards
|
||||
# the same issue is updated in place.
|
||||
issue-number: ${{ steps.report.outputs.number }}
|
||||
title: ${{ env.REPORT_TITLE }}
|
||||
content-filepath: ./lychee/issue.md
|
||||
labels: documentation
|
||||
|
||||
- name: Close report issue once links are healthy
|
||||
# An OPEN state implies the lookup found a canonical issue; a report
|
||||
# that is already closed needs nothing.
|
||||
if: env.EXIT_CODE == 0 && steps.report.outputs.state == 'OPEN'
|
||||
env:
|
||||
ISSUE_NUMBER: ${{ steps.report.outputs.number }}
|
||||
run: |
|
||||
run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
|
||||
gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \
|
||||
--comment "All documentation links resolved in [the latest run]($run_url)."
|
||||
@@ -296,16 +296,18 @@ jobs:
|
||||
cargo update -p aws-types --precise 1.3.9
|
||||
cargo update -p aws-sigv4 --precise 1.3.5
|
||||
cargo update -p aws-credential-types --precise 1.2.8
|
||||
cargo update -p aws-smithy-checksums --precise 0.63.9
|
||||
# aws-smithy-checksums must stay at or above 0.63.13: OpenDAL's S3
|
||||
# service needs crc-fast ~1.9, and older releases pin it to ~1.3.
|
||||
cargo update -p aws-smithy-checksums --precise 0.63.13
|
||||
cargo update -p aws-smithy-runtime --precise 1.9.3
|
||||
cargo update -p aws-smithy-http --precise 0.62.4
|
||||
cargo update -p aws-smithy-eventstream --precise 0.60.12
|
||||
cargo update -p aws-smithy-http --precise 0.62.6
|
||||
cargo update -p aws-smithy-eventstream --precise 0.60.14
|
||||
cargo update -p aws-smithy-http-client --precise 1.1.3
|
||||
cargo update -p aws-smithy-observability --precise 0.1.4
|
||||
cargo update -p aws-smithy-query --precise 0.60.8
|
||||
cargo update -p aws-smithy-runtime-api --precise 1.9.1
|
||||
cargo update -p aws-smithy-async --precise 1.2.6
|
||||
cargo update -p aws-smithy-types --precise 1.3.5
|
||||
cargo update -p aws-smithy-runtime-api --precise 1.9.3
|
||||
cargo update -p aws-smithy-async --precise 1.2.7
|
||||
cargo update -p aws-smithy-types --precise 1.3.6
|
||||
cargo update -p aws-smithy-xml --precise 0.60.11
|
||||
cargo update -p home --precise 0.5.9
|
||||
- name: cargo +${{ matrix.msrv }} check
|
||||
|
||||
Generated
+258
-234
File diff suppressed because it is too large
Load Diff
+15
-15
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=11.0.0-beta.2", default-features = false, "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=11.0.0-beta.2", "tag" = "v11.0.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
@@ -52,7 +52,7 @@ env_logger = "0.11"
|
||||
half = { "version" = "2.7.1", default-features = false, features = [
|
||||
"num-traits",
|
||||
] }
|
||||
futures = "0"
|
||||
futures = "0.3"
|
||||
log = "0.4"
|
||||
metrics = "0.24"
|
||||
metrics-util = "0.19"
|
||||
|
||||
@@ -31,7 +31,7 @@ is also an [asynchronous API client](#connections-asynchronous).
|
||||
## Namespaces (Synchronous)
|
||||
|
||||
A namespace-backed connection resolves tables through a
|
||||
[Lance namespace](https://lancedb.github.io/lance-namespace/) service instead of
|
||||
[Lance namespace](https://lance-format.github.io/lance-namespace/) service instead of
|
||||
listing a storage directory.
|
||||
|
||||
::: lancedb.connect_namespace
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>10.1.0-beta.1</lance-core.version>
|
||||
<lance-core.version>11.0.0-beta.2</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
@@ -197,6 +197,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
expect(table.getChild("d")?.toJSON()).toEqual([9n, 10n, null]);
|
||||
});
|
||||
|
||||
it("will use a provided FixedSizeList schema with typed array values", function () {
|
||||
const schema = new Schema([
|
||||
new Field("text", new Utf8(), false),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(3, new Field("item", new Float32(), false)),
|
||||
false,
|
||||
),
|
||||
]);
|
||||
|
||||
const table = makeArrowTable(
|
||||
[
|
||||
{
|
||||
text: "foo",
|
||||
vector: new Float32Array([1, 2, 3]),
|
||||
},
|
||||
],
|
||||
{ schema },
|
||||
);
|
||||
|
||||
expect(table.getChild("text")?.toJSON()).toEqual(["foo"]);
|
||||
expect(
|
||||
table
|
||||
.getChild("vector")
|
||||
?.toJSON()
|
||||
.map((value) => value.toJSON()),
|
||||
).toEqual([[1, 2, 3]]);
|
||||
});
|
||||
|
||||
it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
|
||||
const schema = new Schema([
|
||||
new Field("a", new Float(Precision.DOUBLE), true),
|
||||
|
||||
@@ -11,8 +11,11 @@ import {
|
||||
Float16,
|
||||
Float32,
|
||||
Float64,
|
||||
Int32,
|
||||
Schema,
|
||||
Utf8,
|
||||
fromDataToBuffer,
|
||||
tableFromIPC,
|
||||
} from "../lancedb/arrow";
|
||||
import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
|
||||
import { getRegistry, register } from "../lancedb/embedding/registry";
|
||||
@@ -184,6 +187,63 @@ describe("embedding functions", () => {
|
||||
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
|
||||
expect(vector0).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it("should append generated vectors to a non-nullable schema", async () => {
|
||||
@register("non_nullable_schema_test")
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
ndims() {
|
||||
return 3;
|
||||
}
|
||||
embeddingDataType(): Float {
|
||||
return new Float64();
|
||||
}
|
||||
async computeSourceEmbeddings(data: string[]) {
|
||||
return data.map(() => [1, 2, 3]);
|
||||
}
|
||||
}
|
||||
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int32()),
|
||||
new Field("text", new Utf8()),
|
||||
new Field("type", new Utf8()),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(3, new Field("item", new Float64())),
|
||||
),
|
||||
]);
|
||||
const func = new MockEmbeddingFunction();
|
||||
const db = await connect(tmpDir.name);
|
||||
const table = await db.createEmptyTable("test_non_nullable", schema, {
|
||||
embeddingFunction: {
|
||||
function: func,
|
||||
sourceColumn: "text",
|
||||
},
|
||||
});
|
||||
|
||||
const data = [
|
||||
{ id: 1, text: "Carrot", type: "vegetable" },
|
||||
{ id: 2, text: "Apple", type: "fruit" },
|
||||
];
|
||||
const buffer = await fromDataToBuffer(
|
||||
data,
|
||||
undefined,
|
||||
await table.schema(),
|
||||
);
|
||||
const generatedTable = tableFromIPC(buffer);
|
||||
const vectorField = generatedTable.schema.fields.find(
|
||||
(field) => field.name === "vector",
|
||||
);
|
||||
expect(vectorField?.nullable).toBe(false);
|
||||
|
||||
await table.add(data);
|
||||
|
||||
const rows = await table.query().toArray();
|
||||
expect(rows).toHaveLength(2);
|
||||
for (const row of rows) {
|
||||
expect([...row.vector]).toEqual([1, 2, 3]);
|
||||
}
|
||||
});
|
||||
|
||||
it("should error when appending to a table with an unregistered embedding function", async () => {
|
||||
@register("mock")
|
||||
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import packageJson = require("../package.json");
|
||||
|
||||
describe("package metadata", () => {
|
||||
it("requires Node.js type declarations compatible with the runtime", () => {
|
||||
expect(packageJson.engines.node).toBe(">= 18");
|
||||
expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
|
||||
expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
|
||||
optional: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -110,6 +110,81 @@ describe("Query outputSchema", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("Search pagination", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), false),
|
||||
new Field("text", new Utf8(), false),
|
||||
new Field(
|
||||
"vector",
|
||||
new FixedSizeList(2, new Field("item", new Float32())),
|
||||
false,
|
||||
),
|
||||
]);
|
||||
const data = makeArrowTable(
|
||||
[
|
||||
{ id: 1n, text: "common", vector: [0, 0] },
|
||||
{ id: 2n, text: "common common", vector: [1, 1] },
|
||||
{ id: 3n, text: "common common common", vector: [2, 2] },
|
||||
{ id: 4n, text: "common common common common", vector: [3, 3] },
|
||||
],
|
||||
{ schema },
|
||||
);
|
||||
table = await db.createTable("test", data);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
tmpDir.removeCallback();
|
||||
});
|
||||
|
||||
it("applies offset after the vector search limit", async () => {
|
||||
const allResults = await table
|
||||
.vectorSearch([0, 0])
|
||||
.select(["id"])
|
||||
.limit(4)
|
||||
.toArray();
|
||||
const secondPage = await table
|
||||
.vectorSearch([0, 0])
|
||||
.select(["id"])
|
||||
.limit(2)
|
||||
.offset(2)
|
||||
.toArray();
|
||||
|
||||
expect(allResults).toHaveLength(4);
|
||||
expect(secondPage).toHaveLength(2);
|
||||
expect(secondPage.map((row) => row.id)).toEqual(
|
||||
allResults.slice(2, 4).map((row) => row.id),
|
||||
);
|
||||
});
|
||||
|
||||
it("applies offset after the full-text search limit", async () => {
|
||||
await table.createIndex("text", { config: Index.fts() });
|
||||
|
||||
const allResults = await table
|
||||
.search("common", "fts")
|
||||
.select(["id"])
|
||||
.limit(4)
|
||||
.toArray();
|
||||
const secondPage = await table
|
||||
.search("common", "fts")
|
||||
.select(["id"])
|
||||
.limit(2)
|
||||
.offset(2)
|
||||
.toArray();
|
||||
|
||||
expect(allResults).toHaveLength(4);
|
||||
expect(secondPage).toHaveLength(2);
|
||||
expect(secondPage.map((row) => row.id)).toEqual(
|
||||
allResults.slice(2, 4).map((row) => row.id),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Query orderBy", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
|
||||
@@ -170,6 +170,38 @@ describe("remote connection", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("surfaces JSON server errors from remote table operations", async () => {
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
const path = req.url ?? "";
|
||||
if (path.endsWith("/describe/")) {
|
||||
res.writeHead(200, { "Content-Type": "application/json" }).end(
|
||||
JSON.stringify({
|
||||
name: "broken_table",
|
||||
version: 1,
|
||||
schema: { fields: [] },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (path.endsWith("/count_rows/")) {
|
||||
res
|
||||
.writeHead(400, { "Content-Type": "application/json" })
|
||||
.end(JSON.stringify({ error: "count rows failed" }));
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404).end();
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("broken_table");
|
||||
|
||||
await expect(table.countRows()).rejects.toThrow("count rows failed");
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("should pass on requested extra headers", async () => {
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
|
||||
@@ -86,6 +86,44 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
await expect(table.countRows()).resolves.toBe(3);
|
||||
});
|
||||
|
||||
it("should support a foreign Float64 vector schema end to end", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const schema = new arrow.Schema([
|
||||
new arrow.Field("resource_id", new arrow.Int32(), false),
|
||||
new arrow.Field(
|
||||
"vector",
|
||||
new arrow.FixedSizeList(
|
||||
3,
|
||||
new arrow.Field("value", new arrow.Float64(), true),
|
||||
),
|
||||
false,
|
||||
),
|
||||
]);
|
||||
const data = [
|
||||
{
|
||||
// biome-ignore lint/style/useNamingConvention: matches the reported schema
|
||||
resource_id: 0,
|
||||
vector: [0.1, 0.1, 0.1],
|
||||
},
|
||||
];
|
||||
|
||||
const resources = await conn.createTable("resources", data, { schema });
|
||||
|
||||
const existing = await resources
|
||||
.query()
|
||||
.where("resource_id = 0")
|
||||
.limit(1)
|
||||
.toArray();
|
||||
expect(existing).toHaveLength(1);
|
||||
|
||||
const matched = await resources
|
||||
.search(Float64Array.from(data[0].vector))
|
||||
.limit(1)
|
||||
.toArray();
|
||||
expect(matched).toHaveLength(1);
|
||||
expect(matched[0]["resource_id"]).toBe(0);
|
||||
});
|
||||
|
||||
it("should support branches", async () => {
|
||||
await table.add([{ id: 1 }]);
|
||||
expect(await table.countRows()).toBe(1);
|
||||
|
||||
Generated
+6
@@ -55,7 +55,13 @@
|
||||
"openai": "4.29.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/node": ">=18",
|
||||
"apache-arrow": ">=15.0.0 <=18.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@types/node": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@aws-crypto/crc32": {
|
||||
|
||||
@@ -101,6 +101,12 @@
|
||||
"openai": "4.29.2"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/node": ">=18",
|
||||
"apache-arrow": ">=15.0.0 <=18.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@types/node": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -26,7 +26,7 @@ lance-namespace-impls.workspace = true
|
||||
lance-io.workspace = true
|
||||
env_logger.workspace = true
|
||||
log.workspace = true
|
||||
pyo3 = { version = "0.28", features = ["extension-module", "abi3-py39", "chrono"] }
|
||||
pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
pyo3-async-runtimes = { version = "0.28", features = [
|
||||
"attributes",
|
||||
@@ -43,7 +43,7 @@ libc = "0.2"
|
||||
[build-dependencies]
|
||||
pyo3-build-config = { version = "0.28", features = [
|
||||
"extension-module",
|
||||
"abi3-py39",
|
||||
"abi3-py310",
|
||||
] }
|
||||
|
||||
[features]
|
||||
|
||||
@@ -60,7 +60,7 @@ tests = [
|
||||
"pytest-asyncio>=0.21",
|
||||
"duckdb>=0.9.0",
|
||||
"pytz>=2023.3",
|
||||
"polars>=0.19, <=1.3.0",
|
||||
"polars>=0.19, <=1.32.3",
|
||||
"pyarrow<25",
|
||||
"pyarrow-stubs>=16.0",
|
||||
"pylance==9.0.0rc1",
|
||||
@@ -140,6 +140,7 @@ include = [
|
||||
"python/lancedb/remote/errors.py",
|
||||
"python/lancedb/embeddings/__init__.py",
|
||||
"python/lancedb/_lancedb.pyi",
|
||||
"python/type_tests/connect.py",
|
||||
]
|
||||
exclude = ["python/tests/"]
|
||||
pythonVersion = "3.13"
|
||||
|
||||
@@ -356,6 +356,10 @@ class Table:
|
||||
async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
|
||||
async def unset_lsm_write_spec(self) -> None: ...
|
||||
async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
|
||||
async def checkpoint_lsm(self) -> None: ...
|
||||
async def flush_lsm(self) -> None: ...
|
||||
async def compact_lsm(self) -> None: ...
|
||||
async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ...
|
||||
async def close_lsm_writers(self) -> None: ...
|
||||
@property
|
||||
def tags(self) -> Tags: ...
|
||||
|
||||
@@ -707,6 +707,9 @@ class LanceDBConnection(DBConnection):
|
||||
self._namespace_client_properties = namespace_client_properties
|
||||
if _inner is not None:
|
||||
self._conn = _inner
|
||||
# Native-derived wrappers resolve this in their async reconstruction
|
||||
# path so construction never synchronously re-enters LOOP.
|
||||
self._read_consistency_interval = read_consistency_interval
|
||||
self._cached_namespace_client = None
|
||||
return
|
||||
|
||||
@@ -756,11 +759,14 @@ class LanceDBConnection(DBConnection):
|
||||
# storage_options. Also, this class really shouldn't be holding any state
|
||||
# beyond _conn.
|
||||
self._conn = AsyncConnection(LOOP.run(do_connect()))
|
||||
# Keep property access synchronous so debugger introspection cannot wait on
|
||||
# the background loop while that thread is suspended at a breakpoint.
|
||||
self._read_consistency_interval = read_consistency_interval
|
||||
self._cached_namespace_client: Optional[LanceNamespace] = None
|
||||
|
||||
@property
|
||||
def read_consistency_interval(self) -> Optional[timedelta]:
|
||||
return LOOP.run(self._conn.get_read_consistency_interval())
|
||||
return self._read_consistency_interval
|
||||
|
||||
@property
|
||||
def session(self) -> Optional[Session]:
|
||||
@@ -771,8 +777,16 @@ class LanceDBConnection(DBConnection):
|
||||
return self._conn.uri
|
||||
|
||||
@classmethod
|
||||
def from_inner(cls, inner: LanceDbConnection):
|
||||
return cls(None, _inner=inner)
|
||||
def from_inner(
|
||||
cls,
|
||||
inner: LanceDbConnection,
|
||||
read_consistency_interval: Optional[timedelta],
|
||||
):
|
||||
return cls(
|
||||
None,
|
||||
read_consistency_interval=read_consistency_interval,
|
||||
_inner=inner,
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
|
||||
|
||||
@@ -101,8 +101,7 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
|
||||
|
||||
@weak_lru(maxsize=1)
|
||||
def ndims(self):
|
||||
model = self.get_model()
|
||||
return model.encode("foo").shape[0]
|
||||
return len(self.generate_embeddings([[self.source_instruction, "foo"]])[0])
|
||||
|
||||
def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
|
||||
return self.generate_embeddings([[self.query_instruction, query]])
|
||||
|
||||
@@ -226,7 +226,7 @@ class PermutationBuilder:
|
||||
|
||||
async def do_execute():
|
||||
inner_tbl = await self._async.execute()
|
||||
return LanceTable.from_inner(inner_tbl)
|
||||
return await LanceTable.from_inner(inner_tbl)
|
||||
|
||||
return LOOP.run(do_execute())
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -153,6 +153,16 @@ def Vector(
|
||||
return FixedSizeList
|
||||
|
||||
|
||||
def _raise_bare_vector_error(*_args):
|
||||
raise TypeError("Vector must be parameterized with a dimension, e.g. Vector(128).")
|
||||
|
||||
|
||||
# Pydantic v1 and v2 otherwise treat the bare Vector factory as a field validator
|
||||
# and inspect its signature, which produces misleading errors about internal types.
|
||||
setattr(Vector, "__get_validators__", _raise_bare_vector_error)
|
||||
setattr(Vector, "__get_pydantic_core_schema__", _raise_bare_vector_error)
|
||||
|
||||
|
||||
def MultiVector(
|
||||
dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True
|
||||
) -> Type:
|
||||
|
||||
@@ -109,6 +109,11 @@ def _should_push_down_query_table(
|
||||
return namespace_client is not None and "QueryTable" in pushdown_operations
|
||||
|
||||
|
||||
def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
|
||||
"""Return a Polars frame unchanged while blocking predicate pushdown."""
|
||||
return frame
|
||||
|
||||
|
||||
_MODEL_BACKED_TOKENIZER_PREFIXES = ("jieba", "lindera")
|
||||
_MODEL_BACKED_TOKENIZER_ERRORS = (
|
||||
"unknown base tokenizer",
|
||||
@@ -928,12 +933,18 @@ class Table(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def to_polars(self, **kwargs) -> "pl.DataFrame":
|
||||
"""Return the table as a polars.DataFrame.
|
||||
def to_polars(self, **kwargs) -> "pl.LazyFrame":
|
||||
"""Return the table as a Polars LazyFrame.
|
||||
|
||||
Note
|
||||
----
|
||||
The Polars streaming engine is not supported because it does not currently
|
||||
implement Python PyArrow dataset scans. Use the default engine when collecting
|
||||
this LazyFrame.
|
||||
|
||||
Returns
|
||||
-------
|
||||
polars.DataFrame
|
||||
polars.LazyFrame
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -2251,11 +2262,15 @@ class LanceTable(Table):
|
||||
return self.name
|
||||
|
||||
@classmethod
|
||||
def from_inner(cls, tbl: LanceDBTable):
|
||||
from .db import LanceDBConnection
|
||||
async def from_inner(cls, tbl: LanceDBTable):
|
||||
from .db import AsyncConnection, LanceDBConnection
|
||||
|
||||
async_tbl = AsyncTable(tbl)
|
||||
conn = LanceDBConnection.from_inner(tbl.database())
|
||||
inner_conn = tbl.database()
|
||||
read_consistency_interval = await AsyncConnection(
|
||||
inner_conn
|
||||
).get_read_consistency_interval()
|
||||
conn = LanceDBConnection.from_inner(inner_conn, read_consistency_interval)
|
||||
return cls(
|
||||
conn,
|
||||
async_tbl.name,
|
||||
@@ -2634,6 +2649,9 @@ class LanceTable(Table):
|
||||
2. Currently we've disabled push-down of the filters from polars
|
||||
because polars pushdown into pyarrow uses pyarrow compute
|
||||
expressions rather than SQl strings (which LanceDB supports)
|
||||
3. The Polars streaming engine is not supported because it does not
|
||||
currently implement Python PyArrow dataset scans. Use the default
|
||||
engine when collecting this LazyFrame.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -2642,8 +2660,12 @@ class LanceTable(Table):
|
||||
from lancedb.integrations.pyarrow import PyarrowDatasetAdapter
|
||||
|
||||
dataset = PyarrowDatasetAdapter(self)
|
||||
return pl.scan_pyarrow_dataset(
|
||||
dataset, allow_pyarrow_filter=False, batch_size=batch_size
|
||||
# Polars 1.32's non-PyArrow callback path passes batch_size twice. Keep
|
||||
# the compatible PyArrow path, but block predicates because this adapter
|
||||
# cannot translate PyArrow expressions into LanceDB filters.
|
||||
return pl.scan_pyarrow_dataset(dataset, batch_size=batch_size).map_batches(
|
||||
_polars_predicate_pushdown_barrier,
|
||||
predicate_pushdown=False,
|
||||
)
|
||||
|
||||
# New unified API overload
|
||||
@@ -4029,6 +4051,28 @@ class LanceTable(Table):
|
||||
[`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec]."""
|
||||
return LOOP.run(self._table.get_lsm_write_spec())
|
||||
|
||||
def checkpoint_lsm(self) -> None:
|
||||
"""Synchronous version of
|
||||
[`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
|
||||
return LOOP.run(self._table.checkpoint_lsm())
|
||||
|
||||
def flush_lsm(self) -> None:
|
||||
"""Synchronous version of
|
||||
[`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm]."""
|
||||
return LOOP.run(self._table.flush_lsm())
|
||||
|
||||
def compact_lsm(self) -> None:
|
||||
"""Synchronous version of
|
||||
[`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
|
||||
return LOOP.run(self._table.compact_lsm())
|
||||
|
||||
def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
|
||||
"""Synchronous version of
|
||||
[`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
|
||||
return LOOP.run(
|
||||
self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
|
||||
)
|
||||
|
||||
def close_lsm_writers(self) -> None:
|
||||
"""Close cached MemWAL shard writers. See
|
||||
[`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers]."""
|
||||
@@ -4739,6 +4783,67 @@ class AsyncTable:
|
||||
"""
|
||||
return await self._inner.get_lsm_write_spec()
|
||||
|
||||
async def checkpoint_lsm(self) -> None:
|
||||
"""Converge this table's LSM write path into its base table.
|
||||
|
||||
One flush, sealing every memtable into L0, then compaction triggers
|
||||
until every generation that existed at that moment has reached base.
|
||||
The loop runs client-side, reading progress from ``get_lsm_stats``.
|
||||
|
||||
Best-effort: generations created *while* it runs are deliberately not
|
||||
waited on, which is what lets it terminate on a table taking writes.
|
||||
Idempotent and safe on a cadence.
|
||||
|
||||
There is no deadline, and the caller owns that. It returns when the
|
||||
target generations are gone, raises on a terminal server fault, and
|
||||
otherwise waits however long the server takes. A slow table and a
|
||||
stuck one are the same picture from the client: the compactor pool is
|
||||
shared across every table on the node, so a checkpoint queued behind
|
||||
unrelated work looks exactly like one that is merging. Wrap this in
|
||||
``asyncio.wait_for`` for a wall-clock bound; abandoning it partway
|
||||
costs nothing.
|
||||
"""
|
||||
return await self._inner.checkpoint_lsm()
|
||||
|
||||
async def flush_lsm(self) -> None:
|
||||
"""Seal every bucket's active memtable into L0.
|
||||
|
||||
Does not touch the base table — moving L0 into base is
|
||||
`compact_lsm`. On a node that has not claimed this table, this claims
|
||||
it and replays its WAL log first.
|
||||
"""
|
||||
return await self._inner.flush_lsm()
|
||||
|
||||
async def compact_lsm(self) -> None:
|
||||
"""Trigger a background L0 to base compaction pass per bucket.
|
||||
|
||||
Returns once the passes are dispatched, not once they finish: watch
|
||||
``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop
|
||||
until the current L0 has reached base.
|
||||
"""
|
||||
return await self._inner.compact_lsm()
|
||||
|
||||
async def get_lsm_stats(
|
||||
self, *, include_generation_rows: bool = False
|
||||
) -> Optional[dict]:
|
||||
"""Read live per-bucket LSM state.
|
||||
|
||||
Answers "how far behind is my fresh tier", "which bucket is hot", and
|
||||
"why is my fresh-tier vector search brute-force". Mutates no table
|
||||
state, though on a node that has not claimed this table it claims it,
|
||||
exactly as a read would.
|
||||
|
||||
Returns ``None`` only when the LSM write path is not enabled.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
include_generation_rows
|
||||
Report a row count per L0 generation. Off by default: each count
|
||||
opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this
|
||||
needing only generation numbers.
|
||||
"""
|
||||
return await self._inner.get_lsm_stats(include_generation_rows)
|
||||
|
||||
async def close_lsm_writers(self) -> None:
|
||||
"""Drain and close any cached MemWAL shard writers for this table.
|
||||
|
||||
|
||||
@@ -395,6 +395,11 @@ def _(value: dict):
|
||||
)
|
||||
|
||||
|
||||
@value_to_sql.register(pa.Scalar)
|
||||
def _(value: pa.Scalar):
|
||||
return value_to_sql(value.as_py())
|
||||
|
||||
|
||||
@value_to_sql.register(np.ndarray)
|
||||
def _(value: np.ndarray):
|
||||
return value_to_sql(value.tolist())
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
|
||||
import inspect
|
||||
import re
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from importlib import resources
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -17,6 +19,10 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
|
||||
|
||||
def test_package_includes_pep_561_marker():
|
||||
assert resources.files(lancedb).joinpath("py.typed").is_file()
|
||||
|
||||
|
||||
def test_basic(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
|
||||
@@ -62,21 +68,44 @@ def test_basic(tmp_path):
|
||||
assert db.open_table("test").name == db["test"].name
|
||||
|
||||
|
||||
def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch):
|
||||
def test_sync_debugger_inspection_does_not_use_background_loop(tmp_path, monkeypatch):
|
||||
from lancedb.background_loop import LOOP
|
||||
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("test", data=[{"id": 1}])
|
||||
|
||||
def fail_run(*args, **kwargs):
|
||||
raise AssertionError("repr should not use the Python background loop")
|
||||
raise AssertionError("debugger inspection should not use the background loop")
|
||||
|
||||
monkeypatch.setattr(LOOP, "run", fail_run)
|
||||
|
||||
# Debuggers enumerate and evaluate every exposed attribute when expanding a
|
||||
# variable. This must remain safe while their breakpoint suspends LOOP's thread.
|
||||
members = dict(inspect.getmembers(db))
|
||||
|
||||
assert members["uri"] == str(tmp_path)
|
||||
assert members["read_consistency_interval"] is None
|
||||
assert repr(db) == f"LanceDBConnection(uri={str(tmp_path)!r})"
|
||||
assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
|
||||
|
||||
|
||||
def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkeypatch):
|
||||
from lancedb.background_loop import LOOP
|
||||
from lancedb.db import LanceDBConnection
|
||||
|
||||
consistency_interval = timedelta(seconds=5)
|
||||
db = lancedb.connect(tmp_path, read_consistency_interval=consistency_interval)
|
||||
db_from_inner = LanceDBConnection.from_inner(db._inner, consistency_interval)
|
||||
|
||||
def fail_run(*args, **kwargs):
|
||||
raise AssertionError("properties should not use the Python background loop")
|
||||
|
||||
monkeypatch.setattr(LOOP, "run", fail_run)
|
||||
|
||||
assert db.read_consistency_interval == consistency_interval
|
||||
assert db_from_inner.read_consistency_interval == consistency_interval
|
||||
|
||||
|
||||
def test_ingest_pd(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
|
||||
|
||||
@@ -64,6 +64,23 @@ def test_embedding_function(tmp_path):
|
||||
assert np.allclose(actual, expected)
|
||||
|
||||
|
||||
def test_instructor_ndims_uses_instruction():
|
||||
instructor = get_registry().get("instructor").create()
|
||||
model = MagicMock()
|
||||
model.encode.return_value = np.zeros((1, 384))
|
||||
|
||||
with patch.object(type(instructor), "get_model", return_value=model):
|
||||
assert instructor.ndims() == 384
|
||||
|
||||
model.encode.assert_called_once_with(
|
||||
[[instructor.source_instruction, "foo"]],
|
||||
batch_size=instructor.batch_size,
|
||||
show_progress_bar=instructor.show_progress_bar,
|
||||
normalize_embeddings=instructor.normalize_embeddings,
|
||||
device=instructor.device,
|
||||
)
|
||||
|
||||
|
||||
def test_embedding_function_variables():
|
||||
@register("variable-testing")
|
||||
class VariableTestingFunction(TextEmbeddingFunction):
|
||||
@@ -115,34 +132,16 @@ def test_embedding_function_variables():
|
||||
assert func.safe_model_dump()["secret_key"] == "$var:secret"
|
||||
|
||||
|
||||
def test_parse_functions_with_variables():
|
||||
@register("variable-parsing-test")
|
||||
class VariableParsingFunction(TextEmbeddingFunction):
|
||||
api_key: str
|
||||
base_url: Optional[str] = None
|
||||
|
||||
@staticmethod
|
||||
def sensitive_keys():
|
||||
return ["api_key"]
|
||||
|
||||
def ndims(self):
|
||||
return 10
|
||||
|
||||
def generate_embeddings(self, texts):
|
||||
# Mock implementation that just returns random embeddings
|
||||
# In real usage, this would use the api_key to call an API
|
||||
return [np.random.rand(self.ndims()).tolist() for _ in texts]
|
||||
|
||||
def test_openai_variables_survive_metadata_round_trip():
|
||||
registry = EmbeddingFunctionRegistry.get_instance()
|
||||
|
||||
registry.set_var("test_api_key", "sk-test-key-12345")
|
||||
registry.set_var("test_base_url", "https://api.example.com")
|
||||
|
||||
conf = EmbeddingFunctionConfig(
|
||||
source_column="text",
|
||||
vector_column="vector",
|
||||
function=registry.get("variable-parsing-test").create(
|
||||
api_key="$var:test_api_key", base_url="$var:test_base_url"
|
||||
function=registry.get("openai").create(
|
||||
api_key="$var:test_api_key", base_url="https://api.example.com"
|
||||
),
|
||||
)
|
||||
|
||||
@@ -150,7 +149,10 @@ def test_parse_functions_with_variables():
|
||||
|
||||
# Create a mock arrow table with the metadata
|
||||
schema = pa.schema(
|
||||
[pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))]
|
||||
[
|
||||
pa.field("text", pa.string()),
|
||||
pa.field("vector", pa.list_(pa.float32(), 1536)),
|
||||
]
|
||||
)
|
||||
table = pa.table({"text": [], "vector": []}, schema=schema)
|
||||
table = table.replace_schema_metadata(metadata)
|
||||
@@ -164,13 +166,15 @@ def test_parse_functions_with_variables():
|
||||
|
||||
assert parsed_func.api_key == "sk-test-key-12345"
|
||||
assert parsed_func.base_url == "https://api.example.com"
|
||||
|
||||
embeddings = parsed_func.generate_embeddings(["test text"])
|
||||
assert len(embeddings) == 1
|
||||
assert len(embeddings[0]) == 10
|
||||
|
||||
assert parsed_func.safe_model_dump()["api_key"] == "$var:test_api_key"
|
||||
|
||||
with patch("lancedb.embeddings.openai.attempt_import_or_raise") as import_openai:
|
||||
parsed_func._openai_client
|
||||
|
||||
import_openai.return_value.OpenAI.assert_called_once_with(
|
||||
api_key="sk-test-key-12345", base_url="https://api.example.com"
|
||||
)
|
||||
|
||||
|
||||
def test_embedding_with_bad_results(tmp_path):
|
||||
@register("null-embedding")
|
||||
|
||||
@@ -12,7 +12,7 @@ import pyarrow.compute as pc
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from lancedb.index import FTS
|
||||
from lancedb.index import BTree, FTS, IvfPq
|
||||
from lancedb.table import AsyncTable, Table
|
||||
|
||||
|
||||
@@ -99,6 +99,86 @@ async def test_async_hybrid_query_filters(table: AsyncTable):
|
||||
assert result["text"].to_pylist() == ["cat", "b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hybrid_query_with_stale_fixed_size_binary_prefilter(
|
||||
tmpdir_factory,
|
||||
):
|
||||
tmp_path = str(tmpdir_factory.mktemp("stale_scalar_prefilter"))
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
|
||||
def fixed_size_binary(value: int) -> bytes:
|
||||
return value.to_bytes(16, byteorder="big")
|
||||
|
||||
num_rows = 1000
|
||||
data = pa.table(
|
||||
{
|
||||
"space_id": pa.array(
|
||||
[fixed_size_binary(i) for i in range(num_rows)],
|
||||
type=pa.binary(16),
|
||||
),
|
||||
"text": ["book"] * num_rows,
|
||||
"vector": pa.array(
|
||||
[[float(i), float(i)] for i in range(num_rows)],
|
||||
type=pa.list_(pa.float32(), 2),
|
||||
),
|
||||
}
|
||||
)
|
||||
table = await db.create_table("test", data)
|
||||
await table.create_index(
|
||||
"vector", config=IvfPq(num_partitions=4, num_sub_vectors=2)
|
||||
)
|
||||
await table.create_index("space_id", config=BTree())
|
||||
await table.create_index("text", config=FTS(with_position=False))
|
||||
|
||||
# Advance the search indices without advancing the scalar index. This is the
|
||||
# state that previously let hybrid search use an incomplete scalar prefilter.
|
||||
await table.add(data)
|
||||
lance_dataset = await table.to_lance()
|
||||
lance_dataset.optimize.optimize_indices(index_names=["vector_idx", "text_idx"])
|
||||
await table.checkout_latest()
|
||||
|
||||
scalar_stats = await table.index_stats("space_id_idx")
|
||||
assert scalar_stats is not None
|
||||
assert scalar_stats.num_indexed_rows == num_rows
|
||||
assert scalar_stats.num_unindexed_rows == num_rows
|
||||
|
||||
for index_name in ["vector_idx", "text_idx"]:
|
||||
search_stats = await table.index_stats(index_name)
|
||||
assert search_stats is not None
|
||||
assert search_stats.num_indexed_rows == num_rows * 2
|
||||
assert search_stats.num_unindexed_rows == 0
|
||||
|
||||
matching_ids = [5, 10, 15, 20, 25, 30]
|
||||
literals = [
|
||||
f"arrow_cast(0x{fixed_size_binary(i).hex()}, 'FixedSizeBinary(16)')"
|
||||
for i in matching_ids
|
||||
]
|
||||
predicate = f"space_id IN ({', '.join(literals)})"
|
||||
expected_ids = sorted(fixed_size_binary(i) for i in matching_ids for _ in range(2))
|
||||
|
||||
vector_query = (
|
||||
table.query().where(predicate).nearest_to([5.0, 5.0]).limit(num_rows * 2)
|
||||
)
|
||||
vector_results = await vector_query.to_arrow()
|
||||
assert sorted(vector_results["space_id"].to_pylist()) == expected_ids
|
||||
|
||||
fts_query = (
|
||||
table.query().where(predicate).nearest_to_text("book").limit(num_rows * 2)
|
||||
)
|
||||
fts_results = await fts_query.to_arrow()
|
||||
assert sorted(fts_results["space_id"].to_pylist()) == expected_ids
|
||||
|
||||
hybrid_results = await (
|
||||
table.query()
|
||||
.where(predicate)
|
||||
.nearest_to([5.0, 5.0])
|
||||
.nearest_to_text("book")
|
||||
.limit(num_rows * 2)
|
||||
.to_arrow()
|
||||
)
|
||||
assert sorted(hybrid_results["space_id"].to_pylist()) == expected_ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_hybrid_query_default_limit(table: AsyncTable):
|
||||
# add 10 new rows
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import lancedb._lancedb as _lancedb
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="ldd is Linux-specific")
|
||||
def test_native_extension_does_not_link_openssl():
|
||||
"""OpenSSL-linked wheels abort when imported on RHEL hosts in FIPS mode."""
|
||||
ldd = shutil.which("ldd")
|
||||
if ldd is None:
|
||||
pytest.skip("ldd is not installed")
|
||||
|
||||
result = subprocess.run(
|
||||
[ldd, _lancedb.__file__],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
openssl_libraries = re.findall(
|
||||
r"^\s*(lib(?:crypto|ssl)\S*)\s+=>", result.stdout, flags=re.MULTILINE
|
||||
)
|
||||
|
||||
assert not openssl_libraries, (
|
||||
"the LanceDB native extension must use rustls instead of linking OpenSSL: "
|
||||
f"{openssl_libraries}"
|
||||
)
|
||||
@@ -372,6 +372,31 @@ async def test_create_vector_index(some_table: AsyncTable):
|
||||
assert stats.num_indices == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_ivf_index_reports_unsplittable_partitions(db_async):
|
||||
dim = 8
|
||||
num_partitions = 300 # More than 256 selects hierarchical k-means.
|
||||
base_vectors = [[float(row == column) for column in range(dim)] for row in range(5)]
|
||||
vectors = pa.array(base_vectors * 200, pa.list_(pa.float32(), dim))
|
||||
table = await db_async.create_table(
|
||||
"unsplittable_partitions",
|
||||
pa.table({"vector": vectors}),
|
||||
)
|
||||
|
||||
error_pattern = (
|
||||
rf"Cannot create {num_partitions} IVF partitions: k-means could only form"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match=error_pattern):
|
||||
await table.create_index(
|
||||
"vector",
|
||||
config=IvfFlat(
|
||||
distance_type="dot",
|
||||
num_partitions=num_partitions,
|
||||
max_iterations=10,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_4bit_ivfpq_index(some_table: AsyncTable):
|
||||
# Can create
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import importlib
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_pyo3_abi_matches_minimum_supported_python():
|
||||
project_dir = Path(__file__).parents[2]
|
||||
pyproject = (project_dir / "pyproject.toml").read_text()
|
||||
cargo_manifest = (project_dir / "Cargo.toml").read_text()
|
||||
|
||||
minimum_python = re.search(
|
||||
r'^requires-python\s*=\s*">=(\d+)\.(\d+)"$', pyproject, re.MULTILINE
|
||||
)
|
||||
assert minimum_python is not None
|
||||
|
||||
major, minor = minimum_python.groups()
|
||||
expected_abi = f"abi3-py{major}{minor}"
|
||||
configured_abis = re.findall(r'"(abi3-py\d+)"', cargo_manifest)
|
||||
|
||||
assert configured_abis == [expected_abi, expected_abi], (
|
||||
"the pyo3 runtime and build ABI features must both match requires-python"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="Windows wheel regression test")
|
||||
def test_windows_wheel_tag_and_native_import():
|
||||
project_dir = Path(__file__).parents[2]
|
||||
wheels = list((project_dir.parent / "target" / "wheels").glob("lancedb-*.whl"))
|
||||
if not wheels:
|
||||
pytest.skip("no wheel artifact is available in this development environment")
|
||||
|
||||
assert len(wheels) == 1
|
||||
assert wheels[0].name.endswith("-cp310-abi3-win_amd64.whl")
|
||||
|
||||
native_module = importlib.import_module("lancedb._lancedb")
|
||||
assert Path(native_module.__file__).suffix == ".pyd"
|
||||
@@ -6,6 +6,7 @@ import math
|
||||
import pytest
|
||||
|
||||
from lancedb import DBConnection, Table, connect
|
||||
from lancedb.background_loop import LOOP
|
||||
from lancedb.permutation import Permutation, Permutations, permutation_builder
|
||||
|
||||
|
||||
@@ -31,6 +32,25 @@ def test_split_random_ratios(mem_db):
|
||||
assert 65 <= split_1_count <= 75 # ~70% ± tolerance
|
||||
|
||||
|
||||
def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
|
||||
import threading
|
||||
|
||||
db = connect(tmp_path)
|
||||
tbl = db.create_table("test_table", pa.table({"x": range(10)}))
|
||||
original_run = LOOP.run
|
||||
|
||||
def fail_on_reentry(future):
|
||||
assert threading.current_thread() is not LOOP.thread
|
||||
return original_run(future)
|
||||
|
||||
monkeypatch.setattr(LOOP, "run", fail_on_reentry)
|
||||
|
||||
permutation_tbl = permutation_builder(tbl).execute()
|
||||
|
||||
assert permutation_tbl.count_rows() == 10
|
||||
assert permutation_tbl._conn.read_consistency_interval is None
|
||||
|
||||
|
||||
def test_split_random_counts(mem_db):
|
||||
"""Test random splitting with absolute counts."""
|
||||
tbl = mem_db.create_table(
|
||||
|
||||
@@ -415,6 +415,17 @@ def test_nullable_vector():
|
||||
assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)])
|
||||
|
||||
|
||||
def test_bare_vector_raises_clear_error():
|
||||
namespace = {
|
||||
"__name__": "test_model_without_pyarrow",
|
||||
"LanceModel": LanceModel,
|
||||
"Vector": Vector,
|
||||
}
|
||||
|
||||
with pytest.raises(TypeError, match=r"Vector must be parameterized.*Vector\(128\)"):
|
||||
exec("class TestModel(LanceModel):\n vector: Vector", namespace)
|
||||
|
||||
|
||||
def test_fixed_size_list_field():
|
||||
class TestModel(pydantic.BaseModel):
|
||||
vec: Vector(16)
|
||||
|
||||
@@ -570,6 +570,15 @@ def test_query_builder(table):
|
||||
assert all(np.array(rs[0]["vector"]) == [1, 2])
|
||||
|
||||
|
||||
def test_query_multiple_vectors(table):
|
||||
results = table.search([np.array([1, 2]), np.array([4, 5])]).limit(1).to_list()
|
||||
|
||||
assert len(results) == 2
|
||||
results_by_query = {result["query_index"]: result for result in results}
|
||||
assert results_by_query[0]["id"] == 1
|
||||
assert results_by_query[1]["id"] == 2
|
||||
|
||||
|
||||
def test_with_row_id(table: lancedb.table.Table):
|
||||
rs = table.search().with_row_id(True).to_arrow()
|
||||
assert "_rowid" in rs.column_names
|
||||
|
||||
@@ -35,6 +35,12 @@ def make_mock_http_handler(handler):
|
||||
return MockLanceDBHandler
|
||||
|
||||
|
||||
@pytest.mark.parametrize("db_name", ["a" * 64, "invalid..database"])
|
||||
def test_connect_rejects_invalid_cloud_dns_hostname(db_name):
|
||||
with pytest.raises(ValueError, match="DNS labels must contain 1 to 63 bytes"):
|
||||
lancedb.connect(f"db://{db_name}", api_key="fake")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def mock_lancedb_connection(handler):
|
||||
with http.server.HTTPServer(
|
||||
|
||||
@@ -2,10 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
|
||||
import ctypes
|
||||
import gc
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import warnings
|
||||
import weakref
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, datetime, timedelta
|
||||
from time import sleep
|
||||
from typing import List
|
||||
@@ -98,6 +102,30 @@ def test_basic(mem_db: DBConnection):
|
||||
assert table.to_arrow() == expected_data
|
||||
|
||||
|
||||
def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection):
|
||||
data = pa.table(
|
||||
{
|
||||
"id": [0, 1, 2, 3, 4],
|
||||
"score_cn": [None, 22, None, 5, 8],
|
||||
"score_mt": [None, 42, None, 5, 8],
|
||||
"vector": [
|
||||
[20, 19, -1, -1],
|
||||
[41, 38, 22, 42],
|
||||
[10, 10, -1, -1],
|
||||
[5, 5, 5, 5],
|
||||
[8, 8, 8, 8],
|
||||
],
|
||||
}
|
||||
).slice(1)
|
||||
|
||||
table = mem_db.create_table("sliced_nullable", data=data)
|
||||
result = table.search([41, 38, 22, 42]).limit(1).to_arrow()
|
||||
|
||||
assert result["id"].to_pylist() == [1]
|
||||
assert result["score_cn"].to_pylist() == [22]
|
||||
assert result["score_mt"].to_pylist() == [42]
|
||||
|
||||
|
||||
def test_table_to_pandas_default_matches_arrow(tmp_db: DBConnection):
|
||||
pd = pytest.importorskip("pandas")
|
||||
data = pa.table({"id": [1, 2], "text": ["one", "two"]})
|
||||
@@ -434,6 +462,38 @@ def test_add(mem_db: DBConnection):
|
||||
_add(table, schema)
|
||||
|
||||
|
||||
def test_add_releases_arrow_buffers_without_gc(mem_db: DBConnection):
|
||||
"""Regression test for https://github.com/lancedb/lancedb/issues/2512."""
|
||||
schema = pa.schema([pa.field("x", pa.int64())])
|
||||
table = mem_db.create_table("test_add_releases_arrow_buffers", schema=schema)
|
||||
|
||||
class BufferOwner:
|
||||
def __init__(self, size: int):
|
||||
self.memory = ctypes.create_string_buffer(size)
|
||||
|
||||
owner_refs = []
|
||||
gc_was_enabled = gc.isenabled()
|
||||
gc.disable()
|
||||
try:
|
||||
for _ in range(3):
|
||||
size = 8 * 1024
|
||||
owner = BufferOwner(size)
|
||||
arrow_buffer = pa.foreign_buffer(
|
||||
ctypes.addressof(owner.memory), size, owner
|
||||
)
|
||||
array = pa.Array.from_buffers(pa.int64(), 1024, [None, arrow_buffer])
|
||||
batch = pa.RecordBatch.from_arrays([array], schema=schema)
|
||||
owner_refs.append(weakref.ref(owner))
|
||||
|
||||
table.add(batch)
|
||||
del batch, array, arrow_buffer, owner
|
||||
|
||||
assert all(owner_ref() is None for owner_ref in owner_refs)
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
|
||||
|
||||
def test_add_write_parallelism(mem_db: DBConnection):
|
||||
schema = pa.schema([pa.field("id", pa.int64())])
|
||||
table = mem_db.create_table("test", schema=schema)
|
||||
@@ -869,6 +929,7 @@ def test_polars(mem_db: DBConnection):
|
||||
|
||||
# enter table to polars dataframe
|
||||
result = table.to_polars()
|
||||
assert isinstance(result, pl.LazyFrame)
|
||||
assert np.allclose(result.collect()["vector"].to_list(), data["vector"])
|
||||
|
||||
# make sure filtering isn't broken
|
||||
@@ -1785,6 +1846,27 @@ def test_add_with_empty_fixed_size_list_drops_bad_rows(mem_db: DBConnection):
|
||||
assert np.allclose(data["embedding"].to_pylist()[0], np.array([0.1] * 16))
|
||||
|
||||
|
||||
def test_add_nullable_fixed_size_list_with_none(mem_db: DBConnection):
|
||||
"""Regression test for issue #2340."""
|
||||
table = mem_db.create_table(
|
||||
"test_nullable_fixed_size_list",
|
||||
schema=pa.schema(
|
||||
[
|
||||
pa.field("id", pa.string()),
|
||||
pa.field("feature", pa.list_(pa.float32(), 256)),
|
||||
pa.field("tags", pa.list_(pa.string())),
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
table.add([{"id": "1", "feature": None, "tags": ["tag1", "tag2"]}])
|
||||
|
||||
result = table.to_arrow()
|
||||
assert result.to_pylist() == [
|
||||
{"id": "1", "feature": None, "tags": ["tag1", "tag2"]}
|
||||
]
|
||||
|
||||
|
||||
def test_add_nullable_struct_with_none(mem_db: DBConnection):
|
||||
"""Regression test for issue #2654: a nullable struct column whose
|
||||
first batch contains only None values must not crash in
|
||||
@@ -1824,6 +1906,33 @@ def test_add_nullable_struct_with_none(mem_db: DBConnection):
|
||||
assert result.column("data").to_pylist() == [{"x": 1.0}, None]
|
||||
|
||||
|
||||
def test_read_mostly_null_list_v2_2_page_boundary(tmp_path):
|
||||
# Regression test for #3194. This row/value count crosses a v2.2 structural
|
||||
# encoding page boundary where Lance 3.0.0 sliced repetition/definition
|
||||
# levels by row offset and decoded child arrays at different lengths.
|
||||
num_rows = 64_885
|
||||
num_values = 217
|
||||
list_type = pa.list_(pa.float32())
|
||||
source = pa.table(
|
||||
{
|
||||
"id": np.arange(num_rows, dtype=np.int64),
|
||||
"coords": pa.array(
|
||||
[[1.0, 2.0, 3.0, 4.0]] * num_values + [None] * (num_rows - num_values),
|
||||
type=list_type,
|
||||
),
|
||||
}
|
||||
)
|
||||
db = lancedb.connect(
|
||||
tmp_path,
|
||||
storage_options={"new_table_data_storage_version": "2.2"},
|
||||
)
|
||||
table = db.create_table("test_sparse_nullable_list", data=source)
|
||||
|
||||
result = table.search().select(["id", "coords"]).limit(num_rows).to_arrow()
|
||||
|
||||
assert result.equals(source)
|
||||
|
||||
|
||||
def test_add_with_integer_embeddings_preserves_casting(mem_db: DBConnection):
|
||||
class Schema(LanceModel):
|
||||
text: str
|
||||
@@ -2109,6 +2218,45 @@ def test_merge(tmp_db: DBConnection, tmp_path):
|
||||
table.merge(other_dataset, left_on="id")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("storage_version", ["legacy", "stable"])
|
||||
def test_search_after_merge(tmp_path, storage_version):
|
||||
pytest.importorskip("lance")
|
||||
pd = pytest.importorskip("pandas")
|
||||
|
||||
db = lancedb.connect(
|
||||
tmp_path,
|
||||
storage_options={"new_table_data_storage_version": storage_version},
|
||||
)
|
||||
rng = np.random.default_rng(42)
|
||||
row_count = 512
|
||||
vectors = rng.standard_normal((row_count, 8)).astype(np.float32)
|
||||
table = db.create_table(
|
||||
"search_after_merge",
|
||||
data=pd.DataFrame(
|
||||
{
|
||||
"id": [str(i) for i in range(row_count)],
|
||||
"vector": list(vectors),
|
||||
}
|
||||
),
|
||||
)
|
||||
table.create_index("vector", config=IvfPq(num_partitions=1, num_sub_vectors=2))
|
||||
|
||||
links = pd.DataFrame(
|
||||
{
|
||||
"id": [str(i) for i in range(row_count // 2)],
|
||||
"link": [f"https://example.com/{i}" for i in range(row_count // 2)],
|
||||
}
|
||||
)
|
||||
table.merge(links, left_on="id")
|
||||
|
||||
query = table.search(vectors[-1]).refine_factor(50).limit(10)
|
||||
assert "ANN" in query.explain_plan(verbose=True)
|
||||
|
||||
result = query.to_arrow()
|
||||
links_by_id = dict(zip(result["id"].to_pylist(), result["link"].to_pylist()))
|
||||
assert links_by_id[str(row_count - 1)] is None
|
||||
|
||||
|
||||
def test_delete(mem_db: DBConnection):
|
||||
table = mem_db.create_table(
|
||||
"my_table",
|
||||
@@ -2124,6 +2272,27 @@ def test_delete(mem_db: DBConnection):
|
||||
assert table.to_arrow()["id"].to_pylist() == [1]
|
||||
|
||||
|
||||
def test_concurrent_deletes_are_thread_safe(mem_db: DBConnection):
|
||||
num_workers = 8
|
||||
table = mem_db.create_table(
|
||||
"my_table", data=[{"id": row_id} for row_id in range(num_workers)]
|
||||
)
|
||||
barrier = threading.Barrier(num_workers)
|
||||
|
||||
def delete(row_id: int):
|
||||
barrier.wait()
|
||||
return table.delete(f"id = {row_id}")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
results = list(pool.map(delete, range(num_workers)))
|
||||
|
||||
assert all(result.num_deleted_rows == 1 for result in results)
|
||||
assert sorted(result.version for result in results) == list(
|
||||
range(2, num_workers + 2)
|
||||
)
|
||||
assert table.count_rows() == 0
|
||||
|
||||
|
||||
def test_delete_expr(mem_db: DBConnection):
|
||||
table = mem_db.create_table(
|
||||
"my_table",
|
||||
@@ -2174,6 +2343,20 @@ def test_update(mem_db: DBConnection):
|
||||
assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
|
||||
|
||||
|
||||
def test_update_with_arrow_scalar(mem_db: DBConnection):
|
||||
schema = pa.schema({"id": pa.int64(), "vector": pa.list_(pa.float32(), 4)})
|
||||
table = mem_db.create_table("my_table", schema=schema)
|
||||
table.add([{"id": 1, "vector": [1.0, 2.0, 3.0, 4.0]}])
|
||||
|
||||
value = table.search().select(["vector"]).limit(1).to_arrow()["vector"][0]
|
||||
assert isinstance(value, pa.FixedSizeListScalar)
|
||||
|
||||
result = table.update(where="id == 1", values={"vector": value})
|
||||
|
||||
assert result.rows_updated == 1
|
||||
assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]]
|
||||
|
||||
|
||||
def test_update_types(mem_db: DBConnection):
|
||||
table = mem_db.create_table(
|
||||
"my_table",
|
||||
@@ -2341,6 +2524,55 @@ def test_merge_insert(mem_db: DBConnection):
|
||||
)
|
||||
|
||||
|
||||
def test_merge_insert_nullable_pandas_into_pydantic_schema(mem_db: DBConnection):
|
||||
# Regression test for https://github.com/lancedb/lancedb/issues/2366
|
||||
pd = pytest.importorskip("pandas")
|
||||
|
||||
class Document(LanceModel):
|
||||
id: int
|
||||
title: str
|
||||
content: str
|
||||
|
||||
table = mem_db.create_table("documents", schema=Document)
|
||||
table.add(
|
||||
pd.DataFrame(
|
||||
{
|
||||
"title": ["Old title", "Unchanged"],
|
||||
"id": [2, 3],
|
||||
"content": ["Old content", "Keep this"],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# Pandas produces nullable Arrow fields, in an order that differs from the
|
||||
# non-nullable Pydantic schema. This is valid as long as the data has no nulls.
|
||||
new_data = pd.DataFrame(
|
||||
{
|
||||
"title": ["Inserted", "Updated"],
|
||||
"id": [1, 2],
|
||||
"content": ["New row", "New content"],
|
||||
}
|
||||
)
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.when_not_matched_insert_all()
|
||||
.execute(new_data)
|
||||
)
|
||||
|
||||
assert result.num_inserted_rows == 1
|
||||
assert result.num_updated_rows == 1
|
||||
expected = pa.Table.from_pylist(
|
||||
[
|
||||
{"id": 1, "title": "Inserted", "content": "New row"},
|
||||
{"id": 2, "title": "Updated", "content": "New content"},
|
||||
{"id": 3, "title": "Unchanged", "content": "Keep this"},
|
||||
],
|
||||
schema=Document.to_arrow_schema(),
|
||||
)
|
||||
assert table.to_arrow().sort_by("id") == expected
|
||||
|
||||
|
||||
def test_merge_insert_by_source_delete_expr(mem_db: DBConnection):
|
||||
table = mem_db.create_table(
|
||||
"my_table",
|
||||
@@ -2441,6 +2673,36 @@ def test_merge_insert_subschema(mem_db: DBConnection, data_format):
|
||||
assert table.to_arrow().sort_by("id") == expected
|
||||
|
||||
|
||||
def test_repeated_partial_merge_insert_with_scalar_index(mem_db: DBConnection):
|
||||
def make_batch(start: int) -> pa.Table:
|
||||
return pa.table(
|
||||
{
|
||||
"id": [f"id-{i:04}" for i in range(start, start + 100)],
|
||||
"category": ["A"] * 100,
|
||||
"value_a": [float(i) for i in range(start, start + 100)],
|
||||
"value_b": [float(i) / 10 for i in range(100)],
|
||||
}
|
||||
)
|
||||
|
||||
table = mem_db.create_table("my_table", data=make_batch(0))
|
||||
table.add(make_batch(100))
|
||||
table.add(make_batch(200))
|
||||
table.create_index("id", config=BTree())
|
||||
|
||||
ids = [f"id-{i:04}" for i in range(100, 200)]
|
||||
for value in (999.0, 888.0):
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.execute(pa.table({"id": ids, "value_a": [value] * 100}))
|
||||
)
|
||||
assert result.num_updated_rows == 100
|
||||
|
||||
actual = table.to_arrow().sort_by("id")
|
||||
assert actual.num_rows == 300
|
||||
assert actual["value_a"].to_pylist()[100:200] == [888.0] * 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_insert_async(mem_db_async: AsyncConnection):
|
||||
data = pa.table({"a": [1, 2, 3], "b": ["a", "b", "c"]})
|
||||
@@ -2537,15 +2799,40 @@ def test_create_with_embedding_function(mem_db: DBConnection):
|
||||
assert actual == expected
|
||||
|
||||
|
||||
def test_create_f16_table_from_arrow_data(mem_db: DBConnection):
|
||||
dimension = 32
|
||||
num_rows = 512
|
||||
values = pa.array(
|
||||
np.random.default_rng(42)
|
||||
.standard_normal(num_rows * dimension)
|
||||
.astype(np.float16)
|
||||
)
|
||||
df = pa.table(
|
||||
{
|
||||
"text": [f"s-{i}" for i in range(num_rows)],
|
||||
"vector": pa.FixedSizeListArray.from_arrays(values, dimension),
|
||||
}
|
||||
)
|
||||
table = mem_db.create_table("f16_tbl", data=df)
|
||||
assert table.schema.field("vector").type == pa.list_(pa.float16(), dimension)
|
||||
table.create_index(num_partitions=2, num_sub_vectors=2)
|
||||
|
||||
query = df["vector"][2].as_py()
|
||||
expected = table.search(query).limit(2).to_arrow()
|
||||
|
||||
assert "s-2" in expected["text"].to_pylist()
|
||||
|
||||
|
||||
def test_create_f16_table(mem_db: DBConnection):
|
||||
class MyTable(LanceModel):
|
||||
text: str
|
||||
vector: Vector(32, value_type=pa.float16())
|
||||
|
||||
rng = np.random.default_rng(42)
|
||||
df = pa.table(
|
||||
{
|
||||
"text": [f"s-{i}" for i in range(512)],
|
||||
"vector": [np.random.randn(32).astype(np.float16) for _ in range(512)],
|
||||
"vector": [rng.standard_normal(32).astype(np.float16) for _ in range(512)],
|
||||
}
|
||||
)
|
||||
table = mem_db.create_table(
|
||||
@@ -3512,8 +3799,8 @@ def test_create_table_empty_list_no_schema_error(mem_db: DBConnection):
|
||||
mem_db.create_table("test_empty_no_schema", data=[])
|
||||
|
||||
|
||||
def test_add_table_with_empty_embeddings(tmp_path):
|
||||
"""Test exact scenario from issue #1968
|
||||
def test_create_table_without_data_with_vector_schema(tmp_path):
|
||||
"""Test exact scenario from issue #1968.
|
||||
|
||||
Regression test for issue #1968:
|
||||
https://github.com/lancedb/lancedb/issues/1968
|
||||
@@ -3525,6 +3812,9 @@ def test_add_table_with_empty_embeddings(tmp_path):
|
||||
embedding: Vector(16)
|
||||
|
||||
table = db.create_table("test", schema=MySchema)
|
||||
assert table.count_rows() == 0
|
||||
assert table.schema == MySchema.to_arrow_schema()
|
||||
|
||||
table.add(
|
||||
[{"text": "bar", "embedding": [0.1] * 16}],
|
||||
on_bad_vectors="drop",
|
||||
|
||||
@@ -75,6 +75,22 @@ class TestVoyageAIModelRegistration:
|
||||
with pytest.raises(ValueError, match="not supported"):
|
||||
func.ndims()
|
||||
|
||||
def test_voyage3_source_embeddings_use_text_api(self, mock_voyageai_client):
|
||||
"""Regression test for text table data being sent to the multimodal API."""
|
||||
mock_voyageai_client.tokenize.return_value = [["hello", "world"]]
|
||||
mock_voyageai_client.embed.return_value.embeddings = [[0.1] * 1024]
|
||||
|
||||
registry = get_registry()
|
||||
func = registry.get("voyageai").create(name="voyage-3")
|
||||
|
||||
embeddings = func.compute_source_embeddings("hello world")
|
||||
|
||||
assert embeddings == [[0.1] * 1024]
|
||||
mock_voyageai_client.embed.assert_called_once_with(
|
||||
texts=["hello world"], model="voyage-3", input_type="document"
|
||||
)
|
||||
mock_voyageai_client.multimodal_embed.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
from typing import assert_type
|
||||
|
||||
import lancedb
|
||||
from lancedb import AsyncConnection, DBConnection
|
||||
|
||||
|
||||
def check_connect_type() -> None:
|
||||
assert_type(lancedb.connect("memory://"), DBConnection)
|
||||
|
||||
|
||||
async def check_connect_async_type() -> None:
|
||||
assert_type(await lancedb.connect_async("memory://"), AsyncConnection)
|
||||
+110
-1
@@ -29,11 +29,72 @@ use pyo3::{
|
||||
Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
pyclass, pyfunction, pymethods,
|
||||
types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods},
|
||||
types::{IntoPyDict, PyAnyMethods, PyBytes, PyDict, PyDictMethods, PyList, PyListMethods},
|
||||
};
|
||||
|
||||
mod scannable;
|
||||
|
||||
/// Convert `LsmStats` to a Python dict, preserving the per-bucket list.
|
||||
///
|
||||
/// Deliberately not flattened to a table-level summary: a table is N
|
||||
/// buckets on one node, and the per-bucket detail is the reason the
|
||||
/// endpoint exists — flattening hides the single hot bucket someone opened
|
||||
/// it to find.
|
||||
fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
|
||||
let out = PyDict::new(py);
|
||||
let buckets = PyList::empty(py);
|
||||
for b in &stats.buckets {
|
||||
let e = PyDict::new(py);
|
||||
e.set_item("shard_id", &b.shard_id)?;
|
||||
e.set_item("status", &b.status)?;
|
||||
e.set_item("writer_epoch", b.writer_epoch)?;
|
||||
e.set_item("manifest_version", b.manifest_version)?;
|
||||
e.set_item("current_generation", b.current_generation)?;
|
||||
e.set_item(
|
||||
"replay_after_wal_entry_position",
|
||||
b.replay_after_wal_entry_position,
|
||||
)?;
|
||||
e.set_item(
|
||||
"wal_entry_position_last_seen",
|
||||
b.wal_entry_position_last_seen,
|
||||
)?;
|
||||
|
||||
let generations = PyList::empty(py);
|
||||
for g in &b.generations {
|
||||
let ge = PyDict::new(py);
|
||||
ge.set_item("generation", g.generation)?;
|
||||
ge.set_item("bytes", g.bytes)?;
|
||||
ge.set_item("rows", g.rows)?;
|
||||
generations.append(ge)?;
|
||||
}
|
||||
e.set_item("generations", generations)?;
|
||||
e.set_item("compacting", b.compacting)?;
|
||||
|
||||
e.set_item(
|
||||
"memtables",
|
||||
b.memtables
|
||||
.as_ref()
|
||||
.map(|ms| {
|
||||
let l = PyList::empty(py);
|
||||
for m in ms {
|
||||
let d = PyDict::new(py);
|
||||
d.set_item("generation", m.generation)?;
|
||||
d.set_item("rows", m.rows)?;
|
||||
d.set_item("bytes", m.bytes)?;
|
||||
d.set_item("batches", m.batches)?;
|
||||
d.set_item("indexes", m.indexes.clone())?;
|
||||
l.append(d)?;
|
||||
}
|
||||
PyResult::Ok(l.unbind())
|
||||
})
|
||||
.transpose()?,
|
||||
)?;
|
||||
buckets.append(e)?;
|
||||
}
|
||||
out.set_item("buckets", buckets)?;
|
||||
Ok(out.unbind())
|
||||
}
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
enum PredicateArg {
|
||||
Expr(PyExpr),
|
||||
@@ -840,6 +901,9 @@ impl Table {
|
||||
|
||||
#[allow(private_interfaces)]
|
||||
pub fn delete(self_: PyRef<'_, Self>, condition: PredicateArg) -> PyResult<Bound<'_, PyAny>> {
|
||||
// Do not hold the Python borrow across the await. The cloned Rust table
|
||||
// handle is thread-safe and allows deletes on the same Python table to
|
||||
// run concurrently without PyO3 reporting "Already borrowed".
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let result = match &condition {
|
||||
@@ -1433,6 +1497,51 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
/// Converge the table's LSM write path into its base table.
|
||||
///
|
||||
/// Best-effort: with writes flowing, new rows may land after the last
|
||||
/// pass. Errors if the table stops making progress.
|
||||
pub fn checkpoint_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner.checkpoint_lsm().await.infer_error()
|
||||
})
|
||||
}
|
||||
|
||||
/// Seal every bucket's active memtable into L0.
|
||||
pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(
|
||||
self_.py(),
|
||||
async move { inner.flush_lsm().await.infer_error() },
|
||||
)
|
||||
}
|
||||
|
||||
/// Trigger a background L0 → base pass per bucket. Returns once the
|
||||
/// passes are dispatched, not once they finish — watch `get_lsm_stats`.
|
||||
pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner.compact_lsm().await.infer_error()
|
||||
})
|
||||
}
|
||||
|
||||
/// Live LSM state, or `None` when the LSM write path is not enabled.
|
||||
#[pyo3(signature = (include_generation_rows=false))]
|
||||
pub fn get_lsm_stats(
|
||||
self_: PyRef<'_, Self>,
|
||||
include_generation_rows: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let stats = inner
|
||||
.get_lsm_stats(include_generation_rows)
|
||||
.await
|
||||
.infer_error()?;
|
||||
Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
|
||||
Generated
+1
-1
@@ -1998,7 +1998,7 @@ requires-dist = [
|
||||
{ name = "pillow", marker = "extra == 'clip'", specifier = ">=12.1.1" },
|
||||
{ name = "pillow", marker = "extra == 'embeddings'", specifier = ">=12.1.1" },
|
||||
{ name = "pillow", marker = "extra == 'siglip'", specifier = ">=12.1.1" },
|
||||
{ name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.3.0" },
|
||||
{ name = "polars", marker = "extra == 'tests'", specifier = ">=0.19,<=1.32.3" },
|
||||
{ name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" },
|
||||
{ name = "pyarrow", specifier = ">=16" },
|
||||
{ name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },
|
||||
|
||||
@@ -49,8 +49,8 @@ lance-namespace = { workspace = true }
|
||||
lance-namespace-impls = { workspace = true }
|
||||
metrics = { workspace = true, optional = true }
|
||||
metrics-util = { workspace = true, optional = true }
|
||||
# Pin the transitive GooseFS SDK until the 0.1.6 compile break is fixed upstream.
|
||||
goosefs-sdk = { version = "=0.1.5", optional = true }
|
||||
# Pin the GooseFS SDK to the version required by Lance's OpenDAL dependency.
|
||||
goosefs-sdk = { version = "=0.1.9", optional = true }
|
||||
moka = { workspace = true }
|
||||
pin-project = { workspace = true }
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
|
||||
@@ -75,6 +75,8 @@ reqwest = { version = "0.12.0", default-features = false, features = [
|
||||
"http2",
|
||||
"json",
|
||||
"macos-system-configuration",
|
||||
# Avoid linking OpenSSL into Python wheels, which breaks on FIPS hosts.
|
||||
"rustls-tls-native-roots",
|
||||
"stream",
|
||||
], optional = true }
|
||||
http = { version = "1", optional = true } # Matching what is in reqwest
|
||||
@@ -98,7 +100,7 @@ anyhow = "1"
|
||||
lance-testing = { workspace = true }
|
||||
tempfile = "3.5.0"
|
||||
random_word = { version = "0.4.3", features = ["en"] }
|
||||
tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync"] }
|
||||
tokio = { version = "1.23", features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "test-util"] }
|
||||
uuid = { version = "1.7.0", features = ["v4"] }
|
||||
walkdir = "2"
|
||||
aws-sdk-dynamodb = { version = "1.55.0" }
|
||||
|
||||
@@ -17,7 +17,7 @@ use arrow_array::builder::LargeBinaryBuilder;
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use lance::dataset::{BlobRangeRequest as LanceBlobRangeRequest, Dataset, WriteParams};
|
||||
use lance_arrow::FieldExt;
|
||||
use lance_encoding::version::LanceFileVersion;
|
||||
use lance_file::version::LanceFileVersion;
|
||||
use lance_io::object_store::ObjectStore;
|
||||
use object_store::path::Path;
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ use crate::remote::{
|
||||
db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION},
|
||||
};
|
||||
use lance::io::ObjectStoreParams;
|
||||
pub use lance_encoding::version::LanceFileVersion;
|
||||
pub use lance_file::version::LanceFileVersion;
|
||||
#[cfg(feature = "remote")]
|
||||
use lance_io::object_store::StorageOptions;
|
||||
use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
|
||||
|
||||
@@ -202,6 +202,17 @@ mod tests {
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_table_in_named_memory_database() {
|
||||
let db = connect("memory://foo").execute().await.unwrap();
|
||||
let batch = record_batch!(("id", Int64, [1, 2, 3])).unwrap();
|
||||
|
||||
let table = db.create_table("my_table", batch).execute().await.unwrap();
|
||||
|
||||
assert_eq!(table.uri().await.unwrap(), "memory://foo/my_table.lance");
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 3);
|
||||
}
|
||||
|
||||
async fn test_create_table_with_data<T>(data: T)
|
||||
where
|
||||
T: Scannable + 'static,
|
||||
|
||||
@@ -12,7 +12,7 @@ use lance::dataset::refs::Ref;
|
||||
use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
|
||||
use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
|
||||
use lance_datafusion::utils::StreamingWriteSource;
|
||||
use lance_encoding::version::LanceFileVersion;
|
||||
use lance_file::version::LanceFileVersion;
|
||||
use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
|
||||
use lance_table::io::commit::commit_handler_from_url;
|
||||
use object_store::local::LocalFileSystem;
|
||||
@@ -1294,9 +1294,11 @@ mod tests {
|
||||
use crate::connection::ConnectRequest;
|
||||
use crate::data::scannable::Scannable;
|
||||
use crate::database::{CreateTableMode, CreateTableRequest};
|
||||
use crate::table::WriteOptions;
|
||||
use crate::query::QueryRequest;
|
||||
use crate::table::{AnyQuery, WriteOptions};
|
||||
use arrow_array::{Int32Array, RecordBatch, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -1376,6 +1378,156 @@ mod tests {
|
||||
assert!(!tempdir.path().join("__manifest").exists());
|
||||
}
|
||||
|
||||
/// Regression test for https://github.com/lancedb/lancedb/issues/1600.
|
||||
///
|
||||
/// Opening a table used to create a separate object-store client instead of
|
||||
/// reusing the one that successfully connected to the database. Repeating
|
||||
/// credential discovery made S3 table opens intermittent, especially in AWS
|
||||
/// Lambda, and the failed open was reported as `TableNotFound`.
|
||||
#[tokio::test]
|
||||
async fn test_open_table_reuses_connection_object_store() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
let uri = tempdir.path().to_str().unwrap();
|
||||
let registry = Arc::new(lance_io::object_store::ObjectStoreRegistry::default());
|
||||
let session = Arc::new(lance::session::Session::new(16, 16, registry.clone()));
|
||||
|
||||
let request = ConnectRequest {
|
||||
uri: uri.to_string(),
|
||||
#[cfg(feature = "remote")]
|
||||
client_config: Default::default(),
|
||||
options: Default::default(),
|
||||
namespace_client_properties: Default::default(),
|
||||
manifest_enabled: false,
|
||||
read_consistency_interval: None,
|
||||
session: Some(session),
|
||||
};
|
||||
let db = ListingDatabase::connect_with_options(&request)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
db.create_table(CreateTableRequest {
|
||||
name: "test".to_string(),
|
||||
namespace_path: vec![],
|
||||
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let before_open = registry.stats();
|
||||
for _ in 0..3 {
|
||||
let table = db
|
||||
.open_table(OpenTableRequest {
|
||||
name: "test".to_string(),
|
||||
namespace_path: vec![],
|
||||
index_cache_size: None,
|
||||
lance_read_params: None,
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
managed_versioning: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 0);
|
||||
}
|
||||
|
||||
let after_open = registry.stats();
|
||||
assert_eq!(after_open.misses, before_open.misses);
|
||||
assert!(after_open.hits >= before_open.hits + 3);
|
||||
}
|
||||
|
||||
/// Regression test for https://github.com/lancedb/lancedb/issues/3197.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn test_open_table_follows_hugging_face_symlinks() {
|
||||
let (tempdir, db) = setup_database().await;
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
db.create_table(CreateTableRequest {
|
||||
name: "test".to_string(),
|
||||
namespace_path: vec![],
|
||||
data: Box::new(
|
||||
RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])
|
||||
.unwrap(),
|
||||
) as Box<dyn Scannable>,
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let table_dir = tempdir.path().join("test.lance");
|
||||
let versions_dir = table_dir.join("_versions");
|
||||
let manifest_path = std::fs::read_dir(&versions_dir)
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().path())
|
||||
.find(|path| path.extension().is_some_and(|ext| ext == "manifest"))
|
||||
.unwrap();
|
||||
let data_path = std::fs::read_dir(table_dir.join("data"))
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().path())
|
||||
.find(|path| path.extension().is_some_and(|ext| ext == "lance"))
|
||||
.unwrap();
|
||||
|
||||
// Hugging Face snapshots keep dataset objects in a separate blob directory and
|
||||
// expose them through relative symlinks.
|
||||
let blobs_dir = tempdir.path().join("blobs");
|
||||
std::fs::create_dir(&blobs_dir).unwrap();
|
||||
let manifest_blob = "9b603c63d0e692e05d58be25605f2f2064cc781e5ff94fe983a405059547b816";
|
||||
let data_blob = "be64f20e5723bd0a27cfdbdb41cf7d6fad94cd572a71973b717fb8340f4310c5";
|
||||
std::fs::rename(&manifest_path, blobs_dir.join(manifest_blob)).unwrap();
|
||||
std::fs::rename(&data_path, blobs_dir.join(data_blob)).unwrap();
|
||||
std::os::unix::fs::symlink(Path::new("../../blobs").join(manifest_blob), &manifest_path)
|
||||
.unwrap();
|
||||
std::os::unix::fs::symlink(Path::new("../../blobs").join(data_blob), &data_path).unwrap();
|
||||
let symlink_len = std::fs::symlink_metadata(&manifest_path).unwrap().len();
|
||||
let target_len = std::fs::metadata(&manifest_path).unwrap().len();
|
||||
assert_ne!(symlink_len, target_len);
|
||||
|
||||
drop(db);
|
||||
let db = ListingDatabase::connect_with_options(&ConnectRequest {
|
||||
uri: tempdir.path().to_str().unwrap().to_string(),
|
||||
#[cfg(feature = "remote")]
|
||||
client_config: Default::default(),
|
||||
options: Default::default(),
|
||||
namespace_client_properties: Default::default(),
|
||||
manifest_enabled: false,
|
||||
read_consistency_interval: None,
|
||||
session: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let table = db
|
||||
.open_table(OpenTableRequest {
|
||||
name: "test".to_string(),
|
||||
namespace_path: vec![],
|
||||
index_cache_size: None,
|
||||
lance_read_params: None,
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
managed_versioning: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let batches = table
|
||||
.query(
|
||||
&AnyQuery::Query(QueryRequest::default()),
|
||||
Default::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clone_table_basic() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
@@ -2280,7 +2432,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_table_uri() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
let (_tempdir, mut db) = setup_database().await;
|
||||
|
||||
let mut pb = PathBuf::new();
|
||||
pb.push(db.uri.clone());
|
||||
@@ -2289,6 +2441,18 @@ mod tests {
|
||||
let expected = pb.to_str().unwrap();
|
||||
let uri = db.table_uri("test").ok().unwrap();
|
||||
assert_eq!(uri, expected);
|
||||
|
||||
// URI paths always use forward slashes, even on Windows. Using
|
||||
// `Path::join` here used to produce `az://container/prefix\\test.lance`,
|
||||
// which Azure treated as a different object from the table returned by
|
||||
// `table_names` (https://github.com/lancedb/lancedb/issues/1072).
|
||||
for base_uri in ["az://container/prefix", "az://container/prefix/"] {
|
||||
db.uri = base_uri.to_string();
|
||||
assert_eq!(
|
||||
db.table_uri("test").unwrap(),
|
||||
"az://container/prefix/test.lance"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Regression: connecting via a URL-style URI (which goes through
|
||||
|
||||
@@ -201,7 +201,7 @@ impl LanceNamespaceDatabase {
|
||||
&self,
|
||||
request: &DbCreateTableRequest,
|
||||
) -> Result<(
|
||||
Option<lance_encoding::version::LanceFileVersion>,
|
||||
Option<lance_file::version::LanceFileVersion>,
|
||||
Option<bool>,
|
||||
Option<bool>,
|
||||
)> {
|
||||
@@ -214,7 +214,7 @@ impl LanceNamespaceDatabase {
|
||||
|
||||
let storage_version_override = storage_options
|
||||
.and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
|
||||
.map(|s| s.parse::<lance_encoding::version::LanceFileVersion>())
|
||||
.map(|s| s.parse::<lance_file::version::LanceFileVersion>())
|
||||
.transpose()?;
|
||||
|
||||
let v2_manifest_override = storage_options
|
||||
|
||||
@@ -169,6 +169,12 @@ impl From<DataFusionError> for Error {
|
||||
|
||||
impl From<lance::Error> for Error {
|
||||
fn from(source: lance::Error) -> Self {
|
||||
if has_unsupported_local_filesystem_source(&source) {
|
||||
return Self::NotSupported {
|
||||
message: "the filesystem does not support an operation required for safe Lance commits (such as atomic rename). Object-storage mounts such as Mountpoint for Amazon S3 are not supported; use the native object-store URI (for example, s3://bucket/path) instead".to_string(),
|
||||
};
|
||||
}
|
||||
|
||||
// Try to unwrap external errors that were wrapped by lance
|
||||
match source {
|
||||
lance::Error::Wrapped { error, .. } => Self::from_box_error(error),
|
||||
@@ -181,6 +187,27 @@ impl From<lance::Error> for Error {
|
||||
}
|
||||
}
|
||||
|
||||
fn has_unsupported_local_filesystem_source(error: &(dyn std::error::Error + 'static)) -> bool {
|
||||
let mut current = Some(error);
|
||||
let mut is_local_filesystem = false;
|
||||
let mut is_unsupported = false;
|
||||
while let Some(error) = current {
|
||||
is_local_filesystem |= error
|
||||
.downcast_ref::<object_store::Error>()
|
||||
.is_some_and(|error| {
|
||||
matches!(error, object_store::Error::Generic { store, .. } if *store == "LocalFileSystem")
|
||||
});
|
||||
is_unsupported |= error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.is_some_and(|error| error.kind() == std::io::ErrorKind::Unsupported);
|
||||
if is_local_filesystem && is_unsupported {
|
||||
return true;
|
||||
}
|
||||
current = error.source();
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
impl Error {
|
||||
fn from_box_error(mut source: Box<dyn std::error::Error + Send + Sync>) -> Self {
|
||||
source = match source.downcast::<Self>() {
|
||||
@@ -270,3 +297,46 @@ impl From<candle_core::Error> for Error {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unsupported_filesystem_operations_have_actionable_error() {
|
||||
let object_store_error = object_store::Error::Generic {
|
||||
store: "LocalFileSystem",
|
||||
source: Box::new(std::io::Error::from(std::io::ErrorKind::Unsupported)),
|
||||
};
|
||||
let lance_error = lance::Error::io_source(Box::new(object_store_error));
|
||||
|
||||
let error = Error::from(lance_error);
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::NotSupported { message }
|
||||
if message.contains("Mountpoint for Amazon S3")
|
||||
&& message.contains("s3://bucket/path")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_io_errors_remain_lance_errors() {
|
||||
let object_store_error = object_store::Error::Generic {
|
||||
store: "LocalFileSystem",
|
||||
source: Box::new(std::io::Error::from(std::io::ErrorKind::PermissionDenied)),
|
||||
};
|
||||
let lance_error = lance::Error::io_source(Box::new(object_store_error));
|
||||
|
||||
assert!(matches!(Error::from(lance_error), Error::Lance { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_non_filesystem_errors_remain_lance_errors() {
|
||||
let lance_error = lance::Error::io_source(Box::new(std::io::Error::from(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
)));
|
||||
|
||||
assert!(matches!(Error::from(lance_error), Error::Lance { .. }));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,9 +132,14 @@ impl ObjectStore for MirroringObjectStore {
|
||||
if to.primary_only() {
|
||||
self.primary.copy_opts(from, to, options).await
|
||||
} else {
|
||||
self.secondary.copy_opts(from, to, options.clone()).await?;
|
||||
self.primary.copy_opts(from, to, options).await?;
|
||||
Ok(())
|
||||
// The secondary store can be process-local and less durable than the
|
||||
// primary, so a source written by another process may not exist here
|
||||
// or may be evicted before the copy begins.
|
||||
match self.secondary.copy_opts(from, to, options.clone()).await {
|
||||
Ok(()) | Err(Error::NotFound { .. }) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
self.primary.copy_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -192,7 +197,8 @@ mod test {
|
||||
use futures::TryStreamExt;
|
||||
use lance::{dataset::WriteParams, io::ObjectStoreParams};
|
||||
use lance_testing::datagen::{BatchGenerator, IncrementingInt32, RandomVector};
|
||||
use object_store::local::LocalFileSystem;
|
||||
use object_store::{local::LocalFileSystem, memory::InMemory};
|
||||
use std::time::Duration;
|
||||
use tempfile;
|
||||
|
||||
use crate::{
|
||||
@@ -201,6 +207,139 @@ mod test {
|
||||
table::WriteOptions,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
struct EvictBeforeCopyStore {
|
||||
inner: Arc<dyn ObjectStore>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for EvictBeforeCopyStore {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "EvictBeforeCopyStore")
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ObjectStore for EvictBeforeCopyStore {
|
||||
async fn put_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
payload: PutPayload,
|
||||
options: PutOptions,
|
||||
) -> Result<PutResult> {
|
||||
self.inner.put_opts(location, payload, options).await
|
||||
}
|
||||
|
||||
async fn put_multipart_opts(
|
||||
&self,
|
||||
location: &Path,
|
||||
options: PutMultipartOptions,
|
||||
) -> Result<Box<dyn MultipartUpload>> {
|
||||
self.inner.put_multipart_opts(location, options).await
|
||||
}
|
||||
|
||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
|
||||
self.inner.get_opts(location, options).await
|
||||
}
|
||||
|
||||
fn delete_stream(
|
||||
&self,
|
||||
locations: BoxStream<'static, Result<Path>>,
|
||||
) -> BoxStream<'static, Result<Path>> {
|
||||
self.inner.delete_stream(locations)
|
||||
}
|
||||
|
||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||
self.inner.list(prefix)
|
||||
}
|
||||
|
||||
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
|
||||
self.inner.list_with_delimiter(prefix).await
|
||||
}
|
||||
|
||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
||||
self.inner.delete(from).await?;
|
||||
self.inner.copy_opts(from, to, options).await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_copy_when_source_is_missing_from_secondary() {
|
||||
let primary_dir = tempfile::tempdir().unwrap();
|
||||
let secondary_dir = tempfile::tempdir().unwrap();
|
||||
let primary: Arc<dyn ObjectStore> =
|
||||
Arc::new(LocalFileSystem::new_with_prefix(primary_dir.path()).unwrap());
|
||||
let secondary: Arc<dyn ObjectStore> =
|
||||
Arc::new(LocalFileSystem::new_with_prefix(secondary_dir.path()).unwrap());
|
||||
let store = MirroringObjectStore {
|
||||
primary: primary.clone(),
|
||||
secondary: secondary.clone(),
|
||||
};
|
||||
let staging = Path::from("_versions/1.manifest-staging");
|
||||
let finalized = Path::from("_versions/1.manifest");
|
||||
|
||||
primary
|
||||
.put(&staging, "manifest contents".into())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), store.copy(&staging, &finalized))
|
||||
.await
|
||||
.expect("copy should not hang when the secondary source is missing")
|
||||
.unwrap();
|
||||
|
||||
let copied = primary
|
||||
.get(&finalized)
|
||||
.await
|
||||
.unwrap()
|
||||
.bytes()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(copied, "manifest contents");
|
||||
assert!(matches!(
|
||||
secondary.head(&finalized).await,
|
||||
Err(Error::NotFound { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_copy_when_secondary_source_disappears_after_head() {
|
||||
let primary: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
|
||||
let secondary_inner: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
|
||||
let secondary: Arc<dyn ObjectStore> = Arc::new(EvictBeforeCopyStore {
|
||||
inner: secondary_inner.clone(),
|
||||
});
|
||||
let store = MirroringObjectStore {
|
||||
primary: primary.clone(),
|
||||
secondary,
|
||||
};
|
||||
let staging = Path::from("_versions/1.manifest-staging");
|
||||
let finalized = Path::from("_versions/1.manifest");
|
||||
|
||||
primary
|
||||
.put(&staging, "manifest contents".into())
|
||||
.await
|
||||
.unwrap();
|
||||
secondary_inner
|
||||
.put(&staging, "manifest contents".into())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
store.copy(&staging, &finalized).await.unwrap();
|
||||
|
||||
let copied = primary
|
||||
.get(&finalized)
|
||||
.await
|
||||
.unwrap()
|
||||
.bytes()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(copied, "manifest contents");
|
||||
assert!(matches!(
|
||||
secondary_inner.head(&finalized).await,
|
||||
Err(Error::NotFound { .. })
|
||||
));
|
||||
}
|
||||
|
||||
// This test is ignored because lance 3.0 introduced LocalWriter optimization
|
||||
// that bypasses the object store wrapper for local writes. The mirroring feature
|
||||
// still works for remote/cloud storage, but can't be tested with local storage.
|
||||
|
||||
@@ -1661,14 +1661,8 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_setters_getters() {
|
||||
// TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051
|
||||
// is fixed
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let dataset_path = tmp_dir.path().join("test.lance");
|
||||
let uri = dataset_path.to_str().unwrap();
|
||||
|
||||
let batches = make_test_batches();
|
||||
let conn = connect(uri).execute().await.unwrap();
|
||||
let conn = connect("memory://foo").execute().await.unwrap();
|
||||
let table = conn
|
||||
.create_table("my_table", batches)
|
||||
.execute()
|
||||
@@ -1763,14 +1757,8 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute() {
|
||||
// TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051
|
||||
// is fixed
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let dataset_path = tmp_dir.path().join("test.lance");
|
||||
let uri = dataset_path.to_str().unwrap();
|
||||
|
||||
let batches = make_non_empty_batches();
|
||||
let conn = connect(uri).execute().await.unwrap();
|
||||
let conn = connect("memory://foo").execute().await.unwrap();
|
||||
let table = conn
|
||||
.create_table("my_table", batches)
|
||||
.execute()
|
||||
@@ -1889,14 +1877,8 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_select_with_transform() {
|
||||
// TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051
|
||||
// is fixed
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let dataset_path = tmp_dir.path().join("test.lance");
|
||||
let uri = dataset_path.to_str().unwrap();
|
||||
|
||||
let batches = make_non_empty_batches();
|
||||
let conn = connect(uri).execute().await.unwrap();
|
||||
let conn = connect("memory://foo").execute().await.unwrap();
|
||||
let table = conn
|
||||
.create_table("my_table", batches)
|
||||
.execute()
|
||||
@@ -1993,15 +1975,9 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_no_vector() {
|
||||
// TODO: Switch back to memory://foo after https://github.com/lancedb/lancedb/issues/1051
|
||||
// is fixed
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let dataset_path = tmp_dir.path().join("test.lance");
|
||||
let uri = dataset_path.to_str().unwrap();
|
||||
|
||||
// test that it's ok to not specify a query vector (just filter / limit)
|
||||
let batches = make_non_empty_batches();
|
||||
let conn = connect(uri).execute().await.unwrap();
|
||||
let conn = connect("memory://foo").execute().await.unwrap();
|
||||
let table = conn
|
||||
.create_table("my_table", batches)
|
||||
.execute()
|
||||
|
||||
@@ -373,6 +373,37 @@ pub fn parse_db_url(db_url: &str) -> Result<ParsedDbUrl> {
|
||||
Ok(ParsedDbUrl { db_name, db_prefix })
|
||||
}
|
||||
|
||||
fn validate_dns_hostname(hostname: &str) -> Result<()> {
|
||||
let ascii_hostname = match url::Host::parse(hostname) {
|
||||
Ok(url::Host::Domain(hostname)) => hostname,
|
||||
Ok(_) => {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "LanceDB Cloud database URI or region produced a non-DNS hostname"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!(
|
||||
"LanceDB Cloud database URI or region produced an invalid hostname: {err}"
|
||||
),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
if ascii_hostname.len() > 253
|
||||
|| ascii_hostname
|
||||
.split('.')
|
||||
.any(|label| label.is_empty() || label.len() > 63)
|
||||
{
|
||||
return Err(Error::InvalidInput {
|
||||
message: "LanceDB Cloud database URI or region produced an invalid hostname: DNS labels must contain 1 to 63 bytes and the full hostname must not exceed 253 bytes".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl RestfulLanceDbClient<Sender> {
|
||||
fn get_timeout(passed: Option<Duration>, env_var: &str) -> Result<Option<Duration>> {
|
||||
if let Some(passed) = passed {
|
||||
@@ -480,7 +511,11 @@ impl RestfulLanceDbClient<Sender> {
|
||||
|
||||
let host = match host_override {
|
||||
Some(host_override) => host_override,
|
||||
None => format!("https://{}.{}.api.lancedb.com", parsed_url.db_name, region),
|
||||
None => {
|
||||
let hostname = format!("{}.{}.api.lancedb.com", parsed_url.db_name, region);
|
||||
validate_dns_hostname(&hostname)?;
|
||||
format!("https://{hostname}")
|
||||
}
|
||||
};
|
||||
debug!("Created client for host: {}", host);
|
||||
let retry_config = client_config.retry_config.clone().try_into()?;
|
||||
@@ -1157,6 +1192,29 @@ mod tests {
|
||||
assert_eq!(headers.get("x-api-key").unwrap(), "api-key");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rejects_invalid_cloud_dns_hostname() {
|
||||
let invalid_database_names = ["a".repeat(64), "invalid..database".to_string()];
|
||||
|
||||
for db_name in invalid_database_names {
|
||||
let parsed_url = parse_db_url(&format!("db://{db_name}")).unwrap();
|
||||
let error = RestfulLanceDbClient::<Sender>::try_new(
|
||||
&parsed_url,
|
||||
"us-east-1",
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
ClientConfig::default(),
|
||||
None,
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(
|
||||
matches!(error, Error::InvalidInput { ref message } if message.contains("DNS labels must contain 1 to 63 bytes")),
|
||||
"unexpected error: {error}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Test implementation of HeaderProvider
|
||||
#[derive(Debug, Clone)]
|
||||
struct TestHeaderProvider {
|
||||
|
||||
@@ -23,11 +23,13 @@ use crate::table::AddResult;
|
||||
use crate::table::BranchDiff;
|
||||
use crate::table::DeleteResult;
|
||||
use crate::table::DropColumnsResult;
|
||||
use crate::table::LsmStats;
|
||||
use crate::table::LsmWriteSpec;
|
||||
use crate::table::MergeBranchResult;
|
||||
use crate::table::MergeResult;
|
||||
use crate::table::Tags;
|
||||
use crate::table::UpdateResult;
|
||||
use crate::table::lsm_stats::GetLsmStatsResponse;
|
||||
use crate::table::merge::MergeFilter;
|
||||
use crate::table::query::create_multi_vector_plan;
|
||||
use crate::table::write_progress::FinishOnDrop;
|
||||
@@ -991,6 +993,18 @@ impl<S: HttpSend> RemoteTable<S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Send an LSM operator request with the transport retry layer **off**.
|
||||
///
|
||||
/// Retry policy on these routes belongs to the checkpoint loop, which
|
||||
/// reads the status and can tell contention from a lost claim. Leaving the
|
||||
/// transport layer on would re-ask on its own schedule first, and surface
|
||||
/// an `Error::Retry` whose status the loop would then have to unwrap.
|
||||
async fn send_lsm_route(&self, request: RequestBuilder) -> Result<(String, reqwest::Response)> {
|
||||
let (request_id, response) = self.send(request, false).await?;
|
||||
let response = self.check_table_response(&request_id, response).await?;
|
||||
Ok((request_id, response))
|
||||
}
|
||||
|
||||
/// Build a POST request and attach the read-freshness headers
|
||||
/// (`x-lancedb-min-version`, `x-lancedb-min-timestamp`).
|
||||
fn post_read(&self, uri: &str) -> RequestBuilder {
|
||||
@@ -2468,6 +2482,40 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
})
|
||||
}
|
||||
|
||||
async fn flush_lsm(&self) -> Result<()> {
|
||||
let request = self
|
||||
.client
|
||||
.post(&format!("/v1/table/{}/flush_lsm/", self.identifier));
|
||||
self.send_lsm_route(request).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compact_lsm(&self) -> Result<()> {
|
||||
let request = self
|
||||
.client
|
||||
.post(&format!("/v1/table/{}/compact_lsm/", self.identifier));
|
||||
self.send_lsm_route(request).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
|
||||
// Read-semantics POST, like `get_lsm_write_spec`.
|
||||
let request = self
|
||||
.post_read(&format!("/v1/table/{}/get_lsm_stats/", self.identifier))
|
||||
.json(&serde_json::json!({
|
||||
"include_generation_rows": include_generation_rows,
|
||||
}));
|
||||
let (request_id, response) = self.send_lsm_route(request).await?;
|
||||
let body = response.text().await.err_to_http(request_id.clone())?;
|
||||
let parsed: GetLsmStatsResponse = serde_json::from_str(&body).map_err(|e| Error::Http {
|
||||
source: format!("Failed to parse get_lsm_stats response: {e}").into(),
|
||||
request_id,
|
||||
status_code: None,
|
||||
})?;
|
||||
// `null` — and only — when the table has no LSM write path.
|
||||
Ok(parsed.lsm_stats)
|
||||
}
|
||||
|
||||
async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> {
|
||||
self.check_mutable().await?;
|
||||
|
||||
@@ -2791,9 +2839,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
}
|
||||
|
||||
async fn index_stats(&self, index_name: &str) -> Result<Option<IndexStatistics>> {
|
||||
let encoded_name = urlencoding::encode(index_name);
|
||||
let mut request = self.post_read(&format!(
|
||||
"/v1/table/{}/index/{}/stats/",
|
||||
self.identifier, index_name
|
||||
"/v1/table/{}/index/{encoded_name}/stats/",
|
||||
self.identifier
|
||||
));
|
||||
let version = self.current_version().await;
|
||||
let mut body = serde_json::json!({ "version": version });
|
||||
@@ -2820,9 +2869,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
}
|
||||
|
||||
async fn drop_index(&self, index_name: &str) -> Result<()> {
|
||||
let encoded_name = urlencoding::encode(index_name);
|
||||
let request = self.apply_branch_query(self.client.post(&format!(
|
||||
"/v1/table/{}/index/{}/drop/",
|
||||
self.identifier, index_name
|
||||
"/v1/table/{}/index/{encoded_name}/drop/",
|
||||
self.identifier
|
||||
)));
|
||||
let (request_id, response) = self.send(request, true).await?;
|
||||
if response.status() == StatusCode::NOT_FOUND {
|
||||
@@ -2835,9 +2885,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
}
|
||||
|
||||
async fn prewarm_index(&self, index_name: &str) -> Result<()> {
|
||||
let encoded_name = urlencoding::encode(index_name);
|
||||
let request = self.client.post(&format!(
|
||||
"/v1/table/{}/index/{}/prewarm/",
|
||||
self.identifier, index_name
|
||||
"/v1/table/{}/index/{encoded_name}/prewarm/",
|
||||
self.identifier
|
||||
));
|
||||
let (request_id, response) = self.send(request, true).await?;
|
||||
if response.status() == StatusCode::NOT_FOUND {
|
||||
@@ -2939,7 +2990,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
}
|
||||
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
pub(crate) struct MergeInsertRequest {
|
||||
pub struct MergeInsertRequest {
|
||||
on: String,
|
||||
when_matched_update_all: bool,
|
||||
when_matched_update_all_filt: Option<String>,
|
||||
@@ -5904,16 +5955,18 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Positions are relative to the first retained token, so dropping the
|
||||
// leading "hello" stop word does not shift the remaining tokens.
|
||||
assert_eq!(
|
||||
tokens,
|
||||
vec![
|
||||
FtsToken {
|
||||
text: "こんにちは".to_string(),
|
||||
position: 1,
|
||||
position: 0,
|
||||
},
|
||||
FtsToken {
|
||||
text: "世界".to_string(),
|
||||
position: 2,
|
||||
position: 1,
|
||||
},
|
||||
]
|
||||
);
|
||||
@@ -6489,6 +6542,41 @@ mod tests {
|
||||
assert!(matches!(e, Error::IndexNotFound { .. }));
|
||||
}
|
||||
|
||||
/// Index names are unvalidated, so reserved characters must be
|
||||
/// percent-encoded or they restructure the request path.
|
||||
#[tokio::test]
|
||||
async fn test_per_index_paths_encode_reserved_characters() {
|
||||
const NAME: &str = "my/index?a#b c";
|
||||
const PREFIX: &str = "/v1/table/my_table/index/my%2Findex%3Fa%23b%20c";
|
||||
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
assert_eq!(request.url().path(), format!("{PREFIX}/stats/"));
|
||||
let body = serde_json::json!({
|
||||
"num_indexed_rows": 1,
|
||||
"num_unindexed_rows": 0,
|
||||
"index_type": "IVF_PQ",
|
||||
"distance_type": "l2"
|
||||
});
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(serde_json::to_string(&body).unwrap())
|
||||
.unwrap()
|
||||
});
|
||||
assert!(table.index_stats(NAME).await.unwrap().is_some());
|
||||
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
assert_eq!(request.url().path(), format!("{PREFIX}/drop/"));
|
||||
http::Response::builder().status(200).body("{}").unwrap()
|
||||
});
|
||||
table.drop_index(NAME).await.unwrap();
|
||||
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
assert_eq!(request.url().path(), format!("{PREFIX}/prewarm/"));
|
||||
http::Response::builder().status(200).body("{}").unwrap()
|
||||
});
|
||||
table.prewarm_index(NAME).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_set_lsm_write_spec_unsharded() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
@@ -6642,6 +6730,499 @@ mod tests {
|
||||
assert!(table.get_lsm_write_spec().await.unwrap().is_none());
|
||||
}
|
||||
|
||||
/// Build a `get_lsm_stats` body for one bucket holding `generations`.
|
||||
fn stats_body(generations: &[u64], compacting: bool) -> String {
|
||||
serde_json::json!({
|
||||
"lsm_stats": {
|
||||
"buckets": [{
|
||||
"shard_id": "b0",
|
||||
"status": "Active",
|
||||
"writer_epoch": 1,
|
||||
"manifest_version": 1,
|
||||
"current_generation": generations.iter().max().copied().unwrap_or(0) + 1,
|
||||
"replay_after_wal_entry_position": 0,
|
||||
"wal_entry_position_last_seen": 0,
|
||||
"generations": generations.iter()
|
||||
.map(|g| serde_json::json!({ "generation": g, "bytes": 1 }))
|
||||
.collect::<Vec<_>>(),
|
||||
"compacting": compacting,
|
||||
"memtables": [],
|
||||
}],
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// `flush_lsm` / `compact_lsm` answer 202 with no body at all.
|
||||
fn accepted() -> http::Response<String> {
|
||||
http::Response::builder()
|
||||
.status(202)
|
||||
.body(String::new())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn ok_json(body: String) -> http::Response<String> {
|
||||
http::Response::builder().status(200).body(body).unwrap()
|
||||
}
|
||||
|
||||
/// A flush landing in an empty L0 finishes on the opening stats read
|
||||
/// alone. Asserting zero compacts is the point: "it returned Ok" is also
|
||||
/// true of a loop that ran a pointless pass.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_checkpoint_short_circuits_on_empty_l0() {
|
||||
let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = compacts.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
if path.contains("compact_lsm") {
|
||||
seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
panic!("an already-converged table must issue no compact calls");
|
||||
}
|
||||
if path.contains("flush_lsm") {
|
||||
return accepted();
|
||||
}
|
||||
assert_eq!(path, "/v1/table/my_table/get_lsm_stats/");
|
||||
ok_json(stats_body(&[], false))
|
||||
});
|
||||
|
||||
table.checkpoint_lsm().await.unwrap();
|
||||
assert_eq!(compacts.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// The loop triggers compaction until every generation that existed at
|
||||
/// the start is gone, one bounded prefix per pass.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_checkpoint_triggers_until_targets_are_drained() {
|
||||
let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = compacts.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
if path.contains("flush_lsm") {
|
||||
return accepted();
|
||||
}
|
||||
if path.contains("compact_lsm") {
|
||||
seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
return accepted();
|
||||
}
|
||||
// Each pass drains the oldest generation.
|
||||
let drained = seen.load(std::sync::atomic::Ordering::SeqCst);
|
||||
let left: Vec<u64> = [1u64, 2, 3].into_iter().skip(drained).collect();
|
||||
ok_json(stats_body(&left, false))
|
||||
});
|
||||
|
||||
table.checkpoint_lsm().await.unwrap();
|
||||
assert_eq!(
|
||||
compacts.load(std::sync::atomic::Ordering::SeqCst),
|
||||
3,
|
||||
"one trigger per generation prefix, then stop"
|
||||
);
|
||||
}
|
||||
|
||||
/// Generations created *during* the checkpoint are not waited on, which
|
||||
/// is what lets the loop terminate on a table taking writes where "L0 is
|
||||
/// empty" never becomes true.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_checkpoint_ignores_generations_created_while_it_runs() {
|
||||
let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = compacts.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
if path.contains("flush_lsm") {
|
||||
return accepted();
|
||||
}
|
||||
if path.contains("compact_lsm") {
|
||||
seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
return accepted();
|
||||
}
|
||||
// Target is 5. One pass drains it; a writer keeps adding above.
|
||||
let n = seen.load(std::sync::atomic::Ordering::SeqCst);
|
||||
let body = if n == 0 {
|
||||
stats_body(&[5], false)
|
||||
} else {
|
||||
stats_body(&[6, 7], false)
|
||||
};
|
||||
ok_json(body)
|
||||
});
|
||||
|
||||
table.checkpoint_lsm().await.unwrap();
|
||||
assert_eq!(
|
||||
compacts.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"the loop must not chase generations written after it started"
|
||||
);
|
||||
}
|
||||
|
||||
/// Contention is a 429 and must be retried. The server keeps it off 503
|
||||
/// precisely so the client can act on the status alone — reading it as
|
||||
/// terminal stops the checkpoint early on a healthy node.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_checkpoint_retries_contention() {
|
||||
let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = compacts.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
if path.contains("flush_lsm") {
|
||||
return accepted();
|
||||
}
|
||||
if path.contains("compact_lsm") {
|
||||
// First two triggers: every bucket already latched.
|
||||
if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2 {
|
||||
return http::Response::builder()
|
||||
.status(429)
|
||||
.body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
|
||||
.unwrap();
|
||||
}
|
||||
return accepted();
|
||||
}
|
||||
let accepted_triggers = seen
|
||||
.load(std::sync::atomic::Ordering::SeqCst)
|
||||
.saturating_sub(2);
|
||||
let left: Vec<u64> = if accepted_triggers == 0 {
|
||||
vec![1]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
ok_json(stats_body(&left, false))
|
||||
});
|
||||
|
||||
table
|
||||
.checkpoint_lsm()
|
||||
.await
|
||||
.expect("contention must not abort the checkpoint");
|
||||
assert_eq!(
|
||||
compacts.load(std::sync::atomic::Ordering::SeqCst),
|
||||
3,
|
||||
"assert the retry count, not just the outcome"
|
||||
);
|
||||
}
|
||||
|
||||
/// A transient fault on the poll must not abort the checkpoint. This route
|
||||
/// meets the most contention — it runs every `POLL_INTERVAL` for the
|
||||
/// checkpoint's whole life, with the transport retry layer disabled — yet
|
||||
/// was the one call reached with a bare `?`.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_checkpoint_retries_a_contended_stats_poll() {
|
||||
let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = polls.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
if path.contains("flush_lsm") || path.contains("compact_lsm") {
|
||||
return accepted();
|
||||
}
|
||||
// The opening read lands; the next two polls are latched out.
|
||||
let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
if (1..3).contains(&n) {
|
||||
return http::Response::builder()
|
||||
.status(429)
|
||||
.body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
|
||||
.unwrap();
|
||||
}
|
||||
ok_json(stats_body(if n < 4 { &[1] } else { &[] }, false))
|
||||
});
|
||||
|
||||
table
|
||||
.checkpoint_lsm()
|
||||
.await
|
||||
.expect("a contended poll must be retried, not surfaced");
|
||||
assert_eq!(
|
||||
polls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
5,
|
||||
"the two rejected polls must be re-issued, not skipped"
|
||||
);
|
||||
}
|
||||
|
||||
/// Contention and a lost claim draw on separate budgets: five straight
|
||||
/// 429s on `flush`, more than `MAX_REISSUES`, must still converge. On one
|
||||
/// shared counter this spent the re-issue cap and then reported a lost
|
||||
/// claim nothing had ever reported.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_contention_does_not_exhaust_the_reissue_budget() {
|
||||
let flushes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = flushes.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
if path.contains("flush_lsm") {
|
||||
if seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 5 {
|
||||
return http::Response::builder()
|
||||
.status(429)
|
||||
.body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
|
||||
.unwrap();
|
||||
}
|
||||
return accepted();
|
||||
}
|
||||
if path.contains("compact_lsm") {
|
||||
return accepted();
|
||||
}
|
||||
ok_json(stats_body(&[], false))
|
||||
});
|
||||
|
||||
table
|
||||
.checkpoint_lsm()
|
||||
.await
|
||||
.expect("contention must not be reported as a lost claim");
|
||||
assert_eq!(
|
||||
flushes.load(std::sync::atomic::Ordering::SeqCst),
|
||||
6,
|
||||
"five retries against one seal, then it lands"
|
||||
);
|
||||
}
|
||||
|
||||
/// An exhausted retry budget surfaces the fault that consumed it, not a
|
||||
/// message the loop invented: "429, nine times" points an operator at a
|
||||
/// saturated pool, a generic runtime error points them nowhere.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_exhausted_retries_surface_the_underlying_fault() {
|
||||
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = calls.clone();
|
||||
let table = Table::new_with_handler("my_table", move |_request| {
|
||||
seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
http::Response::builder()
|
||||
.status(429)
|
||||
.body(r#"{"code":21,"error":"Too many concurrent writes"}"#.to_string())
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
let err = table.checkpoint_lsm().await.unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 429),
|
||||
"the fault that spent the budget must be the one reported: {err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
9,
|
||||
"one call plus MAX_RETRIES — the re-issue budget is not spent on top"
|
||||
);
|
||||
}
|
||||
|
||||
/// A draining node is terminal, but the client does not know that from the
|
||||
/// status: draining and a proxy blip are both 503, and telling them apart
|
||||
/// takes parsing the body for a namespace code. So it spends the retry
|
||||
/// budget and then reports what the server said — the drain gate never
|
||||
/// releases, so the answer does not change, and the operator still reads
|
||||
/// "WAL node draining" in the error.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_draining_surfaces_after_the_retry_budget() {
|
||||
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = calls.clone();
|
||||
let table = Table::new_with_handler("my_table", move |_request| {
|
||||
seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
http::Response::builder()
|
||||
.status(503)
|
||||
.body(r#"{"code":19,"error":"WAL node draining"}"#.to_string())
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
let err = table.checkpoint_lsm().await.unwrap_err();
|
||||
let message = err.to_string();
|
||||
assert!(
|
||||
matches!(&err, Error::Http { status_code: Some(s), .. } if s.as_u16() == 503),
|
||||
"the 503 must surface as itself: {err:?}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("WAL node draining"),
|
||||
"the server's own diagnosis must survive to the caller: {message}"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
9,
|
||||
"one call plus MAX_RETRIES, then it reports rather than spinning"
|
||||
);
|
||||
}
|
||||
|
||||
/// A long stall with nothing compacting must keep waiting, not fail. The
|
||||
/// client cannot judge this: a checkpoint queued behind unrelated tables
|
||||
/// on the pod-wide compactor pool reports exactly these numbers — flat
|
||||
/// generations, an idle latch — as one whose merges are failing. The
|
||||
/// deadline is the caller's.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_checkpoint_waits_out_a_long_stall_rather_than_failing() {
|
||||
let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = polls.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
if path.contains("flush_lsm") || path.contains("compact_lsm") {
|
||||
return accepted();
|
||||
}
|
||||
// Flat for far longer than any bound this loop ever had, with
|
||||
// `compacting: false` throughout — then it drains.
|
||||
let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
ok_json(stats_body(if n < 40 { &[1, 2] } else { &[] }, false))
|
||||
});
|
||||
|
||||
table
|
||||
.checkpoint_lsm()
|
||||
.await
|
||||
.expect("a stall is the server being slow, not the client's call to make");
|
||||
assert!(
|
||||
polls.load(std::sync::atomic::Ordering::SeqCst) > 40,
|
||||
"the loop must have kept polling well past the old ten-poll bound"
|
||||
);
|
||||
}
|
||||
|
||||
/// A pass already owns the latch on every outstanding bucket, so the loop
|
||||
/// waits rather than piling on triggers it would only refuse. This is the
|
||||
/// sole thing `compacting` is read for.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_checkpoint_waits_while_a_pass_is_running() {
|
||||
let polls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let compacts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen_polls = polls.clone();
|
||||
let seen_compacts = compacts.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
if path.contains("flush_lsm") {
|
||||
return accepted();
|
||||
}
|
||||
if path.contains("compact_lsm") {
|
||||
seen_compacts.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
return accepted();
|
||||
}
|
||||
// Latched for many polls, then done.
|
||||
let n = seen_polls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
ok_json(if n > 15 {
|
||||
stats_body(&[], false)
|
||||
} else {
|
||||
stats_body(&[1], true)
|
||||
})
|
||||
});
|
||||
|
||||
table
|
||||
.checkpoint_lsm()
|
||||
.await
|
||||
.expect("a running pass is progress, not a stall");
|
||||
assert_eq!(
|
||||
compacts.load(std::sync::atomic::Ordering::SeqCst),
|
||||
0,
|
||||
"never trigger against a bucket already compacting"
|
||||
);
|
||||
}
|
||||
|
||||
/// WAL off ⇒ `None`; WAL on ⇒ a fully populated `Some` with no field
|
||||
/// defaulting to a zero it did not measure. `include_generation_rows`
|
||||
/// rides in the body and is off unless asked for.
|
||||
#[tokio::test]
|
||||
async fn test_get_lsm_stats_round_trip() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
assert_eq!(request.url().path(), "/v1/table/my_table/get_lsm_stats/");
|
||||
let body = request.body().unwrap().as_bytes().unwrap();
|
||||
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||
assert_eq!(
|
||||
body["include_generation_rows"], true,
|
||||
"the flag must reach the server, not be silently dropped"
|
||||
);
|
||||
let response = serde_json::json!({
|
||||
"lsm_stats": {
|
||||
"buckets": [{
|
||||
"shard_id": "b0",
|
||||
"status": "Active",
|
||||
"writer_epoch": 3,
|
||||
"manifest_version": 11,
|
||||
"current_generation": 9,
|
||||
"replay_after_wal_entry_position": 100,
|
||||
"wal_entry_position_last_seen": 140,
|
||||
"generations": [{ "generation": 8, "bytes": 4096, "rows": 30 }],
|
||||
"compacting": false,
|
||||
"memtables": [
|
||||
{ "generation": 9, "rows": 12, "bytes": 900, "batches": 2,
|
||||
"indexes": ["vec_idx"] }
|
||||
],
|
||||
}],
|
||||
}
|
||||
});
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(response.to_string())
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
let stats = table
|
||||
.get_lsm_stats(true)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("a WAL-backed table reports Some");
|
||||
let bucket = &stats.buckets[0];
|
||||
assert_eq!(bucket.replay_after_wal_entry_position, 100);
|
||||
assert_eq!(bucket.wal_entry_position_last_seen, 140);
|
||||
assert!(!bucket.compacting);
|
||||
assert_eq!(bucket.generations[0].generation, 8);
|
||||
assert_eq!(bucket.generations[0].rows, Some(30));
|
||||
// The line that answers "why is my fresh-tier vector search
|
||||
// brute-force" — an absent index name is the whole explanation.
|
||||
let memtables = bucket.memtables.as_ref().unwrap();
|
||||
assert_eq!(memtables[0].indexes, vec!["vec_idx".to_string()]);
|
||||
}
|
||||
|
||||
/// A 404 arrives as `TableNotFound`, not as a lost claim the loop
|
||||
/// re-issues from flush until its cap. The two are distinguished by
|
||||
/// status: 404 is "no such table", 421 is "this node holds no claim".
|
||||
/// They shared 404 once, and the loop chased a name that never existed.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_missing_table_is_not_read_as_a_lost_claim() {
|
||||
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = calls.clone();
|
||||
let table = Table::new_with_handler("my_table", move |_request| {
|
||||
seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
http::Response::builder()
|
||||
.status(404)
|
||||
.body(r#"{"code":4,"error":"Not found: Table not found: my_table"}"#.to_string())
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
let err = table.checkpoint_lsm().await.unwrap_err();
|
||||
assert!(
|
||||
matches!(err, Error::TableNotFound { .. }),
|
||||
"a missing table must say so: {err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"no point re-claiming a table that does not exist"
|
||||
);
|
||||
}
|
||||
|
||||
/// A lost claim — 421, not 404 — does re-issue from flush, the call that
|
||||
/// re-claims and replays.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_registry_miss_reissues_from_flush() {
|
||||
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let seen = calls.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
let path = request.url().path().to_string();
|
||||
let n = seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
if path.contains("flush_lsm") {
|
||||
// First flush lands; the claim is then lost, and the
|
||||
// re-issued flush succeeds.
|
||||
return accepted();
|
||||
}
|
||||
if path.contains("compact_lsm") {
|
||||
if n < 4 {
|
||||
return http::Response::builder()
|
||||
.status(421)
|
||||
.body(r#"{"code":19,"error":"table not claimed"}"#.to_string())
|
||||
.unwrap();
|
||||
}
|
||||
return accepted();
|
||||
}
|
||||
ok_json(stats_body(if n < 6 { &[1] } else { &[] }, false))
|
||||
});
|
||||
|
||||
table
|
||||
.checkpoint_lsm()
|
||||
.await
|
||||
.expect("a lost claim must be recovered by re-flushing, not surfaced");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_lsm_stats_absent_when_wal_off() {
|
||||
let table = Table::new_with_handler("my_table", |_request| {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(serde_json::json!({ "lsm_stats": null }).to_string())
|
||||
.unwrap()
|
||||
});
|
||||
assert!(table.get_lsm_stats(false).await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_wait_for_index() {
|
||||
let table = _make_table_with_indices(0);
|
||||
|
||||
@@ -90,7 +90,7 @@ struct RemoteBlobState {
|
||||
|
||||
/// Seekable Cloud blob handle over HTTP Range.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct RemoteBlobFile {
|
||||
pub struct RemoteBlobFile {
|
||||
requester: Arc<dyn BlobRangeRequester>,
|
||||
state: Mutex<RemoteBlobState>,
|
||||
closed: AtomicBool,
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::table::{AddResult, MergeResult};
|
||||
/// same Arrow-IPC streaming body and error side-channel; only the target
|
||||
/// endpoint, query parameters, and parsed result type differ.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum WriteOp {
|
||||
pub enum WriteOp {
|
||||
/// `add`: stream to `/v1/table/{id}/insert/`, optionally overwriting.
|
||||
Insert { overwrite: bool },
|
||||
/// `merge_insert`: stream to `/v1/table/{id}/merge_insert/` with the merge
|
||||
@@ -49,7 +49,7 @@ pub(crate) enum WriteOp {
|
||||
/// The parsed server response for a completed write, discriminated by the
|
||||
/// operation that produced it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum WriteResult {
|
||||
pub enum WriteResult {
|
||||
Add(AddResult),
|
||||
Merge(MergeResult),
|
||||
}
|
||||
|
||||
@@ -68,10 +68,12 @@ use self::merge::MergeInsertBuilder;
|
||||
pub mod add_columns;
|
||||
mod add_data;
|
||||
pub mod branch_merge;
|
||||
pub mod checkpoint;
|
||||
mod create_index;
|
||||
pub mod datafusion;
|
||||
pub(crate) mod dataset;
|
||||
pub mod delete;
|
||||
pub mod lsm_stats;
|
||||
pub mod merge;
|
||||
pub mod optimize;
|
||||
mod primary_key;
|
||||
@@ -95,6 +97,7 @@ pub use lance::dataset::refs::{BranchContents, Ref, TagContents, Tags as LanceTa
|
||||
pub use lance::dataset::scanner::DatasetRecordBatchStream;
|
||||
use lance::dataset::statistics::DatasetStatisticsExt;
|
||||
pub use lance_index::optimize::OptimizeOptions;
|
||||
pub use lsm_stats::{BucketStats, GenerationStats, LsmStats, MemtableStats};
|
||||
pub use optimize::{
|
||||
CompactionMode, CompactionOptions, IndexRemapMode, OptimizeAction, OptimizeStats,
|
||||
};
|
||||
@@ -687,6 +690,31 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
|
||||
message: "get_lsm_write_spec is not supported on this table type".into(),
|
||||
})
|
||||
}
|
||||
/// Seal every bucket's active memtable into L0.
|
||||
///
|
||||
/// The default implementation returns `NotSupported`.
|
||||
async fn flush_lsm(&self) -> Result<()> {
|
||||
Err(Error::NotSupported {
|
||||
message: "flush_lsm is not supported on this table type".into(),
|
||||
})
|
||||
}
|
||||
/// Trigger a background L0 → base compaction pass per bucket.
|
||||
///
|
||||
/// The default implementation returns `NotSupported`.
|
||||
async fn compact_lsm(&self) -> Result<()> {
|
||||
Err(Error::NotSupported {
|
||||
message: "compact_lsm is not supported on this table type".into(),
|
||||
})
|
||||
}
|
||||
/// Read live LSM state, or `None` when the LSM write path is not
|
||||
/// enabled for this table.
|
||||
///
|
||||
/// The default implementation returns `NotSupported`.
|
||||
async fn get_lsm_stats(&self, _include_generation_rows: bool) -> Result<Option<LsmStats>> {
|
||||
Err(Error::NotSupported {
|
||||
message: "get_lsm_stats is not supported on this table type".into(),
|
||||
})
|
||||
}
|
||||
/// Drain and close any cached MemWAL shard writers for this table.
|
||||
///
|
||||
/// The default implementation is a no-op; table types that maintain
|
||||
@@ -1728,6 +1756,85 @@ impl Table {
|
||||
self.inner.get_lsm_write_spec().await
|
||||
}
|
||||
|
||||
/// Converge this table's LSM write path into its base table.
|
||||
///
|
||||
/// One `flush` to seal every memtable into L0, then compaction triggers
|
||||
/// until every generation that existed at that moment has reached base.
|
||||
/// The loop runs client-side, reading progress from `get_lsm_stats`, so
|
||||
/// there is no held socket and nothing to reconcile if you drop this
|
||||
/// future partway through.
|
||||
///
|
||||
/// **Best-effort.** Generations created *after* the opening flush are
|
||||
/// deliberately not waited on — that is what lets this terminate on a
|
||||
/// table taking writes. Idempotent and safe on a cadence: an
|
||||
/// already-converged table costs two round trips and triggers nothing.
|
||||
///
|
||||
/// **No deadline, and the caller owns that.** It returns when the target
|
||||
/// generations are gone, propagates a terminal server fault, and
|
||||
/// otherwise waits however long the server takes. A slow table and a
|
||||
/// stuck one are the same picture from here: the compactor pool is shared
|
||||
/// across every table on the node, so a checkpoint queued behind
|
||||
/// unrelated work is indistinguishable from one that is merging. Wrap
|
||||
/// this in `tokio::time::timeout` for a wall-clock bound; abandoning it
|
||||
/// partway costs nothing.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use lancedb::Table;
|
||||
/// # async fn example(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let before = table.get_lsm_stats(false).await?;
|
||||
/// table.checkpoint_lsm().await?;
|
||||
/// let after = table.get_lsm_stats(false).await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub async fn checkpoint_lsm(&self) -> Result<()> {
|
||||
checkpoint::checkpoint_lsm(self).await
|
||||
}
|
||||
|
||||
/// Seal every bucket's active memtable into L0 without touching the
|
||||
/// base table.
|
||||
///
|
||||
/// Independently useful: flushing makes memtable rows readable from L0 at
|
||||
/// a lower per-query cost. On a node that has not claimed this table it
|
||||
/// claims it and replays the WAL log first — reporting "nothing to flush"
|
||||
/// without replaying would lie about durable data.
|
||||
pub async fn flush_lsm(&self) -> Result<()> {
|
||||
self.inner.flush_lsm().await
|
||||
}
|
||||
|
||||
/// Run one bounded L0 → base compaction pass per bucket, reporting what
|
||||
/// it merged and what is left.
|
||||
///
|
||||
/// One pass, not convergence: that bounds each request's cost and gives a
|
||||
/// caller driving its own cadence a progress signal per round trip.
|
||||
pub async fn compact_lsm(&self) -> Result<()> {
|
||||
self.inner.compact_lsm().await
|
||||
}
|
||||
|
||||
/// Read live per-bucket LSM state.
|
||||
///
|
||||
/// Answers "how far behind is my fresh tier", "which bucket is hot", and
|
||||
/// "why is my fresh-tier vector search brute-force". Mutates no table
|
||||
/// state, though on a node that has not claimed this table it claims it,
|
||||
/// exactly as a read would.
|
||||
///
|
||||
/// `include_generation_rows` reports a row count per L0 generation. Off by
|
||||
/// default: each count opens an uncached Lance dataset, and
|
||||
/// `checkpoint_lsm` polls this needing only generation numbers.
|
||||
///
|
||||
/// `Ok(None)` only when the LSM write path is not enabled, matching
|
||||
/// [`Table::get_lsm_write_spec`]. Stats is fresh-tier only, so with the
|
||||
/// WAL off there is no manifest to report and a struct of zeros would
|
||||
/// read as measurements.
|
||||
///
|
||||
/// Do not build a checkpoint's termination on this: the completion
|
||||
/// predicate lives in the `flush` and `compact` responses.
|
||||
pub async fn get_lsm_stats(&self, include_generation_rows: bool) -> Result<Option<LsmStats>> {
|
||||
self.inner.get_lsm_stats(include_generation_rows).await
|
||||
}
|
||||
|
||||
/// Drain and close any cached MemWAL shard writers held for this table.
|
||||
///
|
||||
/// When an [`LsmWriteSpec`] is installed, `merge_insert` opens MemWAL shard
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Converging a table's LSM write path into its base table.
|
||||
//!
|
||||
//! `checkpoint_lsm` seals once, then triggers compaction and watches
|
||||
//! generation numbers until the L0 that existed at the start is gone.
|
||||
//!
|
||||
//! The loop runs in the client, not the server: `compact_lsm` dispatches a
|
||||
//! pass and returns, so nothing holds a socket and a client can vanish
|
||||
//! mid-operation with nothing to reconcile. Completion is read from
|
||||
//! generation numbers in the shard manifest — durable state, unlike a count
|
||||
//! in a compact response, which a concurrent write invalidates.
|
||||
//!
|
||||
//! The target set is fixed at the start, so generations created *during* the
|
||||
//! checkpoint are ignored. That is what lets it terminate under write load,
|
||||
//! and what makes it best-effort: it converges the fresh tier as of some
|
||||
//! instant. Idempotent, abandonable at any point, safe on a cadence.
|
||||
//!
|
||||
//! No liveness bound — the caller owns the deadline. The compactor pool is
|
||||
//! shared pod-wide, so a checkpoint queued behind unrelated tables looks
|
||||
//! exactly like one that is merging.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::{Error, Result, Table};
|
||||
|
||||
/// The HTTP status a failed request carried, if it carried one.
|
||||
///
|
||||
/// `None` for anything with no retry story: a `TableNotFound` that
|
||||
/// `check_table_response` already translated, or a connection failure that
|
||||
/// never reached the server. Both are terminal.
|
||||
fn status_of(e: &Error) -> Option<u16> {
|
||||
#[cfg(feature = "remote")]
|
||||
{
|
||||
match e {
|
||||
Error::Http {
|
||||
status_code: Some(status),
|
||||
..
|
||||
} => Some(status.as_u16()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "remote"))]
|
||||
{
|
||||
let _ = e;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// 429 (latch held, pool saturated, or the pod replaying its WAL) and 503 (a
|
||||
/// draining node, or a proxy between here and it).
|
||||
///
|
||||
/// The status is the whole signal: the server deliberately keeps contention
|
||||
/// off 503, so a latch collision is a 429. A draining node *is* terminal, but
|
||||
/// it is also a 503 that stays a 503, so retrying spends one budget and then
|
||||
/// reports the server's own message — cheaper than parsing the body for the
|
||||
/// namespace code it would take to tell the two apart.
|
||||
fn is_retryable(e: &Error) -> bool {
|
||||
matches!(status_of(e), Some(429 | 503))
|
||||
}
|
||||
|
||||
/// 421: the owning node holds no claim. Only `flush` re-claims and replays,
|
||||
/// so this cannot be retried in place — the caller has to start over.
|
||||
fn is_lost_claim(e: &Error) -> bool {
|
||||
status_of(e) == Some(421)
|
||||
}
|
||||
|
||||
/// Interval between `get_lsm_stats` polls. One interval is roughly one
|
||||
/// compaction pass, the granularity at which the answer can change.
|
||||
///
|
||||
/// Fixed rather than configurable, matching `wait_for_index`. It costs
|
||||
/// nothing on an already-converged table and at most one interval of tail
|
||||
/// latency after the final pass lands.
|
||||
const POLL_INTERVAL: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Cap on re-issues from `flush` after a 421, so a crash-looping node cannot
|
||||
/// turn flush → compact → 421 → flush into a spin.
|
||||
///
|
||||
/// Deliberately not shared with [`MAX_RETRIES`]: a claim that keeps
|
||||
/// evaporating is a broken node, while contention is routine and wants a real
|
||||
/// budget. One shared counter let a merely contended table exhaust this cap
|
||||
/// and then blame a claim it never lost.
|
||||
const MAX_REISSUES: usize = 3;
|
||||
|
||||
/// Retryable faults tolerated on a *single* request, reset on every success —
|
||||
/// scattered contention across a long checkpoint must not accumulate toward a
|
||||
/// cap. Roughly 16s of retrying against the backoff below.
|
||||
const MAX_RETRIES: usize = 8;
|
||||
|
||||
/// Backoff between retries, doubling up to [`RETRY_BACKOFF_MAX`]. Latch
|
||||
/// contention clears in about the time one pass takes, so start small; a
|
||||
/// saturated pool wants the ceiling.
|
||||
const RETRY_BACKOFF_BASE: Duration = Duration::from_millis(100);
|
||||
const RETRY_BACKOFF_MAX: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Sleep before re-issuing a retryable request.
|
||||
async fn backoff(attempt: usize) {
|
||||
let delay = RETRY_BACKOFF_BASE
|
||||
.saturating_mul(1u32 << attempt.min(8) as u32)
|
||||
.min(RETRY_BACKOFF_MAX);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
|
||||
/// Whether the drain loop finished or needs the table re-claimed first.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum CheckpointOutcome {
|
||||
Done,
|
||||
ReissueFromFlush,
|
||||
}
|
||||
|
||||
/// What one LSM request produced: its value, or word that the owning node
|
||||
/// holds no claim and only `flush` can get it back.
|
||||
enum Attempt<T> {
|
||||
Ok(T),
|
||||
ReissueFromFlush,
|
||||
}
|
||||
|
||||
/// Issue one LSM request, retrying in place while the fault is retryable.
|
||||
///
|
||||
/// The two recoverable faults have separate budgets: contention clears on its
|
||||
/// own and retries here against [`MAX_RETRIES`], while a 421 needs `flush` to
|
||||
/// re-claim, which only the caller can drive.
|
||||
///
|
||||
/// An exhausted budget propagates the last error *as itself* rather than a
|
||||
/// synthesized one — "429 after nine tries" beats "checkpoint failed", and a
|
||||
/// draining node arrives carrying the server's own message.
|
||||
async fn issue<T, F, Fut>(mut call: F) -> Result<Attempt<T>>
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T>>,
|
||||
{
|
||||
let mut retries = 0;
|
||||
loop {
|
||||
let e = match call().await {
|
||||
Ok(value) => return Ok(Attempt::Ok(value)),
|
||||
Err(e) => e,
|
||||
};
|
||||
if is_lost_claim(&e) {
|
||||
return Ok(Attempt::ReissueFromFlush);
|
||||
}
|
||||
if !is_retryable(&e) || retries >= MAX_RETRIES {
|
||||
return Err(e);
|
||||
}
|
||||
backoff(retries).await;
|
||||
retries += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Drive [`Table::checkpoint_lsm`]: seal once, fix the target watermark
|
||||
/// from the resulting L0, then trigger and poll until it drains.
|
||||
pub(crate) async fn checkpoint_lsm(table: &Table) -> Result<()> {
|
||||
for reissue in 0..=MAX_REISSUES {
|
||||
// The seal turns everything written before this call into a
|
||||
// generation, so the watermark has to be read after it. Idempotent:
|
||||
// sealing an empty memtable is a no-op, so a re-issue does not churn
|
||||
// empty generations.
|
||||
match issue(|| table.flush_lsm()).await? {
|
||||
Attempt::Ok(()) => {}
|
||||
Attempt::ReissueFromFlush => {
|
||||
backoff(reissue).await;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let stats = match issue(|| table.get_lsm_stats(false)).await? {
|
||||
Attempt::Ok(stats) => stats,
|
||||
Attempt::ReissueFromFlush => {
|
||||
backoff(reissue).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(stats) = stats else {
|
||||
// Not WAL-backed; `flush_lsm` would have errored first but for a race.
|
||||
return Ok(());
|
||||
};
|
||||
let targets: HashMap<String, u64> = stats
|
||||
.buckets
|
||||
.iter()
|
||||
.filter_map(|b| Some((b.shard_id.clone(), b.newest_generation()?)))
|
||||
.collect();
|
||||
if targets.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match drain_to_targets(table, &targets).await? {
|
||||
CheckpointOutcome::Done => return Ok(()),
|
||||
CheckpointOutcome::ReissueFromFlush => {
|
||||
backoff(reissue).await;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(Error::Runtime {
|
||||
message: "checkpoint_lsm: the owning node kept losing its claim; \
|
||||
re-issued from flush the maximum number of times"
|
||||
.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Trigger and poll until no bucket holds a generation at or below its
|
||||
/// target.
|
||||
///
|
||||
/// No liveness bound, deliberately. The pod-wide compactor pool (a semaphore
|
||||
/// of 2 by default, shared across every table on the node) is taken *inside*
|
||||
/// the pass, after the bucket latch, so a checkpoint queued behind unrelated
|
||||
/// tables is indistinguishable from one that is merging. An idle-poll counter
|
||||
/// here could only ever have fired on a table that would have finished.
|
||||
async fn drain_to_targets(
|
||||
table: &Table,
|
||||
targets: &HashMap<String, u64>,
|
||||
) -> Result<CheckpointOutcome> {
|
||||
loop {
|
||||
let stats = match issue(|| table.get_lsm_stats(false)).await? {
|
||||
Attempt::Ok(stats) => stats,
|
||||
Attempt::ReissueFromFlush => return Ok(CheckpointOutcome::ReissueFromFlush),
|
||||
};
|
||||
let Some(stats) = stats else {
|
||||
return Ok(CheckpointOutcome::Done);
|
||||
};
|
||||
// `compacting` is the bucket's compaction latch, held from dispatch
|
||||
// until the pass ends — including while it waits on the pod-wide
|
||||
// permit. So it answers one question only: do not pile on. Buckets
|
||||
// with nothing outstanding are skipped, not counted as idle.
|
||||
let mut outstanding = 0;
|
||||
let mut all_compacting = true;
|
||||
for b in &stats.buckets {
|
||||
let Some(target) = targets.get(&b.shard_id) else {
|
||||
continue;
|
||||
};
|
||||
let n = b.outstanding_generations(*target);
|
||||
if n > 0 {
|
||||
outstanding += n;
|
||||
all_compacting &= b.compacting;
|
||||
}
|
||||
}
|
||||
if outstanding == 0 {
|
||||
return Ok(CheckpointOutcome::Done);
|
||||
}
|
||||
|
||||
if !all_compacting {
|
||||
match table.compact_lsm().await {
|
||||
Ok(()) => {}
|
||||
Err(e) if is_lost_claim(&e) => return Ok(CheckpointOutcome::ReissueFromFlush),
|
||||
Err(e) if !is_retryable(&e) => return Err(e),
|
||||
// A 429 here means the server could latch no bucket at all,
|
||||
// which the poll above already handles. Not retried in place:
|
||||
// the latch it would contend for is the one doing the work, so
|
||||
// fall through and re-read — `POLL_INTERVAL` is the backoff.
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(POLL_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "remote"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn http(status: u16) -> Error {
|
||||
Error::Http {
|
||||
source: "server said no".into(),
|
||||
request_id: "rid".into(),
|
||||
status_code: reqwest::StatusCode::from_u16(status).ok(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every status the loop acts on. The two predicates are checked together
|
||||
/// because their overlap is what would be wrong: a status must never be
|
||||
/// both, and 421 in particular must not read as retryable — retrying it in
|
||||
/// place re-issues the call that just said the node holds no claim.
|
||||
#[test]
|
||||
fn taxonomy_round_trips() {
|
||||
for status in [429, 503] {
|
||||
assert!(is_retryable(&http(status)), "{status} must retry");
|
||||
assert!(
|
||||
!is_lost_claim(&http(status)),
|
||||
"{status} is not a lost claim"
|
||||
);
|
||||
}
|
||||
assert!(is_lost_claim(&http(421)), "a lost claim must re-claim");
|
||||
assert!(
|
||||
!is_retryable(&http(421)),
|
||||
"retrying a lost claim in place only asks the same node again"
|
||||
);
|
||||
for status in [400, 404, 409, 500] {
|
||||
assert!(!is_retryable(&http(status)), "{status} is terminal");
|
||||
assert!(!is_lost_claim(&http(status)), "{status} is terminal");
|
||||
}
|
||||
}
|
||||
|
||||
/// An error carrying no status has no retry story and must be terminal —
|
||||
/// a connection that never reached the server, or a `TableNotFound` that
|
||||
/// `check_table_response` translated before the loop saw it.
|
||||
#[test]
|
||||
fn errors_without_a_status_are_terminal() {
|
||||
let no_status = Error::Http {
|
||||
source: "connection reset".into(),
|
||||
request_id: "rid".into(),
|
||||
status_code: None,
|
||||
};
|
||||
assert!(!is_retryable(&no_status));
|
||||
assert!(!is_lost_claim(&no_status));
|
||||
|
||||
let translated = Error::TableNotFound {
|
||||
name: "t".into(),
|
||||
source: "gone".into(),
|
||||
};
|
||||
assert!(!is_retryable(&translated));
|
||||
assert!(!is_lost_claim(&translated));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Live per-bucket LSM state — the shape [`crate::Table::get_lsm_stats`]
|
||||
//! returns and [`super::checkpoint`] polls.
|
||||
//!
|
||||
//! Nothing here is derived: sums and differences (total L0 bytes, WAL lag)
|
||||
//! are the caller's to compute. There is no "WAL is off" shape — that case is
|
||||
//! `None`, because a struct of zeros would read as measurements.
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
/// One flushed L0 generation.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct GenerationStats {
|
||||
pub generation: u64,
|
||||
pub bytes: u64,
|
||||
/// Present only when `include_generation_rows` was requested. Off by
|
||||
/// default because each count opens an uncached Lance dataset, and the
|
||||
/// checkpoint loop polls this route needing only generation numbers.
|
||||
#[serde(default)]
|
||||
pub rows: Option<u64>,
|
||||
}
|
||||
|
||||
/// One in-memory memtable.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct MemtableStats {
|
||||
pub generation: u64,
|
||||
pub rows: u64,
|
||||
pub bytes: u64,
|
||||
pub batches: u64,
|
||||
/// Names of the indexes this memtable carries. An absent name is the whole
|
||||
/// answer to "why is my fresh-tier search on that column brute-force".
|
||||
pub indexes: Vec<String>,
|
||||
}
|
||||
|
||||
/// Live state of one bucket. A table is N buckets on one node; flattening to
|
||||
/// a single number hides the one hot bucket that is usually why someone
|
||||
/// opened this endpoint.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct BucketStats {
|
||||
pub shard_id: String,
|
||||
/// `Active` | `Sealed` (drop-table 2PC in flight).
|
||||
pub status: String,
|
||||
pub writer_epoch: u64,
|
||||
pub manifest_version: u64,
|
||||
pub current_generation: u64,
|
||||
pub replay_after_wal_entry_position: u64,
|
||||
pub wal_entry_position_last_seen: u64,
|
||||
pub generations: Vec<GenerationStats>,
|
||||
/// Whether a pass owns this bucket's compaction latch right now. Says *a*
|
||||
/// driver is running, not *whose*, and the latch is held from dispatch —
|
||||
/// including while the pass queues for a pod-wide compactor permit. Read
|
||||
/// it as "do not pile on", never as "mine is progressing".
|
||||
pub compacting: bool,
|
||||
/// Oldest first, active last. Absent for a `Sealed` bucket, whose
|
||||
/// in-memory state is torn down.
|
||||
#[serde(default)]
|
||||
pub memtables: Option<Vec<MemtableStats>>,
|
||||
}
|
||||
|
||||
impl BucketStats {
|
||||
/// The newest flushed generation, or `None` when L0 is empty.
|
||||
pub(crate) fn newest_generation(&self) -> Option<u64> {
|
||||
self.generations.iter().map(|g| g.generation).max()
|
||||
}
|
||||
|
||||
/// How many generations at or below `target` are still in L0.
|
||||
///
|
||||
/// A count, not a boolean: one pass drains a bounded prefix rather than
|
||||
/// the whole target set, so a boolean would read as "no progress" for
|
||||
/// every pass but the last. Compaction drains oldest-first, so this
|
||||
/// decreases monotonically.
|
||||
pub(crate) fn outstanding_generations(&self, target: u64) -> usize {
|
||||
self.generations
|
||||
.iter()
|
||||
.filter(|g| g.generation <= target)
|
||||
.count()
|
||||
}
|
||||
}
|
||||
|
||||
/// Live LSM state, one entry per bucket.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct LsmStats {
|
||||
pub buckets: Vec<BucketStats>,
|
||||
}
|
||||
|
||||
/// Server-side JSON envelope for `get_lsm_stats`. `lsm_stats` is null when
|
||||
/// the table has no LSM write path.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct GetLsmStatsResponse {
|
||||
#[serde(default)]
|
||||
pub lsm_stats: Option<LsmStats>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn bucket(shard: &str, generations: &[u64], compacting: bool) -> BucketStats {
|
||||
BucketStats {
|
||||
shard_id: shard.into(),
|
||||
status: "Active".into(),
|
||||
writer_epoch: 1,
|
||||
manifest_version: 1,
|
||||
current_generation: generations.iter().max().copied().unwrap_or(0) + 1,
|
||||
replay_after_wal_entry_position: 0,
|
||||
wal_entry_position_last_seen: 0,
|
||||
generations: generations
|
||||
.iter()
|
||||
.map(|g| GenerationStats {
|
||||
generation: *g,
|
||||
bytes: 1,
|
||||
rows: None,
|
||||
})
|
||||
.collect(),
|
||||
compacting,
|
||||
memtables: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The target watermark is the newest generation at the start, and a
|
||||
/// generation created after it must not hold the loop open — that is why
|
||||
/// the predicate terminates under write load.
|
||||
#[test]
|
||||
fn newer_generations_do_not_extend_the_target() {
|
||||
let start = bucket("b0", &[7, 8], false);
|
||||
let target = start.newest_generation().expect("L0 is non-empty");
|
||||
assert_eq!(target, 8);
|
||||
|
||||
// Compaction drained 7 and 8; 9 and 10 arrived while it ran.
|
||||
let later = bucket("b0", &[9, 10], false);
|
||||
assert_eq!(
|
||||
later.outstanding_generations(target),
|
||||
0,
|
||||
"generations above the target are somebody else's problem"
|
||||
);
|
||||
|
||||
// Still holding 8 means still outstanding.
|
||||
assert_eq!(
|
||||
bucket("b0", &[8, 9], false).outstanding_generations(target),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
/// The metric counts generations, not buckets: a pass drains a bounded
|
||||
/// prefix, so one bucket going 3 → 2 → 1 → 0 is three steps.
|
||||
#[test]
|
||||
fn progress_is_measured_in_generations() {
|
||||
let target = 3;
|
||||
let counts: Vec<usize> = [&[1u64, 2, 3][..], &[2, 3][..], &[3][..], &[][..]]
|
||||
.iter()
|
||||
.map(|gens| bucket("b0", gens, false).outstanding_generations(target))
|
||||
.collect();
|
||||
assert_eq!(counts, vec![3, 2, 1, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_l0_has_no_target() {
|
||||
assert!(bucket("b0", &[], false).newest_generation().is_none());
|
||||
}
|
||||
}
|
||||
@@ -315,7 +315,10 @@ pub(crate) async fn execute_merge_insert(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader};
|
||||
use arrow_array::builder::FixedSizeBinaryBuilder;
|
||||
use arrow_array::{
|
||||
Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, UInt64Array,
|
||||
};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -337,6 +340,42 @@ mod tests {
|
||||
Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema))
|
||||
}
|
||||
|
||||
fn fixed_size_binary_merge_batch(
|
||||
id_range: std::ops::Range<u64>,
|
||||
price: u64,
|
||||
) -> Box<dyn RecordBatchReader + Send> {
|
||||
let ids = id_range.collect::<Vec<_>>();
|
||||
let mut id_builder = FixedSizeBinaryBuilder::new(16);
|
||||
for id in &ids {
|
||||
let mut bytes = [0; 16];
|
||||
bytes[..8].copy_from_slice(&id.to_le_bytes());
|
||||
id_builder.append_value(bytes).unwrap();
|
||||
}
|
||||
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::FixedSizeBinary(16), false),
|
||||
Field::new("id_as_int", DataType::UInt64, false),
|
||||
Field::new("name", DataType::Utf8, false),
|
||||
Field::new("market", DataType::Utf8, false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(id_builder.finish()),
|
||||
Arc::new(UInt64Array::from_iter_values(ids.iter().copied())),
|
||||
Arc::new(StringArray::from_iter_values(
|
||||
ids.iter().map(|id| format!("name{id}")),
|
||||
)),
|
||||
Arc::new(StringArray::from_iter_values(std::iter::repeat_n(
|
||||
format!("market_{price}"),
|
||||
ids.len(),
|
||||
))),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_merge_insert() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
@@ -388,6 +427,36 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_merge_insert_fixed_size_binary_non_nullable() {
|
||||
// Regression test for #2869: an unrelated FixedSizeBinary column used to corrupt the
|
||||
// outer join that implements when_not_matched_by_source_delete.
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let table = conn
|
||||
.create_table(
|
||||
"fixed_size_binary_merge",
|
||||
fixed_size_binary_merge_batch(0..256, 100),
|
||||
)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut merge_insert = table.merge_insert(&["id_as_int"]);
|
||||
merge_insert
|
||||
.when_matched_update_all(None)
|
||||
.when_not_matched_insert_all()
|
||||
.when_not_matched_by_source_delete(None);
|
||||
let result = merge_insert
|
||||
.execute(fixed_size_binary_merge_batch(100..356, 200))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.num_updated_rows, 156);
|
||||
assert_eq!(result.num_inserted_rows, 100);
|
||||
assert_eq!(result.num_deleted_rows, 100);
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 256);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_merge_insert_use_index() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
|
||||
@@ -214,12 +214,17 @@ pub(crate) async fn execute_optimize(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use arrow_array::{Int32Array, RecordBatch, StringArray};
|
||||
use arrow_array::{
|
||||
Array, FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray,
|
||||
};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use lance_arrow::FixedSizeListArrayExt;
|
||||
use rstest::rstest;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::connect;
|
||||
use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS;
|
||||
use crate::index::vector::IvfRqIndexBuilder;
|
||||
use crate::index::{Index, scalar::BTreeIndexBuilder};
|
||||
use crate::query::ExecutableQuery;
|
||||
use crate::table::{CompactionOptions, OptimizeAction, OptimizeStats};
|
||||
@@ -304,6 +309,96 @@ mod tests {
|
||||
assert_eq!(all_values, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_compact_with_concurrent_add() {
|
||||
const NUM_FRAGMENTS: usize = 5;
|
||||
const ROWS_PER_FRAGMENT: i32 = 300;
|
||||
|
||||
let tmpdir = tempfile::tempdir().unwrap();
|
||||
let conn = connect(tmpdir.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema,
|
||||
vec![Arc::new(Int32Array::from_iter_values(0..ROWS_PER_FRAGMENT))],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let table = conn
|
||||
.create_table("test_concurrent_compact", batch.clone())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
table
|
||||
.create_index(&["id"], Index::BTree(BTreeIndexBuilder::default()))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
for _ in 0..NUM_FRAGMENTS {
|
||||
table.add(batch.clone()).execute().await.unwrap();
|
||||
}
|
||||
|
||||
// Use separate handles so the two writes actually overlap, as they can
|
||||
// when different Node connections operate on the same S3 table.
|
||||
let compact_table = conn
|
||||
.open_table("test_concurrent_compact")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let append_table = conn
|
||||
.open_table("test_concurrent_compact")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let compact_task = tokio::spawn(async move {
|
||||
compact_table
|
||||
.optimize(OptimizeAction::Compact {
|
||||
options: CompactionOptions {
|
||||
target_rows_per_fragment: 1_000,
|
||||
..Default::default()
|
||||
},
|
||||
remap_options: None,
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
for _ in 0..NUM_FRAGMENTS {
|
||||
append_table.add(batch.clone()).execute().await.unwrap();
|
||||
}
|
||||
compact_task.await.unwrap().unwrap();
|
||||
|
||||
let table = conn
|
||||
.open_table("test_concurrent_compact")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let dataset = table.dataset().unwrap().get().await.unwrap();
|
||||
let fragment_ids = dataset
|
||||
.get_fragments()
|
||||
.iter()
|
||||
.map(|fragment| fragment.id())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(fragment_ids.windows(2).all(|ids| ids[0] < ids[1]));
|
||||
|
||||
// A second compaction exposed the original out-of-order row-id bug.
|
||||
table
|
||||
.optimize(OptimizeAction::Compact {
|
||||
options: CompactionOptions {
|
||||
target_rows_per_fragment: 1_000,
|
||||
..Default::default()
|
||||
},
|
||||
remap_options: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
table.count_rows(None).await.unwrap(),
|
||||
ROWS_PER_FRAGMENT as usize * (NUM_FRAGMENTS * 2 + 1)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optimize_prune_versions() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
@@ -442,6 +537,58 @@ mod tests {
|
||||
assert_eq!(final_row_count, 200);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optimize_vector_index_after_delete_with_stable_row_ids() {
|
||||
const NUM_ROWS: i32 = 400;
|
||||
const DIMENSION: i32 = 32;
|
||||
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
let vectors = FixedSizeListArray::try_new_from_values(
|
||||
Float32Array::from_iter_values((0..NUM_ROWS).flat_map(|id| {
|
||||
(0..DIMENSION).map(move |offset| ((id as f32 * 0.1) + (offset as f32 * 0.3)).sin())
|
||||
})),
|
||||
DIMENSION,
|
||||
)
|
||||
.unwrap();
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int32, false),
|
||||
Field::new("vector", vectors.data_type().clone(), false),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema,
|
||||
vec![
|
||||
Arc::new(Int32Array::from_iter_values(0..NUM_ROWS)),
|
||||
Arc::new(vectors),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let table = conn
|
||||
.create_table("test_vector_index_optimize_after_delete", batch)
|
||||
.storage_option(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, "true")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
table
|
||||
.create_index(
|
||||
&["vector"],
|
||||
Index::IvfRq(IvfRqIndexBuilder::default().num_partitions(4)),
|
||||
)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
table.delete("id % 3 = 0").await.unwrap();
|
||||
|
||||
// Regression test for #3330: deleted stable row IDs used to become
|
||||
// misaligned with row addresses while joining small IVF partitions.
|
||||
table
|
||||
.optimize(OptimizeAction::Index(Default::default()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 266);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optimize_all() {
|
||||
let conn = connect("memory://").execute().await.unwrap();
|
||||
|
||||
@@ -10,7 +10,7 @@ use arrow_array::{
|
||||
use arrow_schema::{DataType, Field, Fields, Schema};
|
||||
use futures::TryStreamExt;
|
||||
use lance::Dataset;
|
||||
use lance_encoding::version::LanceFileVersion;
|
||||
use lance_file::version::LanceFileVersion;
|
||||
use lancedb::{
|
||||
Connection, Error, Result, Table,
|
||||
blob::{BlobRangeRequest, blob},
|
||||
|
||||
Reference in New Issue
Block a user