mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 22:26:53 +00:00
Merge commit '872dfb2987b1f08d2bd88351a7498287bf552972' into xuanwo/function-oci-product
# Conflicts: # nodejs/Cargo.toml
This commit is contained in:
Xuanwo
Generated
+43
-42
@@ -3491,8 +3491,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4863,8 +4863,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4936,8 +4936,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4959,7 +4959,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4973,7 +4973,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4982,8 +4982,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4993,8 +4993,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5031,8 +5031,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5062,8 +5062,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5080,8 +5080,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5090,8 +5090,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5124,8 +5124,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5150,14 +5150,15 @@ dependencies = [
|
||||
"prost",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"serde",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5221,8 +5222,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5244,8 +5245,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5285,8 +5286,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5300,8 +5301,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5315,8 +5316,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5369,8 +5370,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5384,8 +5385,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5425,8 +5426,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5439,8 +5440,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "12.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93"
|
||||
version = "13.0.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
@@ -26,7 +26,8 @@ const olderThan = new Date();
|
||||
olderThan.setDate(olderThan.getDate() - 1));
|
||||
tbl.optimize({cleanupOlderThan: olderThan});
|
||||
|
||||
// Delete all versions except the current version
|
||||
// Delete versions committed before this point. Versions created by the
|
||||
// optimize call itself are newer than the cutoff and will be retained.
|
||||
tbl.optimize({cleanupOlderThan: new Date()});
|
||||
```
|
||||
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>12.0.0-beta.18</lance-core.version>
|
||||
<lance-core.version>13.0.0-beta.1</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
@@ -37,6 +37,7 @@ lzma-sys = { version = "0.1", features = ["static"] }
|
||||
log.workspace = true
|
||||
|
||||
# Pin to resolve build failures; update periodically for security patches.
|
||||
# rustls >= 0.23.45 (RUSTSEC-2026-0285) needs aws-lc-rs >= 1.18.
|
||||
aws-lc-sys = "=0.45.0"
|
||||
aws-lc-rs = "=1.18.1"
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@ import {
|
||||
Operator,
|
||||
instanceOfFullTextQuery,
|
||||
} from "../lancedb/query";
|
||||
import { LocalTable } from "../lancedb/table";
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
"Given a table",
|
||||
@@ -2765,6 +2766,15 @@ describe("when dealing with tags", () => {
|
||||
});
|
||||
});
|
||||
|
||||
/** Returns a Date strictly later than every instant observed before the call. */
|
||||
async function nextMillisecond(): Promise<Date> {
|
||||
const start = Date.now();
|
||||
while (Date.now() <= start) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
}
|
||||
return new Date();
|
||||
}
|
||||
|
||||
describe("when optimizing a dataset", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
@@ -2787,9 +2797,14 @@ describe("when optimizing a dataset", () => {
|
||||
});
|
||||
|
||||
it("cleanups old versions", async () => {
|
||||
const stats = await table.optimize({ cleanupOlderThan: new Date() });
|
||||
// Lance stores version timestamps with nanosecond precision while a JS
|
||||
// Date only has millisecond precision. A cutoff captured in the same
|
||||
// millisecond as the last commit would truncate to *before* that commit
|
||||
// and leave it in place, so wait for the clock to tick over first.
|
||||
const cutoff = await nextMillisecond();
|
||||
const stats = await table.optimize({ cleanupOlderThan: cutoff });
|
||||
expect(stats.prune.bytesRemoved).toBeGreaterThan(0);
|
||||
expect(stats.prune.oldVersionsRemoved).toBe(3);
|
||||
expect(stats.prune.oldVersionsRemoved).toBe(2);
|
||||
});
|
||||
|
||||
it("delete unverified", async () => {
|
||||
@@ -2810,6 +2825,24 @@ describe("when optimizing a dataset", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("passes cleanupOlderThan to the native binding as an absolute timestamp", async () => {
|
||||
const optimize = jest.fn().mockResolvedValue({
|
||||
compaction: {
|
||||
filesAdded: 0,
|
||||
filesRemoved: 0,
|
||||
fragmentsAdded: 0,
|
||||
fragmentsRemoved: 0,
|
||||
},
|
||||
prune: { bytesRemoved: 0, oldVersionsRemoved: 0 },
|
||||
});
|
||||
const table = new LocalTable({ optimize } as never);
|
||||
const cutoff = new Date("2020-01-02T03:04:05.678Z");
|
||||
|
||||
await table.optimize({ cleanupOlderThan: cutoff, deleteUnverified: true });
|
||||
|
||||
expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
|
||||
});
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
"when optimizing a dataset",
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
|
||||
+3
-10
@@ -148,7 +148,8 @@ export interface OptimizeOptions {
|
||||
* olderThan.setDate(olderThan.getDate() - 1));
|
||||
* tbl.optimize({cleanupOlderThan: olderThan});
|
||||
*
|
||||
* // Delete all versions except the current version
|
||||
* // Delete versions committed before this point. Versions created by the
|
||||
* // optimize call itself are newer than the cutoff and will be retained.
|
||||
* tbl.optimize({cleanupOlderThan: new Date()});
|
||||
*/
|
||||
cleanupOlderThan: Date;
|
||||
@@ -1486,16 +1487,8 @@ export class LocalTable extends Table {
|
||||
}
|
||||
|
||||
async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
|
||||
let cleanupOlderThanMs;
|
||||
if (
|
||||
options?.cleanupOlderThan !== undefined &&
|
||||
options?.cleanupOlderThan !== null
|
||||
) {
|
||||
cleanupOlderThanMs =
|
||||
new Date().getTime() - options.cleanupOlderThan.getTime();
|
||||
}
|
||||
return await this.inner.optimize(
|
||||
cleanupOlderThanMs,
|
||||
options?.cleanupOlderThan?.getTime(),
|
||||
options?.deleteUnverified,
|
||||
);
|
||||
}
|
||||
|
||||
+27
-23
@@ -7,7 +7,7 @@ use chrono::{DateTime, Utc};
|
||||
|
||||
use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||
use lancedb::table::{
|
||||
AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration,
|
||||
AddDataMode, ColumnAlteration as LanceColumnAlteration,
|
||||
FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
|
||||
NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
|
||||
};
|
||||
@@ -677,22 +677,20 @@ impl Table {
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn optimize(
|
||||
&self,
|
||||
older_than_ms: Option<i64>,
|
||||
before_timestamp_ms: Option<i64>,
|
||||
delete_unverified: Option<bool>,
|
||||
) -> napi::Result<OptimizeStats> {
|
||||
let inner = self.inner_ref()?;
|
||||
|
||||
let older_than = if let Some(ms) = older_than_ms {
|
||||
if ms == i64::MIN {
|
||||
return Err(napi::Error::from_reason(format!(
|
||||
"older_than_ms can not be {}",
|
||||
i32::MIN,
|
||||
)));
|
||||
}
|
||||
Duration::try_milliseconds(ms)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let before_timestamp = before_timestamp_ms
|
||||
.map(|ms| {
|
||||
DateTime::from_timestamp_millis(ms).ok_or_else(|| {
|
||||
napi::Error::from_reason(format!(
|
||||
"cleanupOlderThan timestamp is out of range: {ms}"
|
||||
))
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
let compaction_stats = inner
|
||||
.optimize(OptimizeAction::Compact {
|
||||
@@ -703,16 +701,22 @@ impl Table {
|
||||
.default_error()?
|
||||
.compaction
|
||||
.unwrap();
|
||||
let prune_stats = inner
|
||||
.optimize(OptimizeAction::Prune {
|
||||
older_than,
|
||||
delete_unverified,
|
||||
error_if_tagged_old_versions: None,
|
||||
})
|
||||
.await
|
||||
.default_error()?
|
||||
.prune
|
||||
.unwrap();
|
||||
let prune_stats = if let Some(before_timestamp) = before_timestamp {
|
||||
inner
|
||||
.optimize_prune_before(before_timestamp, delete_unverified, None)
|
||||
.await
|
||||
} else {
|
||||
inner
|
||||
.optimize(OptimizeAction::Prune {
|
||||
older_than: None,
|
||||
delete_unverified,
|
||||
error_if_tagged_old_versions: None,
|
||||
})
|
||||
.await
|
||||
}
|
||||
.default_error()?
|
||||
.prune
|
||||
.unwrap();
|
||||
inner
|
||||
.optimize(lancedb::table::OptimizeAction::Index(
|
||||
OptimizeOptions::default(),
|
||||
|
||||
@@ -113,6 +113,10 @@ aws-sdk-s3 = { version = "1.55.0" }
|
||||
aws-sdk-kms = { version = "1.48.0" }
|
||||
aws-config = { version = "1.5.10" }
|
||||
aws-smithy-runtime = { version = "1.9.1" }
|
||||
# Constraint only: types 1.7 breaks aws-smithy-json 0.63, which aws-config still
|
||||
# requires. Bounds must stay inside 1.x and allow the MSRV job's 1.3.6 pin.
|
||||
# Drop once aws-config moves to aws-smithy-json 0.64.
|
||||
aws-smithy-types = { version = ">=1.0, <1.7" }
|
||||
datafusion.workspace = true
|
||||
http-body = "1" # Matching reqwest
|
||||
rstest = "0.23.0"
|
||||
|
||||
@@ -1741,6 +1741,33 @@ impl Table {
|
||||
self.inner.optimize(action).await
|
||||
}
|
||||
|
||||
/// Prune versions committed before an absolute timestamp.
|
||||
///
|
||||
/// This is an internal entry point for language bindings whose public API
|
||||
/// accepts an absolute cleanup cutoff.
|
||||
#[doc(hidden)]
|
||||
pub async fn optimize_prune_before(
|
||||
&self,
|
||||
before_timestamp: chrono::DateTime<chrono::Utc>,
|
||||
delete_unverified: Option<bool>,
|
||||
error_if_tagged_old_versions: Option<bool>,
|
||||
) -> Result<OptimizeStats> {
|
||||
let native = self.as_native().ok_or_else(|| Error::NotSupported {
|
||||
message: "optimize is not supported on LanceDB cloud.".into(),
|
||||
})?;
|
||||
let prune = optimize::cleanup_old_versions_before(
|
||||
native,
|
||||
before_timestamp,
|
||||
delete_unverified,
|
||||
error_if_tagged_old_versions,
|
||||
)
|
||||
.await?;
|
||||
Ok(OptimizeStats {
|
||||
compaction: None,
|
||||
prune: Some(prune),
|
||||
})
|
||||
}
|
||||
|
||||
/// Add new columns to the table, providing values to fill in.
|
||||
pub fn add_columns(&self) -> AddColumnsBuilder {
|
||||
AddColumnsBuilder::new(self.inner.clone())
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use lance::dataset::cleanup::RemovalStats;
|
||||
use chrono::{DateTime, Utc};
|
||||
use lance::dataset::cleanup::{CleanupPolicyBuilder, RemovalStats};
|
||||
use lance::dataset::optimize::{CompactionMetrics, IndexRemapperOptions, compact_files};
|
||||
use lance::index::DatasetIndexExt;
|
||||
use lance_index::optimize::OptimizeOptions;
|
||||
@@ -147,6 +148,33 @@ pub(crate) async fn cleanup_old_versions(
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Remove dataset versions committed before an absolute timestamp.
|
||||
pub(crate) async fn cleanup_old_versions_before(
|
||||
table: &NativeTable,
|
||||
before_timestamp: DateTime<Utc>,
|
||||
delete_unverified: Option<bool>,
|
||||
error_if_tagged_old_versions: Option<bool>,
|
||||
) -> Result<RemovalStats> {
|
||||
table.dataset.ensure_mutable()?;
|
||||
let dataset = table.dataset.get().await?;
|
||||
let mut policy = CleanupPolicyBuilder::default().before_timestamp(before_timestamp);
|
||||
if let Some(delete_unverified) = delete_unverified {
|
||||
policy = policy.delete_unverified(delete_unverified);
|
||||
}
|
||||
if let Some(error_if_tagged_old_versions) = error_if_tagged_old_versions {
|
||||
policy = policy.error_if_tagged_old_versions(error_if_tagged_old_versions);
|
||||
}
|
||||
let stats = dataset.cleanup_with_policy(policy.build()).await?;
|
||||
// Computed-column signature sidecars live outside lance's directories;
|
||||
// drop the ones the surviving versions no longer reference.
|
||||
let removed =
|
||||
super::freshness::prune_sidecars(&dataset, delete_unverified.unwrap_or(false)).await?;
|
||||
if removed > 0 {
|
||||
log::debug!("removed {removed} unreferenced computed-column signature sidecars");
|
||||
}
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
/// Compact files in the dataset.
|
||||
///
|
||||
/// This can be run after making several small appends to optimize the table
|
||||
|
||||
@@ -978,6 +978,43 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Absolute timestamp pruning applies the same computed-column sidecar
|
||||
/// cleanup as duration-based pruning.
|
||||
#[tokio::test]
|
||||
async fn test_absolute_pruning_drops_the_sidecars_of_pruned_versions() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let conn = connect(dir.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let batch = record_batch!(("x", Int32, [1, 2])).unwrap();
|
||||
let table = conn
|
||||
.create_table("sidecars", batch)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
declare_doubled(&table).await.unwrap();
|
||||
table.refresh_column("doubled").await.unwrap();
|
||||
append(&table, vec![5]).await;
|
||||
table.refresh_column("doubled").await.unwrap();
|
||||
let sidecars = || {
|
||||
std::fs::read_dir(dir.path().join("sidecars.lance").join("_computed"))
|
||||
.unwrap()
|
||||
.count()
|
||||
};
|
||||
assert_eq!(sidecars(), 2);
|
||||
|
||||
table
|
||||
.optimize_prune_before(chrono::Utc::now(), Some(true), None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(sidecars(), 1);
|
||||
assert_eq!(
|
||||
table.refresh_column("doubled").await.unwrap().rows_filled,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
/// A deleted row is never computed and the rows that stay keep their
|
||||
/// values: a delete recomputes nothing and stamps nothing.
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user