diff --git a/Cargo.lock b/Cargo.lock index 160ee2253..6dad3d6b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3491,8 +3491,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4863,8 +4863,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arc-swap", "arrow", @@ -4936,8 +4936,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-buffer", @@ -4959,7 +4959,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-buffer", @@ -4973,7 +4973,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-schema", @@ -4982,8 +4982,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrayref", "crunchy", @@ -4993,8 +4993,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-buffer", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-array", @@ -5062,8 +5062,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-array", @@ -5080,8 +5080,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "proc-macro2", "quote", @@ -5090,8 +5090,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-arith", "arrow-array", @@ -5124,8 +5124,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-arith", "arrow-array", @@ -5150,14 +5150,15 @@ dependencies = [ "prost", "prost-build", "prost-types", + "serde", "tokio", "tracing", ] [[package]] name = "lance-index" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arc-swap", "arrow", @@ -5221,8 +5222,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-schema", @@ -5244,8 +5245,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-array", @@ -5285,8 +5286,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-schema", @@ -5300,8 +5301,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "async-trait", @@ -5315,8 +5316,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-ipc", @@ -5369,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-buffer", @@ -5384,8 +5385,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-array", @@ -5425,8 +5426,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-schema", @@ -5439,8 +5440,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 1068d48ee..e6560dbf7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/docs/src/js/interfaces/OptimizeOptions.md b/docs/src/js/interfaces/OptimizeOptions.md index 700632342..110eb3813 100644 --- a/docs/src/js/interfaces/OptimizeOptions.md +++ b/docs/src/js/interfaces/OptimizeOptions.md @@ -26,7 +26,8 @@ const olderThan = new Date(); olderThan.setDate(olderThan.getDate() - 1)); tbl.optimize({cleanupOlderThan: olderThan}); -// Delete all versions except the current version +// Delete versions committed before this point. Versions created by the +// optimize call itself are newer than the cutoff and will be retained. tbl.optimize({cleanupOlderThan: new Date()}); ``` diff --git a/java/pom.xml b/java/pom.xml index 37c182feb..9b1c19f96 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 12.0.0-beta.18 + 13.0.0-beta.1 false 2.30.0 1.7 diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 9162521a8..636289907 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -37,6 +37,7 @@ lzma-sys = { version = "0.1", features = ["static"] } log.workspace = true # Pin to resolve build failures; update periodically for security patches. +# rustls >= 0.23.45 (RUSTSEC-2026-0285) needs aws-lc-rs >= 1.18. aws-lc-sys = "=0.45.0" aws-lc-rs = "=1.18.1" diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index cae01d9d5..852bdec80 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -53,6 +53,7 @@ import { Operator, instanceOfFullTextQuery, } from "../lancedb/query"; +import { LocalTable } from "../lancedb/table"; describe.each([arrow15, arrow16, arrow17, arrow18])( "Given a table", @@ -2765,6 +2766,15 @@ describe("when dealing with tags", () => { }); }); +/** Returns a Date strictly later than every instant observed before the call. */ +async function nextMillisecond(): Promise { + const start = Date.now(); + while (Date.now() <= start) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return new Date(); +} + describe("when optimizing a dataset", () => { let tmpDir: tmp.DirResult; let table: Table; @@ -2787,9 +2797,14 @@ describe("when optimizing a dataset", () => { }); it("cleanups old versions", async () => { - const stats = await table.optimize({ cleanupOlderThan: new Date() }); + // Lance stores version timestamps with nanosecond precision while a JS + // Date only has millisecond precision. A cutoff captured in the same + // millisecond as the last commit would truncate to *before* that commit + // and leave it in place, so wait for the clock to tick over first. + const cutoff = await nextMillisecond(); + const stats = await table.optimize({ cleanupOlderThan: cutoff }); expect(stats.prune.bytesRemoved).toBeGreaterThan(0); - expect(stats.prune.oldVersionsRemoved).toBe(3); + expect(stats.prune.oldVersionsRemoved).toBe(2); }); it("delete unverified", async () => { @@ -2810,6 +2825,24 @@ describe("when optimizing a dataset", () => { }); }); +it("passes cleanupOlderThan to the native binding as an absolute timestamp", async () => { + const optimize = jest.fn().mockResolvedValue({ + compaction: { + filesAdded: 0, + filesRemoved: 0, + fragmentsAdded: 0, + fragmentsRemoved: 0, + }, + prune: { bytesRemoved: 0, oldVersionsRemoved: 0 }, + }); + const table = new LocalTable({ optimize } as never); + const cutoff = new Date("2020-01-02T03:04:05.678Z"); + + await table.optimize({ cleanupOlderThan: cutoff, deleteUnverified: true }); + + expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true); +}); + describe.each([arrow15, arrow16, arrow17, arrow18])( "when optimizing a dataset", // biome-ignore lint/suspicious/noExplicitAny: diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 8d8b0d675..a70243402 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -148,7 +148,8 @@ export interface OptimizeOptions { * olderThan.setDate(olderThan.getDate() - 1)); * tbl.optimize({cleanupOlderThan: olderThan}); * - * // Delete all versions except the current version + * // Delete versions committed before this point. Versions created by the + * // optimize call itself are newer than the cutoff and will be retained. * tbl.optimize({cleanupOlderThan: new Date()}); */ cleanupOlderThan: Date; @@ -1486,16 +1487,8 @@ export class LocalTable extends Table { } async optimize(options?: Partial): Promise { - let cleanupOlderThanMs; - if ( - options?.cleanupOlderThan !== undefined && - options?.cleanupOlderThan !== null - ) { - cleanupOlderThanMs = - new Date().getTime() - options.cleanupOlderThan.getTime(); - } return await this.inner.optimize( - cleanupOlderThanMs, + options?.cleanupOlderThan?.getTime(), options?.deleteUnverified, ); } diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 7ae4402ab..cf7ec4020 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema}; use lancedb::table::{ - AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration, + AddDataMode, ColumnAlteration as LanceColumnAlteration, FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken, NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable, }; @@ -677,22 +677,20 @@ impl Table { #[napi(catch_unwind)] pub async fn optimize( &self, - older_than_ms: Option, + before_timestamp_ms: Option, delete_unverified: Option, ) -> napi::Result { let inner = self.inner_ref()?; - let older_than = if let Some(ms) = older_than_ms { - if ms == i64::MIN { - return Err(napi::Error::from_reason(format!( - "older_than_ms can not be {}", - i32::MIN, - ))); - } - Duration::try_milliseconds(ms) - } else { - None - }; + let before_timestamp = before_timestamp_ms + .map(|ms| { + DateTime::from_timestamp_millis(ms).ok_or_else(|| { + napi::Error::from_reason(format!( + "cleanupOlderThan timestamp is out of range: {ms}" + )) + }) + }) + .transpose()?; let compaction_stats = inner .optimize(OptimizeAction::Compact { @@ -703,16 +701,22 @@ impl Table { .default_error()? .compaction .unwrap(); - let prune_stats = inner - .optimize(OptimizeAction::Prune { - older_than, - delete_unverified, - error_if_tagged_old_versions: None, - }) - .await - .default_error()? - .prune - .unwrap(); + let prune_stats = if let Some(before_timestamp) = before_timestamp { + inner + .optimize_prune_before(before_timestamp, delete_unverified, None) + .await + } else { + inner + .optimize(OptimizeAction::Prune { + older_than: None, + delete_unverified, + error_if_tagged_old_versions: None, + }) + .await + } + .default_error()? + .prune + .unwrap(); inner .optimize(lancedb::table::OptimizeAction::Index( OptimizeOptions::default(), diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 0bca3327f..1557cbfa8 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -113,6 +113,10 @@ aws-sdk-s3 = { version = "1.55.0" } aws-sdk-kms = { version = "1.48.0" } aws-config = { version = "1.5.10" } aws-smithy-runtime = { version = "1.9.1" } +# Constraint only: types 1.7 breaks aws-smithy-json 0.63, which aws-config still +# requires. Bounds must stay inside 1.x and allow the MSRV job's 1.3.6 pin. +# Drop once aws-config moves to aws-smithy-json 0.64. +aws-smithy-types = { version = ">=1.0, <1.7" } datafusion.workspace = true http-body = "1" # Matching reqwest rstest = "0.23.0" diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 508c2ca49..5732955f5 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1741,6 +1741,33 @@ impl Table { self.inner.optimize(action).await } + /// Prune versions committed before an absolute timestamp. + /// + /// This is an internal entry point for language bindings whose public API + /// accepts an absolute cleanup cutoff. + #[doc(hidden)] + pub async fn optimize_prune_before( + &self, + before_timestamp: chrono::DateTime, + delete_unverified: Option, + error_if_tagged_old_versions: Option, + ) -> Result { + let native = self.as_native().ok_or_else(|| Error::NotSupported { + message: "optimize is not supported on LanceDB cloud.".into(), + })?; + let prune = optimize::cleanup_old_versions_before( + native, + before_timestamp, + delete_unverified, + error_if_tagged_old_versions, + ) + .await?; + Ok(OptimizeStats { + compaction: None, + prune: Some(prune), + }) + } + /// Add new columns to the table, providing values to fill in. pub fn add_columns(&self) -> AddColumnsBuilder { AddColumnsBuilder::new(self.inner.clone()) diff --git a/rust/lancedb/src/table/optimize.rs b/rust/lancedb/src/table/optimize.rs index 6e3fc0048..2a502b059 100644 --- a/rust/lancedb/src/table/optimize.rs +++ b/rust/lancedb/src/table/optimize.rs @@ -8,7 +8,8 @@ use std::sync::Arc; -use lance::dataset::cleanup::RemovalStats; +use chrono::{DateTime, Utc}; +use lance::dataset::cleanup::{CleanupPolicyBuilder, RemovalStats}; use lance::dataset::optimize::{CompactionMetrics, IndexRemapperOptions, compact_files}; use lance::index::DatasetIndexExt; use lance_index::optimize::OptimizeOptions; @@ -147,6 +148,33 @@ pub(crate) async fn cleanup_old_versions( Ok(stats) } +/// Remove dataset versions committed before an absolute timestamp. +pub(crate) async fn cleanup_old_versions_before( + table: &NativeTable, + before_timestamp: DateTime, + delete_unverified: Option, + error_if_tagged_old_versions: Option, +) -> Result { + table.dataset.ensure_mutable()?; + let dataset = table.dataset.get().await?; + let mut policy = CleanupPolicyBuilder::default().before_timestamp(before_timestamp); + if let Some(delete_unverified) = delete_unverified { + policy = policy.delete_unverified(delete_unverified); + } + if let Some(error_if_tagged_old_versions) = error_if_tagged_old_versions { + policy = policy.error_if_tagged_old_versions(error_if_tagged_old_versions); + } + let stats = dataset.cleanup_with_policy(policy.build()).await?; + // Computed-column signature sidecars live outside lance's directories; + // drop the ones the surviving versions no longer reference. + let removed = + super::freshness::prune_sidecars(&dataset, delete_unverified.unwrap_or(false)).await?; + if removed > 0 { + log::debug!("removed {removed} unreferenced computed-column signature sidecars"); + } + Ok(stats) +} + /// Compact files in the dataset. /// /// This can be run after making several small appends to optimize the table diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 2ed479256..996f2e383 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -978,6 +978,43 @@ mod tests { ); } + /// Absolute timestamp pruning applies the same computed-column sidecar + /// cleanup as duration-based pruning. + #[tokio::test] + async fn test_absolute_pruning_drops_the_sidecars_of_pruned_versions() { + let dir = tempfile::tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, [1, 2])).unwrap(); + let table = conn + .create_table("sidecars", batch) + .execute() + .await + .unwrap(); + declare_doubled(&table).await.unwrap(); + table.refresh_column("doubled").await.unwrap(); + append(&table, vec![5]).await; + table.refresh_column("doubled").await.unwrap(); + let sidecars = || { + std::fs::read_dir(dir.path().join("sidecars.lance").join("_computed")) + .unwrap() + .count() + }; + assert_eq!(sidecars(), 2); + + table + .optimize_prune_before(chrono::Utc::now(), Some(true), None) + .await + .unwrap(); + assert_eq!(sidecars(), 1); + assert_eq!( + table.refresh_column("doubled").await.unwrap().rows_filled, + 0 + ); + } + /// A deleted row is never computed and the rows that stay keep their /// values: a delete recomputes nothing and stamps nothing. #[tokio::test]