mirror of
https://github.com/lancedb/lancedb.git
synced 2026-06-19 04:00:47 +00:00
Compare commits
2 Commits
python-v0.
...
will/index
| Author | SHA1 | Date | |
|---|---|---|---|
|
92e4e57f61 | ||
|
3347c3afcf |
80
Cargo.lock
generated
80
Cargo.lock
generated
@@ -3473,8 +3473,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.4",
|
||||
@@ -4776,8 +4776,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4852,8 +4852,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4874,7 +4874,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4888,7 +4888,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4897,8 +4897,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"paste",
|
||||
@@ -4907,8 +4907,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4923,6 +4923,7 @@ dependencies = [
|
||||
"itertools 0.13.0",
|
||||
"lance-arrow",
|
||||
"libc",
|
||||
"libm",
|
||||
"log",
|
||||
"moka",
|
||||
"num_cpus",
|
||||
@@ -4938,13 +4939,14 @@ dependencies = [
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"twox-hash",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4974,8 +4976,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4993,8 +4995,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5029,8 +5031,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5061,8 +5063,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5103,7 +5105,6 @@ dependencies = [
|
||||
"lance-select",
|
||||
"lance-table",
|
||||
"lance-tokenizer",
|
||||
"libm",
|
||||
"libsais-rs",
|
||||
"log",
|
||||
"ndarray",
|
||||
@@ -5123,14 +5124,13 @@ dependencies = [
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"twox-hash",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -5172,8 +5172,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5189,8 +5189,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5202,8 +5202,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5252,8 +5252,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5269,8 +5269,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5310,8 +5310,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5324,8 +5324,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "8.0.0-beta.6"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.6#d62489375b041015115b15bf0e3d5c6c63e8e246"
|
||||
version = "8.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v8.0.0-beta.7#a266f2bc23584b37b8fc03004502d2f089df0fe6"
|
||||
dependencies = [
|
||||
"icu_segmenter",
|
||||
"jieba-rs",
|
||||
|
||||
28
Cargo.toml
28
Cargo.toml
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=8.0.0-beta.6", default-features = false, "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=8.0.0-beta.6", default-features = false, "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=8.0.0-beta.6", default-features = false, "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=8.0.0-beta.6", "tag" = "v8.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=8.0.0-beta.7", default-features = false, "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=8.0.0-beta.7", default-features = false, "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=8.0.0-beta.7", default-features = false, "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=8.0.0-beta.7", "tag" = "v8.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>8.0.0-beta.6</lance-core.version>
|
||||
<lance-core.version>8.0.0-beta.7</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use scalar::FtsIndexBuilder;
|
||||
use serde::Deserialize;
|
||||
use serde_with::skip_serializing_none;
|
||||
@@ -364,6 +365,51 @@ pub struct IndexConfig {
|
||||
/// Currently this is always a Vec of size 1. In the future there may
|
||||
/// be more columns to represent composite indices.
|
||||
pub columns: Vec<String>,
|
||||
/// The UUID of the first segment of the index.
|
||||
///
|
||||
/// An index may be made up of multiple segments, each with their own UUID.
|
||||
/// This is the UUID of the first segment. `None` if it could not be
|
||||
/// determined (e.g. for remote tables, which do not yet surface this).
|
||||
pub index_uuid: Option<String>,
|
||||
/// The protobuf type URL, a precise type identifier for the index.
|
||||
///
|
||||
/// `None` if unavailable (e.g. for remote tables).
|
||||
pub type_url: Option<String>,
|
||||
/// When the index was created, taken as the minimum creation time across
|
||||
/// all segments.
|
||||
///
|
||||
/// `None` if unavailable, such as for indices created before creation
|
||||
/// timestamps were tracked, or for remote tables.
|
||||
pub created_at: Option<DateTime<Utc>>,
|
||||
/// The number of rows indexed, across all segments.
|
||||
///
|
||||
/// This is approximate and may include rows that have since been deleted.
|
||||
/// `None` if unavailable (e.g. for remote tables).
|
||||
pub num_indexed_rows: Option<u64>,
|
||||
/// The number of rows in the table that are not yet covered by this index.
|
||||
///
|
||||
/// Computed as the table's total row count minus [`Self::num_indexed_rows`].
|
||||
/// Optimizing the index will fold these rows into it. `None` if unavailable
|
||||
/// (e.g. for remote tables).
|
||||
pub num_unindexed_rows: Option<u64>,
|
||||
/// The total size in bytes of all index files across all segments.
|
||||
///
|
||||
/// `None` if size information is unavailable, such as for indices created
|
||||
/// before file sizes were tracked, or for remote tables.
|
||||
pub size_bytes: Option<u64>,
|
||||
/// The number of segments that make up the index.
|
||||
///
|
||||
/// `None` if unavailable (e.g. for remote tables).
|
||||
pub num_segments: Option<u32>,
|
||||
/// The on-disk index format version, taken from the first segment.
|
||||
///
|
||||
/// `None` if unavailable (e.g. for remote tables).
|
||||
pub index_version: Option<i32>,
|
||||
/// Index-type-specific details, serialized as JSON.
|
||||
///
|
||||
/// The shape of this JSON varies by index type. `None` if the details
|
||||
/// could not be produced (e.g. no plugin available) or for remote tables.
|
||||
pub index_details: Option<String>,
|
||||
}
|
||||
|
||||
#[skip_serializing_none]
|
||||
|
||||
@@ -2119,6 +2119,17 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
name: index.index_name,
|
||||
index_type: stats.index_type,
|
||||
columns,
|
||||
// These are left None until the server response wires
|
||||
// them through. See https://github.com/lancedb/lancedb/issues/3494
|
||||
index_uuid: None,
|
||||
type_url: None,
|
||||
created_at: None,
|
||||
num_indexed_rows: None,
|
||||
num_unindexed_rows: None,
|
||||
size_bytes: None,
|
||||
num_segments: None,
|
||||
index_version: None,
|
||||
index_details: None,
|
||||
})),
|
||||
Ok(None) => Ok(None), // The index must have been deleted since we listed it.
|
||||
Err(e) => Err(e),
|
||||
@@ -4037,16 +4048,184 @@ mod tests {
|
||||
name: "vector_idx".into(),
|
||||
index_type: IndexType::IvfPq,
|
||||
columns: vec!["vector".into()],
|
||||
index_uuid: None,
|
||||
type_url: None,
|
||||
created_at: None,
|
||||
num_indexed_rows: None,
|
||||
num_unindexed_rows: None,
|
||||
size_bytes: None,
|
||||
num_segments: None,
|
||||
index_version: None,
|
||||
index_details: None,
|
||||
},
|
||||
IndexConfig {
|
||||
name: "my_idx".into(),
|
||||
index_type: IndexType::LabelList,
|
||||
columns: vec!["metadata.`my.column`".into()],
|
||||
index_uuid: None,
|
||||
type_url: None,
|
||||
created_at: None,
|
||||
num_indexed_rows: None,
|
||||
num_unindexed_rows: None,
|
||||
size_bytes: None,
|
||||
num_segments: None,
|
||||
index_version: None,
|
||||
index_details: None,
|
||||
},
|
||||
];
|
||||
assert_eq!(indices, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_list_indices_nested_field_paths() {
|
||||
let schema = nested_index_schema();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
assert_eq!(request.method(), "POST");
|
||||
|
||||
let response_body = match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => {
|
||||
return http::Response::builder()
|
||||
.status(200)
|
||||
.body(describe_response(&schema))
|
||||
.unwrap();
|
||||
}
|
||||
"/v1/table/my_table/index/list/" => {
|
||||
serde_json::json!({
|
||||
"indexes": [
|
||||
{
|
||||
"index_name": "row_id_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000001",
|
||||
"columns": ["rowId"],
|
||||
"index_status": "done",
|
||||
},
|
||||
{
|
||||
"index_name": "row_dash_id_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000002",
|
||||
"columns": ["`ROW-ID`"],
|
||||
"index_status": "done",
|
||||
},
|
||||
{
|
||||
"index_name": "user_id_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000003",
|
||||
"columns": ["userId"],
|
||||
"index_status": "done",
|
||||
},
|
||||
{
|
||||
"index_name": "mixed_case_metadata_user_id_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000004",
|
||||
"columns": ["MetaData.userId"],
|
||||
"index_status": "done",
|
||||
},
|
||||
{
|
||||
"index_name": "metadata_user_id_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000005",
|
||||
"columns": ["Metadata.USER_ID"],
|
||||
"index_status": "done",
|
||||
},
|
||||
{
|
||||
"index_name": "image_embedding_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000006",
|
||||
"columns": ["Image.Embedding"],
|
||||
"index_status": "done",
|
||||
},
|
||||
{
|
||||
"index_name": "payload_text_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000007",
|
||||
"columns": ["Payload.Text"],
|
||||
"index_status": "done",
|
||||
},
|
||||
{
|
||||
"index_name": "meta_data_user_id_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000008",
|
||||
"columns": ["`META-DATA`.`USER-ID`"],
|
||||
"index_status": "done",
|
||||
},
|
||||
{
|
||||
"index_name": "literal_dot_idx",
|
||||
"index_uuid": "00000000-0000-0000-0000-000000000009",
|
||||
"columns": ["literal.`A.B`"],
|
||||
"index_status": "done",
|
||||
},
|
||||
]
|
||||
})
|
||||
}
|
||||
"/v1/table/my_table/index/row_id_idx/stats/"
|
||||
| "/v1/table/my_table/index/row_dash_id_idx/stats/"
|
||||
| "/v1/table/my_table/index/user_id_idx/stats/"
|
||||
| "/v1/table/my_table/index/mixed_case_metadata_user_id_idx/stats/"
|
||||
| "/v1/table/my_table/index/metadata_user_id_idx/stats/"
|
||||
| "/v1/table/my_table/index/meta_data_user_id_idx/stats/"
|
||||
| "/v1/table/my_table/index/literal_dot_idx/stats/" => {
|
||||
serde_json::json!({
|
||||
"num_indexed_rows": 100000,
|
||||
"num_unindexed_rows": 0,
|
||||
"index_type": "BTREE"
|
||||
})
|
||||
}
|
||||
"/v1/table/my_table/index/image_embedding_idx/stats/" => {
|
||||
serde_json::json!({
|
||||
"num_indexed_rows": 100000,
|
||||
"num_unindexed_rows": 0,
|
||||
"index_type": "IVF_PQ",
|
||||
"distance_type": "l2"
|
||||
})
|
||||
}
|
||||
"/v1/table/my_table/index/payload_text_idx/stats/" => {
|
||||
serde_json::json!({
|
||||
"num_indexed_rows": 100000,
|
||||
"num_unindexed_rows": 0,
|
||||
"index_type": "FTS"
|
||||
})
|
||||
}
|
||||
path => panic!("Unexpected path: {}", path),
|
||||
};
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(serde_json::to_string(&response_body).unwrap())
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
let indices = table.list_indices().await.unwrap();
|
||||
// The remote path leaves the rich metadata fields None until the server
|
||||
// wires them through. See https://github.com/lancedb/lancedb/issues/3494
|
||||
let expected: Vec<IndexConfig> = [
|
||||
("row_id_idx", IndexType::BTree, "rowId"),
|
||||
("row_dash_id_idx", IndexType::BTree, "`row-id`"),
|
||||
("user_id_idx", IndexType::BTree, "userId"),
|
||||
(
|
||||
"mixed_case_metadata_user_id_idx",
|
||||
IndexType::BTree,
|
||||
"MetaData.userId",
|
||||
),
|
||||
("metadata_user_id_idx", IndexType::BTree, "metadata.user_id"),
|
||||
("image_embedding_idx", IndexType::IvfPq, "image.embedding"),
|
||||
("payload_text_idx", IndexType::FTS, "payload.text"),
|
||||
(
|
||||
"meta_data_user_id_idx",
|
||||
IndexType::BTree,
|
||||
"`meta-data`.`user-id`",
|
||||
),
|
||||
("literal_dot_idx", IndexType::BTree, "literal.`a.b`"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(name, index_type, column)| IndexConfig {
|
||||
name: name.into(),
|
||||
index_type,
|
||||
columns: vec![column.into()],
|
||||
index_uuid: None,
|
||||
type_url: None,
|
||||
created_at: None,
|
||||
num_indexed_rows: None,
|
||||
num_unindexed_rows: None,
|
||||
size_bytes: None,
|
||||
num_segments: None,
|
||||
index_version: None,
|
||||
index_details: None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(indices, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_list_versions() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
|
||||
@@ -2924,6 +2924,7 @@ impl BaseTable for NativeTable {
|
||||
|
||||
async fn list_indices(&self) -> Result<Vec<IndexConfig>> {
|
||||
let dataset = self.dataset.get().await?;
|
||||
let total_rows = dataset.count_rows(None).await? as u64;
|
||||
let indices = dataset
|
||||
.describe_indices(None)
|
||||
.await?
|
||||
@@ -2959,10 +2960,25 @@ impl BaseTable for NativeTable {
|
||||
columns.push(field_path);
|
||||
}
|
||||
|
||||
let segments = idx_desc.segments();
|
||||
let index_uuid = segments.first().map(|seg| seg.uuid.to_string());
|
||||
let created_at = segments.iter().filter_map(|seg| seg.created_at).min();
|
||||
let index_version = segments.first().map(|seg| seg.index_version);
|
||||
let num_indexed_rows = idx_desc.rows_indexed();
|
||||
|
||||
Some(IndexConfig {
|
||||
name: idx_desc.name().to_string(),
|
||||
index_type,
|
||||
columns,
|
||||
index_uuid,
|
||||
type_url: Some(idx_desc.type_url().to_string()),
|
||||
created_at,
|
||||
num_indexed_rows: Some(num_indexed_rows),
|
||||
num_unindexed_rows: Some(total_rows.saturating_sub(num_indexed_rows)),
|
||||
size_bytes: idx_desc.total_size_bytes(),
|
||||
num_segments: Some(segments.len() as u32),
|
||||
index_version,
|
||||
index_details: idx_desc.details().ok(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
@@ -3417,6 +3433,15 @@ mod tests {
|
||||
let index = index_configs.into_iter().next().unwrap();
|
||||
assert_eq!(index.index_type, crate::index::IndexType::IvfPq);
|
||||
assert_eq!(index.columns, vec!["embeddings".to_string()]);
|
||||
assert!(index.index_uuid.is_some());
|
||||
assert!(index.type_url.is_some());
|
||||
assert_eq!(index.num_segments, Some(1));
|
||||
assert_eq!(index.num_indexed_rows, Some(512));
|
||||
assert_eq!(index.num_unindexed_rows, Some(0));
|
||||
assert!(index.created_at.is_some());
|
||||
assert!(index.size_bytes.is_some());
|
||||
assert!(index.index_version.is_some());
|
||||
assert!(index.index_details.is_some());
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 512);
|
||||
assert_eq!(table.name(), "test");
|
||||
|
||||
@@ -3766,6 +3791,17 @@ mod tests {
|
||||
assert_eq!(index.index_type, crate::index::IndexType::BTree);
|
||||
assert_eq!(index.columns, vec!["i".to_string()]);
|
||||
|
||||
// The richer metadata surfaced from describe_indices should be populated.
|
||||
assert!(index.index_uuid.is_some());
|
||||
assert!(index.type_url.is_some());
|
||||
assert_eq!(index.num_segments, Some(1));
|
||||
assert_eq!(index.num_indexed_rows, Some(1));
|
||||
assert_eq!(index.num_unindexed_rows, Some(0));
|
||||
assert!(index.created_at.is_some());
|
||||
assert!(index.size_bytes.is_some());
|
||||
assert!(index.index_version.is_some());
|
||||
assert!(index.index_details.is_some());
|
||||
|
||||
let indices = table.as_native().unwrap().load_indices().await.unwrap();
|
||||
let index_name = &indices[0].index_name;
|
||||
let stats = table.index_stats(index_name).await.unwrap().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user