mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-26 07:58:31 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Will Jones
|
b015dca20f | ||
|
Will Jones
|
2ab80fe087 |
Generated
+22
-22
@@ -3422,7 +3422,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4778,7 +4778,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4853,7 +4853,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4875,7 +4875,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4889,7 +4889,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4899,7 +4899,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4910,7 +4910,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4951,7 +4951,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4982,7 +4982,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5000,7 +5000,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5010,7 +5010,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5046,7 +5046,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5078,7 +5078,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5146,7 +5146,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5169,7 +5169,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5207,7 +5207,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5224,7 +5224,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5237,7 +5237,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5292,7 +5292,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5308,7 +5308,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5348,7 +5348,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5362,7 +5362,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
source = "git+https://github.com/lance-format/lance.git?rev=b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564#b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564"
|
||||
dependencies = [
|
||||
"icu_segmenter",
|
||||
"jieba-rs",
|
||||
|
||||
+17
-14
@@ -13,20 +13,23 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
# TEMPORARY: `ObjectStore::read_dir_stream` is not in a lance release yet, so these point at
|
||||
# lance-format/lance#8120 cherry-picked onto the v10.1.0-beta.1 tag. Put the tag back once that
|
||||
# PR has merged and shipped in a release.
|
||||
lance = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=10.1.0-beta.1", default-features = false, "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=10.1.0-beta.1", "rev" = "b7ef2bae3c7aeb85da637a8c3bc6643ab7d81564", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
|
||||
@@ -8,12 +8,15 @@ use std::fs::create_dir_all;
|
||||
use std::path::Path;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use futures::TryStreamExt;
|
||||
use lance::dataset::refs::Ref;
|
||||
use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
|
||||
use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
|
||||
use lance_datafusion::utils::StreamingWriteSource;
|
||||
use lance_encoding::version::LanceFileVersion;
|
||||
use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
|
||||
use lance_io::object_store::{
|
||||
DirCursor, ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider,
|
||||
};
|
||||
use lance_table::io::commit::commit_handler_from_url;
|
||||
use object_store::local::LocalFileSystem;
|
||||
use snafu::ResultExt;
|
||||
@@ -718,6 +721,54 @@ impl ListingDatabase {
|
||||
self.namespace_database.clone()
|
||||
}
|
||||
|
||||
/// List up to `limit` table names, resuming after the table named `start_after`.
|
||||
///
|
||||
/// The cursor and the page size go into the object store's list request rather than
|
||||
/// being applied to a full listing, so the cost of a page is set by the size of the
|
||||
/// page and not by the size of the database. Stores with no paginated list API fall
|
||||
/// back to a full listing, which is what this did for every store before.
|
||||
///
|
||||
/// Names come back in the order the store lists the directories in, which is by key:
|
||||
/// `foo-bar` precedes `foo`, because the `-` of `foo-bar.lance` sorts below the `.` of
|
||||
/// `foo.lance`. Pagination has to follow the order the cursor is pushed down in, so
|
||||
/// that is the order both listing methods report and the order `start_after` resumes
|
||||
/// in. It matches sorting by name except between a name and one that extends it.
|
||||
async fn list_table_dirs(
|
||||
&self,
|
||||
start_after: Option<&str>,
|
||||
limit: Option<usize>,
|
||||
) -> Result<Vec<String>> {
|
||||
let dir_suffix = format!(".{}", LANCE_EXTENSION);
|
||||
let options = ReadDirOptions {
|
||||
// An empty name means "from the start": that is how comparing names against it
|
||||
// behaved, and how a client looping on a page token spells its first request.
|
||||
// Built into a cursor it would instead sit after every name below `.lance`.
|
||||
resume_from: start_after
|
||||
.filter(|name| !name.is_empty())
|
||||
.map(|name| DirCursor::after_directory(format!("{name}{dir_suffix}"))),
|
||||
page_size: limit,
|
||||
};
|
||||
let mut entries = self
|
||||
.object_store
|
||||
.read_dir_stream(self.base_path.clone(), options);
|
||||
|
||||
let mut names = Vec::new();
|
||||
while limit.is_none_or(|limit| names.len() < limit) {
|
||||
let Some(entry) = entries.try_next().await? else {
|
||||
break;
|
||||
};
|
||||
// A table is the directory `<name>.lance`; anything else under the database
|
||||
// prefix belongs to something other than a table.
|
||||
if !entry.is_dir() {
|
||||
continue;
|
||||
}
|
||||
if let Some(name) = entry.name.strip_suffix(&dir_suffix) {
|
||||
names.push(name.to_string());
|
||||
}
|
||||
}
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
|
||||
let object_store_params = ObjectStoreParams {
|
||||
storage_options_accessor: if self.storage_options.is_empty() {
|
||||
@@ -959,80 +1010,37 @@ impl Database for ListingDatabase {
|
||||
if !request.namespace_path.is_empty() {
|
||||
return self.namespace_database().table_names(request).await;
|
||||
}
|
||||
let mut f = self
|
||||
.object_store
|
||||
.read_dir(self.base_path.clone())
|
||||
.await?
|
||||
.iter()
|
||||
.map(Path::new)
|
||||
.filter(|path| {
|
||||
let is_lance = path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e == LANCE_EXTENSION);
|
||||
is_lance.unwrap_or(false)
|
||||
})
|
||||
.filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
|
||||
.collect::<Vec<String>>();
|
||||
f.sort();
|
||||
if let Some(start_after) = request.start_after {
|
||||
let index = f
|
||||
.iter()
|
||||
.position(|name| name.as_str() > start_after.as_str())
|
||||
.unwrap_or(f.len());
|
||||
f.drain(0..index);
|
||||
}
|
||||
if let Some(limit) = request.limit {
|
||||
f.truncate(limit as usize);
|
||||
}
|
||||
Ok(f)
|
||||
self.list_table_dirs(
|
||||
request.start_after.as_deref(),
|
||||
request.limit.map(|limit| limit as usize),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
|
||||
if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
|
||||
return self.namespace_database().list_tables(request).await;
|
||||
}
|
||||
let mut f = self
|
||||
.object_store
|
||||
.read_dir(self.base_path.clone())
|
||||
.await?
|
||||
.iter()
|
||||
.map(Path::new)
|
||||
.filter(|path| {
|
||||
let is_lance = path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e == LANCE_EXTENSION);
|
||||
is_lance.unwrap_or(false)
|
||||
})
|
||||
.filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
|
||||
.collect::<Vec<String>>();
|
||||
f.sort();
|
||||
let limit = request.limit.map(|limit| limit as usize);
|
||||
// Reading one past the page is how we learn whether another page follows, without
|
||||
// a second request. The extra name is dropped before the response goes out.
|
||||
let mut tables = self
|
||||
.list_table_dirs(
|
||||
request.page_token.as_deref(),
|
||||
limit.map(|limit| limit.saturating_add(1)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Handle pagination with page_token
|
||||
if let Some(ref page_token) = request.page_token {
|
||||
let index = f
|
||||
.iter()
|
||||
.position(|name| name.as_str() > page_token.as_str())
|
||||
.unwrap_or(f.len());
|
||||
f.drain(0..index);
|
||||
}
|
||||
|
||||
// Determine if there's a next page
|
||||
let next_page_token = if let Some(limit) = request.limit {
|
||||
if f.len() > limit as usize {
|
||||
let token = f[limit as usize].clone();
|
||||
f.truncate(limit as usize);
|
||||
Some(token)
|
||||
} else {
|
||||
None
|
||||
let next_page_token = match limit {
|
||||
Some(limit) if tables.len() > limit => {
|
||||
tables.truncate(limit);
|
||||
tables.last().cloned()
|
||||
}
|
||||
} else {
|
||||
None
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Ok(ListTablesResponse {
|
||||
tables: f,
|
||||
tables,
|
||||
page_token: next_page_token,
|
||||
})
|
||||
}
|
||||
@@ -1322,6 +1330,130 @@ mod tests {
|
||||
(tempdir, db)
|
||||
}
|
||||
|
||||
async fn create_tables(db: &ListingDatabase, names: &[&str]) {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
for name in names {
|
||||
db.create_table(CreateTableRequest {
|
||||
name: name.to_string(),
|
||||
namespace_path: vec![],
|
||||
data: Box::new(RecordBatch::new_empty(schema.clone())) as Box<dyn Scannable>,
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// Paging with the returned token has to visit every table exactly once. The token is
|
||||
/// the last name of the page, which is what `page_token` resumes after.
|
||||
#[tokio::test]
|
||||
async fn test_list_tables_pages_over_every_table_once() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
create_tables(&db, &["a", "b", "c", "d", "e"]).await;
|
||||
|
||||
let mut seen = Vec::new();
|
||||
let mut page_token = None;
|
||||
loop {
|
||||
let page = db
|
||||
.list_tables(ListTablesRequest {
|
||||
limit: Some(2),
|
||||
page_token,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
seen.extend(page.tables);
|
||||
match page.page_token {
|
||||
Some(token) => page_token = Some(token),
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(seen, vec!["a", "b", "c", "d", "e"]);
|
||||
}
|
||||
|
||||
/// The last page reports no token, so a caller paging by token knows to stop without
|
||||
/// asking for an empty page.
|
||||
#[tokio::test]
|
||||
async fn test_list_tables_exhausted_page_has_no_token() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
create_tables(&db, &["a", "b"]).await;
|
||||
|
||||
let page = db
|
||||
.list_tables(ListTablesRequest {
|
||||
limit: Some(2),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(page.tables, vec!["a", "b"]);
|
||||
assert_eq!(page.page_token, None);
|
||||
}
|
||||
|
||||
/// Listing follows the order the object store lists directories in, so a name that
|
||||
/// extends another comes first: `-` sorts below the `.` of `.lance`. Pagination pushes
|
||||
/// its cursor into the list request, so it cannot report a different order than the
|
||||
/// one it resumes in.
|
||||
#[tokio::test]
|
||||
async fn test_listing_order_follows_the_store_not_the_table_name() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
create_tables(&db, &["users", "users-archive", "users.old"]).await;
|
||||
|
||||
#[allow(deprecated)]
|
||||
let names = db.table_names(TableNamesRequest::default()).await.unwrap();
|
||||
assert_eq!(names, vec!["users-archive", "users", "users.old"]);
|
||||
|
||||
// Resuming after a name skips everything the store lists before it, which is what
|
||||
// paging by the previous page's last name relies on.
|
||||
#[allow(deprecated)]
|
||||
let after = db
|
||||
.table_names(TableNamesRequest {
|
||||
start_after: Some("users-archive".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(after, vec!["users", "users.old"]);
|
||||
}
|
||||
|
||||
/// An empty `start_after` means "from the start". A name that sorts below `.lance` is
|
||||
/// what disappears if it is treated as a cursor instead.
|
||||
#[tokio::test]
|
||||
async fn test_empty_start_after_lists_from_the_start() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
create_tables(&db, &["-dash", "alpha"]).await;
|
||||
|
||||
#[allow(deprecated)]
|
||||
let names = db
|
||||
.table_names(TableNamesRequest {
|
||||
start_after: Some(String::new()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(names, vec!["-dash", "alpha"]);
|
||||
}
|
||||
|
||||
/// Only directories named `<name>.lance` are tables; loose files and other directories
|
||||
/// under the database prefix are not.
|
||||
#[tokio::test]
|
||||
async fn test_listing_ignores_non_table_children() {
|
||||
let (tempdir, db) = setup_database().await;
|
||||
create_tables(&db, &["real"]).await;
|
||||
std::fs::write(tempdir.path().join("loose.lance"), b"not a table").unwrap();
|
||||
create_dir_all(tempdir.path().join("scratch")).unwrap();
|
||||
|
||||
#[allow(deprecated)]
|
||||
let names = db.table_names(TableNamesRequest::default()).await.unwrap();
|
||||
|
||||
assert_eq!(names, vec!["real"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_listing_database_root_ops_do_not_create_manifest() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user