mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-27 08:28:28 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 184ab02795 | |||
| 36e1e2a909 | |||
| a1351a3c4e | |||
| 64978c8419 | |||
| 241420239b | |||
| 9d67ea2bb0 |
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.28.0-beta.11"
|
current_version = "0.29.0"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
paths:
|
paths:
|
||||||
- java/**
|
- java/**
|
||||||
- .github/workflows/java.yml
|
- .github/workflows/java.yml
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- rust/**
|
- rust/**
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- Cargo.toml
|
- Cargo.toml
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- Cargo.toml
|
- Cargo.toml
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- Cargo.toml
|
- Cargo.toml
|
||||||
|
|||||||
Generated
+208
-1223
File diff suppressed because it is too large
Load Diff
+15
-15
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
|||||||
rust-version = "1.91.0"
|
rust-version = "1.91.0"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
lance = { "version" = "=7.0.0-beta.7", default-features = false, "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance = { "version" = "=6.0.1", default-features = false }
|
||||||
lance-core = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-core = "=6.0.1"
|
||||||
lance-datagen = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datagen = "=6.0.1"
|
||||||
lance-file = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-file = "=6.0.1"
|
||||||
lance-io = { "version" = "=7.0.0-beta.7", default-features = false, "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-io = { "version" = "=6.0.1", default-features = false }
|
||||||
lance-index = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-index = "=6.0.1"
|
||||||
lance-linalg = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-linalg = "=6.0.1"
|
||||||
lance-namespace = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace = "=6.0.1"
|
||||||
lance-namespace-impls = { "version" = "=7.0.0-beta.7", default-features = false, "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace-impls = { "version" = "=6.0.1", default-features = false }
|
||||||
lance-table = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-table = "=6.0.1"
|
||||||
lance-testing = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-testing = "=6.0.1"
|
||||||
lance-datafusion = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datafusion = "=6.0.1"
|
||||||
lance-encoding = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-encoding = "=6.0.1"
|
||||||
lance-arrow = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-arrow = "=6.0.1"
|
||||||
ahash = "0.8"
|
ahash = "0.8"
|
||||||
# Note that this one does not include pyarrow
|
# Note that this one does not include pyarrow
|
||||||
arrow = { version = "58.0.0", optional = false }
|
arrow = { version = "58.0.0", optional = false }
|
||||||
@@ -54,7 +54,7 @@ half = { "version" = "2.7.1", default-features = false, features = [
|
|||||||
futures = "0"
|
futures = "0"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
moka = { version = "0.12", features = ["future"] }
|
moka = { version = "0.12", features = ["future"] }
|
||||||
object_store = "0.13.2"
|
object_store = "0.12.0"
|
||||||
pin-project = "1.0.7"
|
pin-project = "1.0.7"
|
||||||
rand = "0.9"
|
rand = "0.9"
|
||||||
snafu = "0.8"
|
snafu = "0.8"
|
||||||
|
|||||||
@@ -80,6 +80,13 @@ ignore = [
|
|||||||
# https://rustsec.org/advisories/RUSTSEC-2025-0119
|
# https://rustsec.org/advisories/RUSTSEC-2025-0119
|
||||||
{ id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
|
{ id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
|
||||||
|
|
||||||
|
# rustls-pemfile: unmaintained (repo archived; code folded into
|
||||||
|
# rustls-pki-types). Reached only transitively via object_store → lance.
|
||||||
|
# No safe upgrade is available; clearing this requires object_store to
|
||||||
|
# migrate to the rustls-pki-types PemObject API.
|
||||||
|
# https://rustsec.org/advisories/RUSTSEC-2025-0134
|
||||||
|
{ id = "RUSTSEC-2025-0134", reason = "transitive via object_store/lance; waiting on object_store migration to rustls-pki-types" },
|
||||||
|
|
||||||
# bincode: unmaintained. Reached through lindera and lindera-dictionary,
|
# bincode: unmaintained. Reached through lindera and lindera-dictionary,
|
||||||
# which are required by the native Lindera tokenizer path. Lindera has not
|
# which are required by the native Lindera tokenizer path. Lindera has not
|
||||||
# migrated to another serialization format yet.
|
# migrated to another serialization format yet.
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-core</artifactId>
|
<artifactId>lancedb-core</artifactId>
|
||||||
<version>0.28.0-beta.11</version>
|
<version>0.29.0</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.28.0-beta.11</version>
|
<version>0.29.0-final.0</version>
|
||||||
<relativePath>../pom.xml</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.28.0-beta.11</version>
|
<version>0.29.0-final.0</version>
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
<name>${project.artifactId}</name>
|
<name>${project.artifactId}</name>
|
||||||
<description>LanceDB Java SDK Parent POM</description>
|
<description>LanceDB Java SDK Parent POM</description>
|
||||||
@@ -28,7 +28,7 @@
|
|||||||
<properties>
|
<properties>
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
<arrow.version>15.0.0</arrow.version>
|
<arrow.version>15.0.0</arrow.version>
|
||||||
<lance-core.version>7.0.0-beta.7</lance-core.version>
|
<lance-core.version>6.0.0</lance-core.version>
|
||||||
<spotless.skip>false</spotless.skip>
|
<spotless.skip>false</spotless.skip>
|
||||||
<spotless.version>2.30.0</spotless.version>
|
<spotless.version>2.30.0</spotless.version>
|
||||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||||
|
|||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
version = "0.28.0-beta.11"
|
version = "0.29.0"
|
||||||
publish = false
|
publish = false
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
description.workspace = true
|
description.workspace = true
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-arm64",
|
"name": "@lancedb/lancedb-darwin-arm64",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.darwin-arm64.node",
|
"main": "lancedb.darwin-arm64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-gnu.node",
|
"main": "lancedb.linux-arm64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-musl.node",
|
"main": "lancedb.linux-arm64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-gnu.node",
|
"main": "lancedb.linux-x64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-musl.node",
|
"main": "lancedb.linux-x64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"os": [
|
"os": [
|
||||||
"win32"
|
"win32"
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"os": ["win32"],
|
"os": ["win32"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.win32-x64-msvc.node",
|
"main": "lancedb.win32-x64-msvc.node",
|
||||||
|
|||||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64",
|
"x64",
|
||||||
"arm64"
|
"arm64"
|
||||||
|
|||||||
+1
-1
@@ -11,7 +11,7 @@
|
|||||||
"ann"
|
"ann"
|
||||||
],
|
],
|
||||||
"private": false,
|
"private": false,
|
||||||
"version": "0.28.0-beta.11",
|
"version": "0.29.0",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./dist/index.js",
|
".": "./dist/index.js",
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.31.0-beta.11"
|
current_version = "0.32.0"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.31.0-beta.11"
|
version = "0.32.0"
|
||||||
publish = false
|
publish = false
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "Python bindings for LanceDB"
|
description = "Python bindings for LanceDB"
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ repository = "https://github.com/lancedb/lancedb"
|
|||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
pylance = [
|
pylance = [
|
||||||
"pylance>=5.0.0b5",
|
"pylance>=6.0.0",
|
||||||
]
|
]
|
||||||
tests = [
|
tests = [
|
||||||
"aiohttp>=3.9.0",
|
"aiohttp>=3.9.0",
|
||||||
@@ -58,7 +58,7 @@ tests = [
|
|||||||
"pytz>=2023.3",
|
"pytz>=2023.3",
|
||||||
"polars>=0.19, <=1.3.0",
|
"polars>=0.19, <=1.3.0",
|
||||||
"pyarrow-stubs>=16.0",
|
"pyarrow-stubs>=16.0",
|
||||||
"pylance>=5.0.0b5",
|
"pylance>=6.0.0",
|
||||||
"requests>=2.31.0",
|
"requests>=2.31.0",
|
||||||
"datafusion>=52,<53",
|
"datafusion>=52,<53",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.28.0-beta.11"
|
version = "0.29.0"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
@@ -108,12 +108,7 @@ test-log = "0.2"
|
|||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
aws = [
|
aws = ["lance/aws", "lance-io/aws", "lance-namespace-impls/dir-aws"]
|
||||||
"lance/aws",
|
|
||||||
"lance-io/aws",
|
|
||||||
"lance-namespace-impls/dir-aws",
|
|
||||||
"object_store/aws",
|
|
||||||
]
|
|
||||||
oss = ["lance/oss", "lance-io/oss", "lance-namespace-impls/dir-oss"]
|
oss = ["lance/oss", "lance-io/oss", "lance-namespace-impls/dir-oss"]
|
||||||
gcs = ["lance/gcp", "lance-io/gcp", "lance-namespace-impls/dir-gcp"]
|
gcs = ["lance/gcp", "lance-io/gcp", "lance-namespace-impls/dir-gcp"]
|
||||||
azure = [
|
azure = [
|
||||||
|
|||||||
@@ -722,7 +722,7 @@ impl ListingDatabase {
|
|||||||
let commit_handler = commit_handler_from_url(&uri, &Some(object_store_params)).await?;
|
let commit_handler = commit_handler_from_url(&uri, &Some(object_store_params)).await?;
|
||||||
for name in names {
|
for name in names {
|
||||||
let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
|
let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
|
||||||
let full_path = self.base_path.clone().join(dir_name.clone());
|
let full_path = self.base_path.child(dir_name.clone());
|
||||||
|
|
||||||
commit_handler.delete(&full_path).await?;
|
commit_handler.delete(&full_path).await?;
|
||||||
|
|
||||||
|
|||||||
@@ -5,12 +5,11 @@
|
|||||||
|
|
||||||
use std::{fmt::Formatter, sync::Arc};
|
use std::{fmt::Formatter, sync::Arc};
|
||||||
|
|
||||||
use futures::{StreamExt, TryFutureExt, stream::BoxStream};
|
use futures::{TryFutureExt, stream::BoxStream};
|
||||||
use lance::io::WrappingObjectStore;
|
use lance::io::WrappingObjectStore;
|
||||||
use object_store::{
|
use object_store::{
|
||||||
CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
|
Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||||
ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
|
PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart, path::Path,
|
||||||
UploadPart, path::Path,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
@@ -94,6 +93,20 @@ impl ObjectStore for MirroringObjectStore {
|
|||||||
self.primary.get_opts(location, options).await
|
self.primary.get_opts(location, options).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn head(&self, location: &Path) -> Result<ObjectMeta> {
|
||||||
|
self.primary.head(location).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete(&self, location: &Path) -> Result<()> {
|
||||||
|
if !location.primary_only() {
|
||||||
|
match self.secondary.delete(location).await {
|
||||||
|
Err(Error::NotFound { .. }) | Ok(_) => {}
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.primary.delete(location).await
|
||||||
|
}
|
||||||
|
|
||||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||||
self.primary.list(prefix)
|
self.primary.list(prefix)
|
||||||
}
|
}
|
||||||
@@ -102,41 +115,22 @@ impl ObjectStore for MirroringObjectStore {
|
|||||||
self.primary.list_with_delimiter(prefix).await
|
self.primary.list_with_delimiter(prefix).await
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete_stream(
|
async fn copy(&self, from: &Path, to: &Path) -> Result<()> {
|
||||||
&self,
|
|
||||||
locations: BoxStream<'static, Result<Path>>,
|
|
||||||
) -> BoxStream<'static, Result<Path>> {
|
|
||||||
let primary = self.primary.clone();
|
|
||||||
let secondary = self.secondary.clone();
|
|
||||||
locations
|
|
||||||
.map(move |location| {
|
|
||||||
let primary = primary.clone();
|
|
||||||
let secondary = secondary.clone();
|
|
||||||
async move {
|
|
||||||
let location = location?;
|
|
||||||
if !location.primary_only() {
|
|
||||||
match secondary.delete(&location).await {
|
|
||||||
Err(Error::NotFound { .. }) | Ok(_) => {}
|
|
||||||
Err(e) => return Err(e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
primary.delete(&location).await?;
|
|
||||||
Ok(location)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.buffered(10)
|
|
||||||
.boxed()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
|
||||||
if to.primary_only() {
|
if to.primary_only() {
|
||||||
self.primary.copy_opts(from, to, options).await
|
self.primary.copy(from, to).await
|
||||||
} else {
|
} else {
|
||||||
self.secondary.copy_opts(from, to, options.clone()).await?;
|
self.secondary.copy(from, to).await?;
|
||||||
self.primary.copy_opts(from, to, options).await?;
|
self.primary.copy(from, to).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> {
|
||||||
|
if !to.primary_only() {
|
||||||
|
self.secondary.copy(from, to).await?;
|
||||||
|
}
|
||||||
|
self.primary.copy_if_not_exists(from, to).await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ use bytes::Bytes;
|
|||||||
use futures::stream::BoxStream;
|
use futures::stream::BoxStream;
|
||||||
use lance::io::WrappingObjectStore;
|
use lance::io::WrappingObjectStore;
|
||||||
use object_store::{
|
use object_store::{
|
||||||
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||||
PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
|
PutMultipartOptions, PutOptions, PutPayload, PutResult, Result as OSResult, UploadPart,
|
||||||
UploadPart, path::Path,
|
path::Path,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
@@ -81,6 +81,11 @@ impl IoTrackingStore {
|
|||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
#[deny(clippy::missing_trait_methods)]
|
#[deny(clippy::missing_trait_methods)]
|
||||||
impl ObjectStore for IoTrackingStore {
|
impl ObjectStore for IoTrackingStore {
|
||||||
|
async fn put(&self, location: &Path, bytes: PutPayload) -> OSResult<PutResult> {
|
||||||
|
self.record_write(bytes.content_length() as u64);
|
||||||
|
self.target.put(location, bytes).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn put_opts(
|
async fn put_opts(
|
||||||
&self,
|
&self,
|
||||||
location: &Path,
|
location: &Path,
|
||||||
@@ -91,6 +96,14 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
self.target.put_opts(location, bytes, opts).await
|
self.target.put_opts(location, bytes, opts).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn put_multipart(&self, location: &Path) -> OSResult<Box<dyn MultipartUpload>> {
|
||||||
|
let target = self.target.put_multipart(location).await?;
|
||||||
|
Ok(Box::new(IoTrackingMultipartUpload {
|
||||||
|
target,
|
||||||
|
stats: self.stats.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
async fn put_multipart_opts(
|
async fn put_multipart_opts(
|
||||||
&self,
|
&self,
|
||||||
location: &Path,
|
location: &Path,
|
||||||
@@ -103,6 +116,15 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get(&self, location: &Path) -> OSResult<GetResult> {
|
||||||
|
let result = self.target.get(location).await;
|
||||||
|
if let Ok(result) = &result {
|
||||||
|
let num_bytes = result.range.end - result.range.start;
|
||||||
|
self.record_read(num_bytes);
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
|
async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
|
||||||
let result = self.target.get_opts(location, options).await;
|
let result = self.target.get_opts(location, options).await;
|
||||||
if let Ok(result) = &result {
|
if let Ok(result) = &result {
|
||||||
@@ -112,6 +134,14 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get_range(&self, location: &Path, range: std::ops::Range<u64>) -> OSResult<Bytes> {
|
||||||
|
let result = self.target.get_range(location, range).await;
|
||||||
|
if let Ok(result) = &result {
|
||||||
|
self.record_read(result.len() as u64);
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
async fn get_ranges(
|
async fn get_ranges(
|
||||||
&self,
|
&self,
|
||||||
location: &Path,
|
location: &Path,
|
||||||
@@ -124,11 +154,20 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete_stream(
|
async fn head(&self, location: &Path) -> OSResult<ObjectMeta> {
|
||||||
&self,
|
self.record_read(0);
|
||||||
locations: BoxStream<'static, OSResult<Path>>,
|
self.target.head(location).await
|
||||||
) -> BoxStream<'static, OSResult<Path>> {
|
}
|
||||||
|
|
||||||
|
async fn delete(&self, location: &Path) -> OSResult<()> {
|
||||||
self.record_write(0);
|
self.record_write(0);
|
||||||
|
self.target.delete(location).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_stream<'a>(
|
||||||
|
&'a self,
|
||||||
|
locations: BoxStream<'a, OSResult<Path>>,
|
||||||
|
) -> BoxStream<'a, OSResult<Path>> {
|
||||||
self.target.delete_stream(locations)
|
self.target.delete_stream(locations)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -151,14 +190,24 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
self.target.list_with_delimiter(prefix).await
|
self.target.list_with_delimiter(prefix).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> OSResult<()> {
|
async fn copy(&self, from: &Path, to: &Path) -> OSResult<()> {
|
||||||
self.record_write(0);
|
self.record_write(0);
|
||||||
self.target.copy_opts(from, to, options).await
|
self.target.copy(from, to).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> OSResult<()> {
|
async fn rename(&self, from: &Path, to: &Path) -> OSResult<()> {
|
||||||
self.record_write(0);
|
self.record_write(0);
|
||||||
self.target.rename_opts(from, to, options).await
|
self.target.rename(from, to).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
|
||||||
|
self.record_write(0);
|
||||||
|
self.target.rename_if_not_exists(from, to).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
|
||||||
|
self.record_write(0);
|
||||||
|
self.target.copy_if_not_exists(from, to).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user