rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-08-18 12:08:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: make dynamic AWS credentials fail closed

Browse Source
This commit is contained in:
Gatefixer
2026-08-08 15:36:42 +00:00
parent 3a3e488b34
commit 83c84b3b6c
48 changed files with 30268 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+33 -6
View File
@@ -5213,7 +5213,7 @@ dependencies = [
"moka",
"object_store",
"object_store_opendal",
"opendal",
"opendal 0.58.1",
"path_abs",
"pin-project",
"prost",
@@ -5454,7 +5454,7 @@ dependencies = [
"num-traits",
"object_store",
"object_store_opendal",
"opendal",
"opendal 0.58.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project",
"polars",
"polars-arrow",
@@ -6422,7 +6422,7 @@ dependencies = [
"futures",
"mea",
"object_store",
"opendal",
"opendal 0.58.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project",
"tokio",
]
@@ -6476,8 +6476,6 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "opendal"
version = "0.58.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d"
dependencies = [
"ctor 1.0.12",
"opendal-core",
@@ -6493,7 +6491,17 @@ dependencies = [
"opendal-service-goosefs",
"opendal-service-hf",
"opendal-service-oss",
"opendal-service-s3",
"opendal-service-s3 0.58.1",
]
[[package]]
name = "opendal"
version = "0.58.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d"
dependencies = [
"opendal-core",
"opendal-service-s3 0.58.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -6716,6 +6724,25 @@ dependencies = [
"serde",
]
[[package]]
name = "opendal-service-s3"
version = "0.58.1"
dependencies = [
"base64 0.23.1",
"bytes",
"crc-fast",
"http 1.5.0",
"log",
"md-5 0.11.0",
"opendal-core",
"quick-xml 0.41.0",
"reqsign-aws-v4",
"reqsign-core",
"reqsign-file-read-tokio",
"serde",
"url",
]
[[package]]
name = "opendal-service-s3"
version = "0.58.1"
Cargo.toml
+14 -30
View File
@@ -14,20 +14,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance", default-features = false }
lance-core = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-core" }
lance-datagen = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-datagen" }
lance-file = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-file" }
lance-io = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-io", default-features = false }
lance-index = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-index" }
lance-linalg = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-linalg" }
lance-namespace = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-namespace" }
lance-namespace-impls = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-namespace-impls", default-features = false }
lance-table = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-table" }
lance-testing = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-testing" }
lance-datafusion = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-datafusion" }
lance-encoding = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-encoding" }
lance-arrow = { version = "=11.0.0-beta.3", path = "lance-artifact/rust/lance-arrow" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
@@ -68,22 +68,6 @@ regex = "1.10"
semver = "1.0.25"
chrono = "0.4"
[patch."https://github.com/lance-format/lance.git"]
lance = { path = "lance-artifact/rust/lance" }
lance-arrow = { path = "lance-artifact/rust/lance-arrow" }
lance-core = { path = "lance-artifact/rust/lance-core" }
lance-datafusion = { path = "lance-artifact/rust/lance-datafusion" }
lance-datagen = { path = "lance-artifact/rust/lance-datagen" }
lance-encoding = { path = "lance-artifact/rust/lance-encoding" }
lance-file = { path = "lance-artifact/rust/lance-file" }
lance-index = { path = "lance-artifact/rust/lance-index" }
lance-io = { path = "lance-artifact/rust/lance-io" }
lance-linalg = { path = "lance-artifact/rust/lance-linalg" }
lance-namespace = { path = "lance-artifact/rust/lance-namespace" }
lance-namespace-impls = { path = "lance-artifact/rust/lance-namespace-impls" }
lance-table = { path = "lance-artifact/rust/lance-table" }
lance-testing = { path = "lance-artifact/rust/lance-testing" }
[profile.ci]
debug = "line-tables-only"
inherits = "dev"
lance-artifact/Cargo.toml
+3 -1
View File
@@ -1,5 +1,7 @@
[workspace]
members = [
"third-party/opendal",
"third-party/opendal-service-s3",
"rust/examples",
"rust/lance",
"rust/lance-arrow",
@@ -175,7 +177,7 @@ moka = { version = "0.12", features = ["future", "sync"] }
ndarray = { version = "0.16.1", features = ["matrixmultiply-threading"] }
num-traits = "0.2"
object_store = { version = "0.13.2" }
opendal = { version = "0.58.1" }
opendal = { version = "0.58.1", path = "./third-party/opendal" }
object_store_opendal = { version = "0.58" }
reqsign-aws-v4 = { version = "3.1.0" }
reqsign-core = { version = "3.2.1" }
lance-artifact/LANCEDB_PATCH.md
+10
View File
@@ -10,3 +10,13 @@ keeps long-lived multipart uploads refreshable without rebuilding a store or cha
metadata. Keeping the change inside `AwsStoreProvider` leaves arbitrary registry providers and
their complete `ObjectStore` results untouched. Remove this patch when the same behavior is
available in the pinned Lance release.
The LanceDB workspace consumes every coupled Lance crate through a direct path into this artifact.
That keeps Git consumers on the same checked-in source instead of relying on a root `[patch]`,
which Cargo ignores when LanceDB itself is used as a dependency.
The artifact also contains Apache OpenDAL 0.58.1's `opendal` and `opendal-service-s3` crates. The
only OpenDAL change adds a direct custom-provider hook alongside its existing credential-chain
hook. Lance uses the direct hook so a selected dynamic authority can propagate refresh and
validation errors; static or ambient credentials are considered only when that authority returns
`Ok(None)`. Remove these copies when upstream OpenDAL exposes an equivalent hook.
lance-artifact/rust/lance-io/src/object_store/providers/aws.rs
+97 -7
View File
@@ -153,6 +153,27 @@ enum RequestTimeAwsCredentialProvider {
StorageOptions(Arc<StorageOptionsAccessor>),
}
#[derive(Debug)]
struct FailClosedAwsCredentialProvider {
dynamic: RequestTimeAwsCredentialProvider,
fallback: ProvideCredentialChain<ReqsignAwsCredential>,
}
impl ProvideCredential for FailClosedAwsCredentialProvider {
type Credential = ReqsignAwsCredential;
async fn provide_credential(
&self,
ctx: &ReqsignContext,
) -> reqsign_core::Result<Option<Self::Credential>> {
match self.dynamic.provide_credential(ctx).await {
Ok(Some(credential)) => Ok(Some(credential)),
Ok(None) => self.fallback.provide_credential(ctx).await,
Err(error) => Err(error),
}
}
}
fn request_time_expiration(options: &HashMap<String, String>) -> Option<ReqsignTimestamp> {
let expires_at_millis = options.get("expires_at_millis")?.parse::<i64>().ok()?;
let expires_at = ReqsignTimestamp::from_millisecond(expires_at_millis).ok()?;
@@ -248,14 +269,14 @@ fn build_opendal_s3_store(
return Ok(OpendalStore::new(operator));
};
let mut credential_chain = ProvideCredentialChain::new().push(dynamic_credentials);
let mut fallback = ProvideCredentialChain::new();
if let (Some(key_id), Some(secret_key)) = (&config.access_key_id, &config.secret_access_key) {
let static_provider = if let Some(token) = config.session_token.as_deref() {
ReqsignStaticCredentialProvider::new(key_id, secret_key).with_session_token(token)
} else {
ReqsignStaticCredentialProvider::new(key_id, secret_key)
};
credential_chain = credential_chain.push(static_provider);
fallback = fallback.push(static_provider);
}
let mut default_provider = ReqsignDefaultCredentialProvider::builder();
@@ -265,9 +286,14 @@ fn build_opendal_s3_store(
if config.disable_ec2_metadata {
default_provider = default_provider.no_imds();
}
credential_chain = credential_chain.push(default_provider.build());
fallback = fallback.push(default_provider.build());
// Supplying a custom chain replaces OpenDAL's entire chain. Recreate assume-role wrapping so
let credential_provider = FailClosedAwsCredentialProvider {
dynamic: dynamic_credentials,
fallback,
};
// Supplying a custom provider replaces OpenDAL's entire chain. Recreate assume-role wrapping so
// request-time source credentials retain the same authority as ordinary OpenDAL config.
if let Some(role_arn) = config.role_arn.take() {
let region = opendal_s3_region(&config)?;
@@ -276,7 +302,7 @@ fn build_opendal_s3_store(
.with_env(ReqsignOsEnv)
.with_http_send(HttpTransporter::default());
let request_signer = ReqsignAwsV4Signer::new("sts", &region);
let signer = ReqsignSigner::new(context, credential_chain, request_signer);
let signer = ReqsignSigner::new(context, credential_provider, request_signer);
let mut assume_role = AssumeRoleCredentialProvider::new(role_arn, signer)
.with_region(region)
.with_regional_sts_endpoint();
@@ -292,12 +318,16 @@ fn build_opendal_s3_store(
if let Some(tags) = config.assume_role_session_tags.clone() {
assume_role = assume_role.with_tags(tags.into_iter().collect());
}
credential_chain = ProvideCredentialChain::new().push(assume_role);
let builder = config.into_builder().credential_provider(assume_role);
let operator = Operator::new(builder).map_err(|error| {
Error::invalid_input(format!("Failed to create S3 operator: {error:?}"))
})?;
return Ok(OpendalStore::new(operator));
}
let builder = config
.into_builder()
.credential_provider_chain(credential_chain);
.credential_provider(credential_provider);
let operator = Operator::new(builder).map_err(|error| {
Error::invalid_input(format!("Failed to create S3 operator: {error:?}"))
})?;
@@ -952,6 +982,66 @@ mod tests {
);
}
#[derive(Debug)]
struct FailingStorageOptionsProvider;
#[async_trait::async_trait]
impl StorageOptionsProvider for FailingStorageOptionsProvider {
async fn fetch_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
Err(Error::invalid_input("injected credential refresh failure"))
}
fn provider_id(&self) -> String {
"failing-opendal-credential-test".to_string()
}
}
fn static_reqsign_fallback() -> ProvideCredentialChain<ReqsignAwsCredential> {
ProvideCredentialChain::new().push(ReqsignStaticCredentialProvider::new(
"fallback-key",
"fallback-secret",
))
}
#[tokio::test]
async fn test_opendal_dynamic_credential_errors_do_not_fall_back() {
let provider = FailClosedAwsCredentialProvider {
dynamic: RequestTimeAwsCredentialProvider::StorageOptions(Arc::new(
StorageOptionsAccessor::with_provider(Arc::new(FailingStorageOptionsProvider)),
)),
fallback: static_reqsign_fallback(),
};
let error = provider
.provide_credential(&ReqsignContext::new())
.await
.unwrap_err();
assert!(
error
.to_string()
.contains("injected credential refresh failure"),
"dynamic credential errors must reach the caller: {error}"
);
}
#[tokio::test]
async fn test_opendal_dynamic_credential_absence_allows_fallback() {
let provider = FailClosedAwsCredentialProvider {
dynamic: RequestTimeAwsCredentialProvider::StorageOptions(Arc::new(
StorageOptionsAccessor::with_static_options(HashMap::new()),
)),
fallback: static_reqsign_fallback(),
};
let credential = provider
.provide_credential(&ReqsignContext::new())
.await
.unwrap()
.unwrap();
assert_eq!(credential.access_key_id, "fallback-key");
assert_eq!(credential.secret_access_key, "fallback-secret");
}
#[tokio::test]
async fn test_injected_aws_creds_option_is_used() {
let mock_provider = Arc::new(MockAwsCredentialsProvider::default());
lance-artifact/third-party/opendal-service-s3/.cargo_vcs_info.json
Vendored
+6
View File
@@ -0,0 +1,6 @@
{
"git": {
"sha1": "6f544a131be78e553d28e79f3cfe5ac948f2963e"
},
"path_in_vcs": "core/services/s3"
}
lance-artifact/third-party/opendal-service-s3/Cargo.lock
Generated Vendored
+1494
View File
File diff suppressed because it is too large Load Diff
lance-artifact/third-party/opendal-service-s3/Cargo.toml
Vendored
+100
View File
@@ -0,0 +1,100 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2024"
rust-version = "1.91"
name = "opendal-service-s3"
version = "0.58.1"
authors = ["Apache OpenDAL <dev@opendal.apache.org>"]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "Apache OpenDAL S3 service implementation"
homepage = "https://opendal.apache.org/"
readme = "README.md"
license = "Apache-2.0"
repository = "https://github.com/apache/opendal"
[package.metadata.docs.rs]
all-features = true
[lib]
name = "opendal_service_s3"
path = "src/lib.rs"
[dependencies.base64]
version = "0.23"
[dependencies.bytes]
version = "1.10"
[dependencies.crc-fast]
version = "1.9.0"
[dependencies.http]
version = "1.4"
[dependencies.log]
version = "0.4.32"
[dependencies.md-5]
version = "0.11.0"
[dependencies.opendal-core]
version = "0.58.1"
features = ["reqsign"]
default-features = false
[dependencies.quick-xml]
version = "0.41.0"
features = [
"serialize",
"overlapped-lists",
]
default-features = false
[dependencies.reqsign-aws-v4]
version = "3.0.3"
default-features = false
[dependencies.reqsign-core]
version = "3.2.0"
default-features = false
[dependencies.reqsign-file-read-tokio]
version = "3.0.3"
default-features = false
[dependencies.serde]
version = "1"
features = ["derive"]
default-features = false
[dependencies.url]
version = "2.5"
[dev-dependencies.pretty_assertions]
version = "1"
[dev-dependencies.serde_json]
version = "1"
[dev-dependencies.tokio]
version = "1.52"
features = [
"macros",
"rt-multi-thread",
]
default-features = false
lance-artifact/third-party/opendal-service-s3/Cargo.toml.orig
Generated Vendored
+54
View File
@@ -0,0 +1,54 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[package]
description = "Apache OpenDAL S3 service implementation"
name = "opendal-service-s3"
readme = "README.md"
authors = { workspace = true }
edition = { workspace = true }
homepage = { workspace = true }
license = { workspace = true }
repository = { workspace = true }
rust-version = { workspace = true }
version = { workspace = true }
[package.metadata.docs.rs]
all-features = true
[dependencies]
base64 = { workspace = true }
bytes = { workspace = true }
crc-fast = "1.9.0"
http = { workspace = true }
log = { workspace = true }
md-5 = "0.11.0"
opendal-core = { path = "../../core", version = "0.58.1", default-features = false, features = [
"reqsign",
] }
quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
reqsign-aws-v4 = { version = "3.0.3", default-features = false }
reqsign-core = { version = "3.2.0", default-features = false }
reqsign-file-read-tokio = { version = "3.0.3", default-features = false }
serde = { workspace = true, features = ["derive"] }
url = { workspace = true }
[dev-dependencies]
pretty_assertions = "1"
serde_json = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
lance-artifact/third-party/opendal-service-s3/LICENSE
Vendored
+201
View File
@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
lance-artifact/third-party/opendal-service-s3/NOTICE
Vendored
+5
View File
@@ -0,0 +1,5 @@
Apache OpenDAL
Copyright 2023-2026 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
lance-artifact/third-party/opendal-service-s3/README.md
Vendored
+56
View File
@@ -0,0 +1,56 @@
# Apache OpenDAL™ Amazon S3 Service
`opendal-service-s3` provides access to Amazon S3 and S3-compatible object storage for
applications built with Apache OpenDAL™.
## Use through `opendal`
Applications should normally enable this service through the `opendal` facade with the
`services-s3` feature:
```shell
cargo add opendal --features services-s3
```
The service is available as `opendal::services::S3`. Configure the
service builder, then pass it to `opendal::Operator::new`.
## Use with `opendal-core`
Add the split crates directly:
```shell
cargo add opendal-core opendal-service-s3
```
Pass a configured service builder to `Operator::new`:
```rust
use opendal_core::{Operator, OperatorRegistry, Result};
use opendal_service_s3::{register_s3_service, S3};
fn build_operator(builder: S3) -> Result<Operator> {
Operator::new(builder)
}
fn register_for_uri() {
register_s3_service(OperatorRegistry::get());
}
```
`register_for_uri` is only needed for scheme-driven construction through
`Operator::from_uri` or `Operator::via_iter`.
Services that send HTTP requests also require an HTTP transport in
`OperationContext`. See the
[`opendal-core` composition example](https://crates.io/crates/opendal-core).
## Documentation
- [Service configuration and examples](https://opendal.apache.org/services/s3)
- [Rust API documentation](https://docs.rs/opendal-service-s3)
- [Apache OpenDAL documentation](https://opendal.apache.org/docs/)
## License
Licensed under the Apache License, Version 2.0.
lance-artifact/third-party/opendal-service-s3/src/backend.rs
Vendored
+1428
View File
File diff suppressed because it is too large Load Diff
lance-artifact/third-party/opendal-service-s3/src/compatible_services.md
Vendored
+153
View File
@@ -0,0 +1,153 @@
## Compatible Services
### AWS S3
[AWS S3](https://aws.amazon.com/s3/) is the default implementations of s3 services. Only `bucket` is required.
```rust,ignore
builder.bucket("<bucket_name>");
```
### Alibaba Object Storage Service (OSS)
[OSS](https://www.alibabacloud.com/product/object-storage-service) is a s3 compatible service provided by [Alibaba Cloud](https://www.alibabacloud.com).
To connect to OSS, we need to set:
- `endpoint`: The endpoint of oss, for example: `https://oss-cn-hangzhou.aliyuncs.com`
- `bucket`: The bucket name of oss.
> OSS provide internal endpoint for used at alibabacloud internally, please visit [OSS Regions and endpoints](https://www.alibabacloud.com/help/en/object-storage-service/latest/regions-and-endpoints) for more details.
> OSS only supports the virtual host style, users could meet errors like:
>
> ```xml
> <?xml version="1.0" encoding="UTF-8"?>
> <Error>
> <Code>SecondLevelDomainForbidden</Code>
> <Message>The bucket you are attempting to access must be addressed using OSS third level domain.</Message>
> <RequestId>62A1C265292C0632377F021F</RequestId>
> <HostId>oss-cn-hangzhou.aliyuncs.com</HostId>
> </Error>
> ```
>
> In that case, please enable virtual host style for requesting.
```rust,ignore
builder.endpoint("https://oss-cn-hangzhou.aliyuncs.com");
builder.region("<region>");
builder.bucket("<bucket_name>");
builder.enable_virtual_host_style();
```
### Minio
[minio](https://min.io/) is an open-source s3 compatible services.
To connect to minio, we need to set:
- `endpoint`: The endpoint of minio, for example: `http://127.0.0.1:9000`
- `region`: The region of minio. If you don't care about it, just set it to "auto", it will be ignored.
- `bucket`: The bucket name of minio.
```rust,ignore
builder.endpoint("http://127.0.0.1:9000");
builder.region("<region>");
builder.bucket("<bucket_name>");
```
### QingStor Object Storage
[QingStor Object Storage](https://www.qingcloud.com/products/qingstor) is a S3-compatible service provided by [QingCloud](https://www.qingcloud.com/).
To connect to QingStor Object Storage, we need to set:
- `endpoint`: The endpoint of QingStor s3 compatible endpoint, for example: `https://s3.pek3b.qingstor.com`
- `bucket`: The bucket name.
### Scaleway Object Storage
[Scaleway Object Storage](https://www.scaleway.com/en/object-storage/) is a S3-compatible and multi-AZ redundant object storage service.
To connect to Scaleway Object Storage, we need to set:
- `endpoint`: The endpoint of scaleway, for example: `https://s3.nl-ams.scw.cloud`
- `region`: The region of scaleway.
- `bucket`: The bucket name of scaleway.
### Tencent Cloud Object Storage (COS)
[COS](https://intl.cloud.tencent.com/products/cos) is a s3 compatible service provided by [Tencent Cloud](https://intl.cloud.tencent.com/).
To connect to COS, we need to set:
- `endpoint`: The endpoint of cos, for example: `https://cos.ap-beijing.myqcloud.com`
- `bucket`: The bucket name of cos.
### Tigris
[Tigris](https://www.tigrisdata.com/) is a globally distributed S3-compatible object storage service.
> Tigris serves data from the region nearest to the requester through a single global endpoint, and charges no egress fees.
To connect to Tigris, we need to set:
- `endpoint`: The endpoint of Tigris: `https://t3.storage.dev`
- `region`: The region of Tigris. Please set it to `auto`.
- `bucket`: The bucket name of Tigris.
Credentials are standard S3 access keys, provided via `access_key_id`/`secret_access_key` or the `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` environment variables. See [Tigris S3 SDK docs](https://www.tigrisdata.com/docs/sdks/s3/) for details.
```rust,ignore
builder.endpoint("https://t3.storage.dev");
builder.region("auto");
builder.bucket("<bucket_name>");
```
> `https://fly.storage.tigris.dev` is a legacy endpoint that still works, but `https://t3.storage.dev` is the current one.
Tigris buckets support zero-copy [snapshots and forks](https://www.tigrisdata.com/docs/snapshots-and-forks/). A forked bucket is a regular bucket, so it can be read and written through OpenDAL like any other — useful for testing against production data without copying it.
### Wasabi Object Storage
[Wasabi](https://wasabi.com/) is a s3 compatible service.
> Cloud storage pricing that is 80% less than Amazon S3.
To connect to wasabi, we need to set:
- `endpoint`: The endpoint of wasabi, for example: `https://s3.us-east-2.wasabisys.com`
- `bucket`: The bucket name of wasabi.
> Refer to [What are the service URLs for Wasabi's different storage regions?](https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031) for more details.
### Cloudflare R2
[Cloudflare R2](https://developers.cloudflare.com/r2/) provides s3 compatible API.
> Cloudflare R2 Storage allows developers to store large amounts of unstructured data without the costly egress bandwidth fees associated with typical cloud storage services.
To connect to r2, we need to set:
- `endpoint`: The endpoint of r2, for example: `https://<account_id>.r2.cloudflarestorage.com`
- `bucket`: The bucket name of r2.
- `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region.
- `enable_exact_buf_write`: R2 requires the non-tailing parts size to be exactly the same. Please enable this option to avoid the error `All non-trailing parts must have the same length`.
R2 has the following capability differences from S3:
- `delete_max_size`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please override `delete_max_size` to `700`.
- `stat_with_override_cache_control`, `stat_with_override_content_disposition`, `stat_with_override_content_type`: R2 doesn't support stat with response override queries. Please override them to `false`.
### Google Cloud Storage XML API
[Google Cloud Storage XML API](https://cloud.google.com/storage/docs/xml-api/overview) provides s3 compatible API.
- `endpoint`: The endpoint of Google Cloud Storage XML API, for example: `https://storage.googleapis.com`
- `bucket`: The bucket name.
- To access GCS via S3 API, please enable `features = ["native-tls"]` in your `Cargo.toml` to avoid connection being reset when using `rustls`. Tracking in <https://github.com/seanmonstar/reqwest/issues/1809>
### Ceph Rados Gateway
Ceph supports a RESTful API that is compatible with the basic data access model of the Amazon S3 API.
For more information, refer: <https://docs.ceph.com/en/latest/radosgw/s3/>
lance-artifact/third-party/opendal-service-s3/src/config.rs
Vendored
+476
View File
@@ -0,0 +1,476 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::collections::HashMap;
use std::fmt::Debug;
use opendal_core::Configurator;
use opendal_core::OperatorUri;
use opendal_core::Result;
use serde::Deserialize;
use serde::Serialize;
use crate::backend::S3Builder;
/// Config for Aws S3 and compatible services (including minio, digitalocean space,
/// Tencent Cloud Object Storage(COS) and so on) support.
#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
#[serde(default)]
#[non_exhaustive]
pub struct S3Config {
/// root of this backend.
///
/// All operations will happen under this root.
///
/// default to `/` if not set.
///
/// <!-- @group General -->
/// <!-- @default / -->
pub root: Option<String>,
/// bucket name of this backend.
///
/// required.
///
/// <!-- @group General -->
/// <!-- @example my-bucket -->
#[serde(alias = "aws_bucket", alias = "aws_bucket_name", alias = "bucket_name")]
pub bucket: String,
/// Deprecated: S3 versioning capability is enabled by default.
///
/// <!-- @group Deprecated -->
#[deprecated(
since = "0.57.0",
note = "S3 versioning capability is enabled by default and this option is no longer needed."
)]
pub enable_versioning: bool,
/// endpoint of this backend.
///
/// Endpoint must be full uri, e.g.
///
/// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com`
/// - Cloudflare R2: `https://<ACCOUNT_ID>.r2.cloudflarestorage.com`
/// - Aliyun OSS: `https://{region}.aliyuncs.com`
/// - Tencent COS: `https://cos.{region}.myqcloud.com`
/// - Minio: `http://127.0.0.1:9000`
///
/// If user inputs endpoint without scheme like "s3.amazonaws.com", we
/// will prepend "https://" before it.
///
/// - If endpoint is set, we will take user's input first.
/// - If not, we will try to load it from environment.
/// - If still not set, default to `https://s3.amazonaws.com`.
///
/// <!-- @group General -->
/// <!-- @default https://s3.amazonaws.com -->
#[serde(
alias = "aws_endpoint",
alias = "aws_endpoint_url",
alias = "endpoint_url"
)]
pub endpoint: Option<String>,
/// Region represent the signing region of this endpoint. This is required
/// if you are using the default AWS S3 endpoint.
///
/// If using a custom endpoint,
/// - If region is set, we will take user's input first.
/// - If not, we will try to load it from environment.
///
/// <!-- @group General -->
/// <!-- @example us-east-1 -->
#[serde(alias = "aws_region")]
pub region: Option<String>,
/// access_key_id of this backend.
///
/// - If access_key_id is set, we will take user's input first.
/// - If not, we will try to load it from environment.
///
/// <!-- @group Credentials -->
#[serde(alias = "aws_access_key_id")]
pub access_key_id: Option<String>,
/// secret_access_key of this backend.
///
/// - If secret_access_key is set, we will take user's input first.
/// - If not, we will try to load it from environment.
///
/// <!-- @group Credentials -->
#[serde(alias = "aws_secret_access_key")]
pub secret_access_key: Option<String>,
/// session_token (aka, security token) of this backend.
///
/// This token will expire after sometime, it's recommended to set session_token
/// by hand.
///
/// <!-- @group Credentials -->
#[serde(alias = "aws_session_token", alias = "aws_token", alias = "token")]
pub session_token: Option<String>,
/// role_arn for this backend.
///
/// If `role_arn` is set, we will use already known config as source
/// credential to assume role with `role_arn`.
///
/// <!-- @group Assume role -->
pub role_arn: Option<String>,
/// external_id for this backend.
///
/// <!-- @group Assume role -->
pub external_id: Option<String>,
/// role_session_name for this backend.
///
/// <!-- @group Assume role -->
pub role_session_name: Option<String>,
/// assume_role_duration_seconds for this backend.
///
/// <!-- @group Assume role -->
pub assume_role_duration_seconds: Option<u32>,
/// assume_role_session_tags for this backend.
///
/// <!-- @group Assume role -->
pub assume_role_session_tags: Option<HashMap<String, String>>,
/// Disable config load so that opendal will not load config from
/// environment.
///
/// For examples:
///
/// - envs like `AWS_ACCESS_KEY_ID`
/// - files like `~/.aws/config`
///
/// <!-- @group Credentials -->
pub disable_config_load: bool,
/// Disable load credential from ec2 metadata.
///
/// This option is used to disable the default behavior of opendal
/// to load credential from ec2 metadata, a.k.a., IMDSv2
///
/// <!-- @group Credentials -->
pub disable_ec2_metadata: bool,
/// Skip signature will skip loading credentials and signing requests.
///
/// <!-- @group Credentials -->
pub skip_signature: bool,
/// Allow anonymous will allow opendal to send request without signing
/// when credential is not loaded.
///
/// <!-- @group Deprecated -->
#[deprecated(
since = "0.57.0",
note = "Please use `skip_signature` instead of `allow_anonymous`"
)]
pub allow_anonymous: bool,
/// server_side_encryption for this backend.
///
/// Available values: `AES256`, `aws:kms`.
///
/// <!-- @group Encryption -->
#[serde(alias = "aws_server_side_encryption")]
pub server_side_encryption: Option<String>,
/// server_side_encryption_aws_kms_key_id for this backend
///
/// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
/// is not set, S3 will use aws managed kms key to encrypt data.
/// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
/// is a valid kms key id, S3 will use the provided kms key to encrypt data.
/// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be
/// returned.
/// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id`
/// is a noop.
///
/// <!-- @group Encryption -->
#[serde(alias = "aws_sse_kms_key_id")]
pub server_side_encryption_aws_kms_key_id: Option<String>,
/// server_side_encryption_customer_algorithm for this backend.
///
/// Available values: `AES256`.
///
/// <!-- @group Encryption -->
pub server_side_encryption_customer_algorithm: Option<String>,
/// server_side_encryption_customer_key for this backend.
///
/// Value: BASE64-encoded key that matches algorithm specified in
/// `server_side_encryption_customer_algorithm`.
///
/// <!-- @group Encryption -->
#[serde(alias = "aws_sse_customer_key_base64")]
pub server_side_encryption_customer_key: Option<String>,
/// Set server_side_encryption_customer_key_md5 for this backend.
///
/// Value: MD5 digest of key specified in `server_side_encryption_customer_key`.
///
/// <!-- @group Encryption -->
pub server_side_encryption_customer_key_md5: Option<String>,
/// default storage_class for this backend.
///
/// Available values:
/// - `DEEP_ARCHIVE`
/// - `GLACIER`
/// - `GLACIER_IR`
/// - `INTELLIGENT_TIERING`
/// - `ONEZONE_IA`
/// - `EXPRESS_ONEZONE`
/// - `OUTPOSTS`
/// - `REDUCED_REDUNDANCY`
/// - `STANDARD`
/// - `STANDARD_IA`
///
/// S3 compatible services don't support all of them
///
/// <!-- @group Behavior -->
pub default_storage_class: Option<String>,
/// Enable virtual host style so that opendal will send API requests
/// in virtual host style instead of path style.
///
/// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name`
/// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com`
///
/// <!-- @group Behavior -->
#[serde(
alias = "aws_virtual_hosted_style_request",
alias = "virtual_hosted_style_request"
)]
pub enable_virtual_host_style: bool,
/// Deprecated: S3 delete batch capability is enabled by default.
///
/// <!-- @group Deprecated -->
#[deprecated(
since = "0.57.0",
note = "S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."
)]
pub batch_max_operations: Option<usize>,
/// Deprecated: S3 delete batch capability is enabled by default.
///
/// <!-- @group Deprecated -->
#[deprecated(
since = "0.57.0",
note = "S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."
)]
pub delete_max_size: Option<usize>,
/// Deprecated: S3 stat override capabilities are enabled by default.
///
/// <!-- @group Deprecated -->
#[deprecated(
since = "0.57.0",
note = "S3 stat override capabilities are enabled by default. Use CapabilityOverrideLayer to override them for specific endpoints."
)]
pub disable_stat_with_override: bool,
/// Checksum Algorithm to use when sending checksums in HTTP headers.
/// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example.
///
/// Available options:
/// - "crc32c"
/// - "md5"
///
/// <!-- @group Behavior -->
#[serde(alias = "aws_checksum_algorithm")]
pub checksum_algorithm: Option<String>,
/// Deprecated: S3 write with If-Match capability is enabled by default.
///
/// <!-- @group Deprecated -->
#[deprecated(
since = "0.57.0",
note = "S3 write with If-Match capability is enabled by default and this option is no longer needed."
)]
pub disable_write_with_if_match: bool,
/// Deprecated: S3 append capability is enabled by default.
///
/// <!-- @group Deprecated -->
#[deprecated(
since = "0.57.0",
note = "S3 append capability is enabled by default and this option is no longer needed."
)]
pub enable_write_with_append: bool,
/// OpenDAL uses List Objects V2 by default to list objects.
/// However, some legacy services do not yet support V2.
/// This option allows users to switch back to the older List Objects V1.
///
/// <!-- @group Behavior -->
pub disable_list_objects_v2: bool,
/// Indicates whether the client agrees to pay for the requests made to the S3 bucket.
///
/// <!-- @group Behavior -->
#[serde(alias = "aws_request_payer", alias = "request_payer")]
pub enable_request_payer: bool,
/// Default ACL for new objects.
/// Note that some s3 services like minio do not support this option.
///
/// <!-- @group Behavior -->
pub default_acl: Option<String>,
}
impl Debug for S3Config {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("S3Config")
.field("root", &self.root)
.field("bucket", &self.bucket)
.field("endpoint", &self.endpoint)
.field("region", &self.region)
.finish_non_exhaustive()
}
}
impl Configurator for S3Config {
type Builder = S3Builder;
fn from_uri(uri: &OperatorUri) -> Result<Self> {
let mut map = uri.options().clone();
if let Some(name) = uri.name() {
map.insert("bucket".to_string(), name.to_string());
}
if let Some(root) = uri.root() {
map.insert("root".to_string(), root.to_string());
}
Self::from_iter(map)
}
#[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
S3Builder {
config: self,
credential_provider: None,
}
}
}
#[cfg(test)]
mod tests {
use std::iter;
use super::*;
use opendal_core::Configurator;
use opendal_core::OperatorUri;
#[test]
fn test_s3_config_original_field_names() {
let json = r#"{
"bucket": "test-bucket",
"access_key_id": "test-key",
"secret_access_key": "test-secret",
"region": "us-west-2",
"endpoint": "https://s3.amazonaws.com",
"session_token": "test-token"
}"#;
let config: S3Config = serde_json::from_str(json).unwrap();
assert_eq!(config.bucket, "test-bucket");
assert_eq!(config.access_key_id, Some("test-key".to_string()));
assert_eq!(config.secret_access_key, Some("test-secret".to_string()));
assert_eq!(config.region, Some("us-west-2".to_string()));
assert_eq!(
config.endpoint,
Some("https://s3.amazonaws.com".to_string())
);
assert_eq!(config.session_token, Some("test-token".to_string()));
}
#[test]
fn test_s3_config_aws_prefixed_aliases() {
let json = r#"{
"aws_bucket": "test-bucket",
"aws_access_key_id": "test-key",
"aws_secret_access_key": "test-secret",
"aws_region": "us-west-2",
"aws_endpoint": "https://s3.amazonaws.com",
"aws_session_token": "test-token"
}"#;
let config: S3Config = serde_json::from_str(json).unwrap();
assert_eq!(config.bucket, "test-bucket");
assert_eq!(config.access_key_id, Some("test-key".to_string()));
assert_eq!(config.secret_access_key, Some("test-secret".to_string()));
assert_eq!(config.region, Some("us-west-2".to_string()));
assert_eq!(
config.endpoint,
Some("https://s3.amazonaws.com".to_string())
);
assert_eq!(config.session_token, Some("test-token".to_string()));
}
#[test]
fn test_s3_config_additional_aliases() {
let json = r#"{
"bucket_name": "test-bucket",
"token": "test-token",
"endpoint_url": "https://s3.amazonaws.com",
"virtual_hosted_style_request": true,
"aws_checksum_algorithm": "crc32c",
"request_payer": true
}"#;
let config: S3Config = serde_json::from_str(json).unwrap();
assert_eq!(config.bucket, "test-bucket");
assert_eq!(config.session_token, Some("test-token".to_string()));
assert_eq!(
config.endpoint,
Some("https://s3.amazonaws.com".to_string())
);
assert!(config.enable_virtual_host_style);
assert_eq!(config.checksum_algorithm, Some("crc32c".to_string()));
assert!(config.enable_request_payer);
}
#[test]
fn test_s3_config_encryption_aliases() {
let json = r#"{
"bucket": "test-bucket",
"aws_server_side_encryption": "aws:kms",
"aws_sse_kms_key_id": "test-kms-key",
"aws_sse_customer_key_base64": "dGVzdC1jdXN0b21lci1rZXk="
}"#;
let config: S3Config = serde_json::from_str(json).unwrap();
assert_eq!(config.bucket, "test-bucket");
assert_eq!(config.server_side_encryption, Some("aws:kms".to_string()));
assert_eq!(
config.server_side_encryption_aws_kms_key_id,
Some("test-kms-key".to_string())
);
assert_eq!(
config.server_side_encryption_customer_key,
Some("dGVzdC1jdXN0b21lci1rZXk=".to_string())
);
}
#[test]
fn from_uri_extracts_bucket_and_root() {
let uri = OperatorUri::new("s3://example-bucket/path/to/root", iter::empty()).unwrap();
let cfg = S3Config::from_uri(&uri).unwrap();
assert_eq!(cfg.bucket, "example-bucket");
assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
}
#[test]
fn from_uri_extracts_endpoint() {
let uri = OperatorUri::new(
"s3://example-bucket/path/to/root?endpoint=https%3A%2F%2Fcustom-s3-endpoint.com",
iter::empty(),
)
.unwrap();
let cfg = S3Config::from_uri(&uri).unwrap();
assert_eq!(cfg.bucket, "example-bucket");
assert_eq!(cfg.root.as_deref(), Some("path/to/root"));
assert_eq!(
cfg.endpoint.as_deref(),
Some("https://custom-s3-endpoint.com")
);
}
}
lance-artifact/third-party/opendal-service-s3/src/copier.rs
Vendored
+290
View File
@@ -0,0 +1,290 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::sync::Arc;
use bytes::Buf;
use http::StatusCode;
use crate::core::S3Error;
use crate::core::from_s3_error;
use crate::core::parse_error;
use crate::core::*;
use opendal_core::raw::*;
use opendal_core::*;
pub type S3Copiers = oio::MultipartCopier<S3Copier>;
pub fn new_s3_copier(
core: Arc<S3Core>,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
) -> Result<S3Copiers> {
let capability = core.capability;
let max_part_size = capability.copy_multi_max_size.ok_or_else(|| {
Error::new(
ErrorKind::Unexpected,
"multipart copy requires copy_multi_max_size capability",
)
})?;
let (copy_once_threshold, part_size) = match opts.chunk() {
Some(chunk) => {
let min_part_size = capability.copy_multi_min_size.ok_or_else(|| {
Error::new(
ErrorKind::Unexpected,
"multipart copy requires copy_multi_min_size capability",
)
})?;
let part_size = chunk.clamp(min_part_size, max_part_size) as u64;
(part_size.saturating_sub(1), part_size)
}
None => {
let part_size = max_part_size as u64;
(part_size, part_size)
}
};
Ok(oio::MultipartCopier::new(
(ctx.executor().clone(), capability),
S3Copier {
core,
ctx: ctx.clone(),
from: from.to_string(),
to: to.to_string(),
args,
},
opts.source_content_length_hint(),
copy_once_threshold,
part_size,
opts.concurrent(),
))
}
pub struct S3Copier {
core: Arc<S3Core>,
ctx: OperationContext,
from: String,
to: String,
args: OpCopy,
}
impl oio::MultipartCopy for S3Copier {
async fn source_metadata(&self) -> Result<Metadata> {
let mut args = OpStat::default();
if let Some(version) = self.args.source_version() {
args = args.with_version(version);
}
let resp = self
.core
.s3_head_object(&self.ctx, &self.from, args)
.await?;
match resp.status() {
StatusCode::OK => {
let headers = resp.headers();
parse_into_metadata(&self.from, headers)
}
_ => Err(parse_error(resp)),
}
}
async fn copy_once(&self) -> Result<Metadata> {
let resp = self
.core
.s3_copy_object(&self.ctx, &self.from, &self.to, &self.args)
.await?;
match resp.status() {
StatusCode::OK => {
let (parts, body) = resp.into_parts();
let bs = body.to_bytes();
let version = parse_header_to_str(&parts.headers, constants::X_AMZ_VERSION_ID)?
.map(str::to_string);
let result: CopyObjectResult =
quick_xml::de::from_reader(bs.as_ref()).map_err(new_xml_deserialize_error)?;
// S3 may return 200 OK with an <Error> body for CopyObject.
if result.etag.is_empty() {
return Err(from_s3_error(
S3Error {
code: result.code,
message: result.message,
resource: String::new(),
request_id: result.request_id,
},
parts,
));
}
let mut meta = Metadata::new(EntryMode::from_path(&self.to));
meta.set_etag(&result.etag);
if !result.last_modified.is_empty() {
meta.set_last_modified(result.last_modified.parse()?);
}
if let Some(version) = version {
meta.set_version(&version);
}
Ok(meta)
}
_ => Err(parse_error(resp)),
}
}
async fn initiate_copy(&self) -> Result<String> {
let resp = self
.core
.s3_initiate_multipart_copy(&self.ctx, &self.to)
.await?;
match resp.status() {
StatusCode::OK => {
let bs = resp.into_body();
let result: InitiateMultipartUploadResult =
quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
Ok(result.upload_id)
}
_ => Err(parse_error(resp)),
}
}
async fn copy_part(
&self,
upload_id: &str,
part_number: usize,
range: BytesRange,
) -> Result<oio::MultipartPart> {
let size = range.size().expect("multipart copy range must be sized");
let part_number = part_number + 1;
let req = self
.core
.s3_upload_part_copy_request(S3UploadPartCopyRequest {
from: &self.from,
to: &self.to,
source_version: self.args.source_version(),
upload_id,
part_number,
range,
})?;
let resp = self.core.send(&self.ctx, req).await?;
match resp.status() {
StatusCode::OK => {
let (parts, body) = resp.into_parts();
let bs = body.to_bytes();
let result: CopyObjectResult =
quick_xml::de::from_reader(bs.as_ref()).map_err(new_xml_deserialize_error)?;
// S3 may return 200 OK with an <Error> body for UploadPartCopy.
if result.etag.is_empty() {
return Err(from_s3_error(
S3Error {
code: result.code,
message: result.message,
resource: String::new(),
request_id: result.request_id,
},
parts,
));
}
Ok(oio::MultipartPart {
part_number,
etag: result.etag,
checksum: None,
size: Some(size),
})
}
_ => Err(parse_error(resp)),
}
}
async fn complete_copy(
&self,
upload_id: &str,
parts: &[oio::MultipartPart],
) -> Result<Metadata> {
let parts = parts
.iter()
.map(|p| CompleteMultipartUploadRequestPart {
part_number: p.part_number,
etag: p.etag.clone(),
..Default::default()
})
.collect();
let resp = self
.core
.s3_complete_multipart_copy(&self.ctx, &self.to, upload_id, parts, &self.args)
.await?;
let status = resp.status();
match status {
StatusCode::OK => {
let (parts, body) = resp.into_parts();
let version = parse_header_to_str(&parts.headers, constants::X_AMZ_VERSION_ID)?
.map(str::to_string);
let ret: CompleteMultipartUploadResult =
quick_xml::de::from_reader(body.reader()).map_err(new_xml_deserialize_error)?;
// S3 may return 200 OK with an <Error> body for CompleteMultipartUpload.
if ret.etag.is_empty() {
return Err(from_s3_error(
S3Error {
code: ret.code,
message: ret.message,
resource: "".to_string(),
request_id: ret.request_id,
},
parts,
));
}
let mut meta = Metadata::new(EntryMode::from_path(&self.to));
meta.set_etag(&ret.etag);
if let Some(version) = version {
meta.set_version(&version);
}
Ok(meta)
}
_ => Err(parse_error(resp)),
}
}
async fn abort_copy(&self, upload_id: &str) -> Result<()> {
let resp = self
.core
.s3_abort_multipart_copy(&self.ctx, &self.to, upload_id)
.await?;
match resp.status() {
StatusCode::NO_CONTENT => Ok(()),
_ => Err(parse_error(resp)),
}
}
}
lance-artifact/third-party/opendal-service-s3/src/core.rs
Vendored
+2201
View File
File diff suppressed because it is too large Load Diff
lance-artifact/third-party/opendal-service-s3/src/deleter.rs
Vendored
+111
View File
@@ -0,0 +1,111 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::sync::Arc;
use bytes::Buf;
use http::StatusCode;
use crate::core::parse_error;
use crate::core::parse_s3_error_code;
use crate::core::*;
use opendal_core::raw::oio::BatchDeleteResult;
use opendal_core::raw::*;
use opendal_core::*;
pub struct S3Deleter {
core: Arc<S3Core>,
ctx: OperationContext,
}
impl S3Deleter {
pub fn new(core: Arc<S3Core>, ctx: OperationContext) -> Self {
Self { core, ctx }
}
}
impl oio::BatchDelete for S3Deleter {
async fn delete_once(&self, path: String, args: OpDelete) -> Result<()> {
// This would delete the bucket, do not perform
if self.core.root == "/" && path == "/" {
return Ok(());
}
let resp = self.core.s3_delete_object(&self.ctx, &path, &args).await?;
let status = resp.status();
match status {
StatusCode::NO_CONTENT => Ok(()),
// Allow 404 when deleting a non-existing object
// This is not a standard behavior, only some s3 alike service like GCS XML API do this.
// ref: <https://cloud.google.com/storage/docs/xml-api/delete-object>
StatusCode::NOT_FOUND => Ok(()),
_ => Err(parse_error(resp)),
}
}
async fn delete_batch(&self, batch: Vec<(String, OpDelete)>) -> Result<BatchDeleteResult> {
let resp = self.core.s3_delete_objects(&self.ctx, &batch).await?;
let status = resp.status();
if status != StatusCode::OK {
return Err(parse_error(resp));
}
let bs = resp.into_body();
let result: DeleteObjectsResult =
quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
let mut errors = result.error;
let mut batched_result = BatchDeleteResult {
// `errors.len()` is server-controlled and may exceed `batch.len()`; use
// `saturating_sub` (mirroring services/tos) to avoid an unsigned underflow
// that wraps to a huge `with_capacity` (release: abort).
succeeded: Vec::with_capacity(batch.len().saturating_sub(errors.len())),
failed: Vec::with_capacity(errors.len()),
};
for (path, op) in batch {
let abs_path = build_abs_path(&self.core.root, &path);
// Assume errors are rare, so lookup and erase is acceptable.
if let Some(idx) = errors
.iter()
.position(|e| e.key == abs_path && e.version_id.as_deref() == op.version())
{
let error = errors.swap_remove(idx);
batched_result
.failed
.push((path, op, parse_delete_objects_result_error(error)));
} else {
batched_result.succeeded.push((path, op));
}
}
Ok(batched_result)
}
}
fn parse_delete_objects_result_error(err: DeleteObjectsResultError) -> Error {
let (kind, retryable) =
parse_s3_error_code(err.code.as_str()).unwrap_or((ErrorKind::Unexpected, false));
let mut err: Error = Error::new(kind, format!("{err:?}"));
if retryable {
err = err.set_temporary();
}
err
}
lance-artifact/third-party/opendal-service-s3/src/docs.md
Vendored
+256
View File
@@ -0,0 +1,256 @@
## Capabilities
Depending on its configuration and the backing system, this service can expose:
- [ ] create_dir
- [x] stat
- [x] read
- [x] write
- [x] delete
- [x] list
- [x] copy
- [ ] rename
- [x] presign
Inspect the effective capability set with [`opendal_core::Operator::info`] and
[`opendal_core::OperatorInfo::capability`] after building an operator.
## Configuration
Use [`crate::S3Config`] for serializable configuration and this builder's
methods for direct construction. The field and method documentation defines
accepted values, defaults, and environment interaction.
## Temporary security credentials
OpenDAL now provides support for S3 temporary security credentials in IAM.
The way to take advantage of this feature is to build your S3 backend with `Builder::session_token`.
But OpenDAL will not refresh the temporary security credentials, please keep in mind to refresh those credentials in time.
## Server Side Encryption
OpenDAL provides full support of S3 Server Side Encryption(SSE) features.
The easiest way to configure them is to use helper functions like
- SSE-KMS: `server_side_encryption_with_aws_managed_kms_key`
- SSE-KMS: `server_side_encryption_with_customer_managed_kms_key`
- SSE-S3: `server_side_encryption_with_s3_key`
- SSE-C: `server_side_encryption_with_customer_key`
If those functions don't fulfill need, low-level options are also provided:
- Use service managed kms key
- `server_side_encryption="aws:kms"`
- Use customer provided kms key
- `server_side_encryption="aws:kms"`
- `server_side_encryption_aws_kms_key_id="your-kms-key"`
- Use S3 managed key
- `server_side_encryption="AES256"`
- Use customer key
- `server_side_encryption_customer_algorithm="AES256"`
- `server_side_encryption_customer_key="base64-of-your-aes256-key"`
- `server_side_encryption_customer_key_md5="base64-of-your-aes256-key-md5"`
After SSE have been configured, all requests send by this backed will attach those headers.
Reference: [Protecting data using server-side encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/serv-side-encryption.html)
## Example
## Via Builder
### Basic Setup
```rust,no_run
use opendal_core::Operator;
use opendal_core::Result;
use opendal_service_s3::S3;
#[tokio::main]
async fn main() -> Result<()> {
// Create s3 backend builder.
let mut builder = S3::default()
// Set the root for s3, all operations will happen under this root.
//
// NOTE: the root must be absolute path.
.root("/path/to/dir")
// Set the bucket name. This is required.
.bucket("test")
// Set the region. This is required for some services, if you don't care about it, for example Minio service, just set it to "auto", it will be ignored.
.region("us-east-1")
// Set the endpoint.
//
// For examples:
// - "https://s3.amazonaws.com"
// - "http://127.0.0.1:9000"
// - "https://oss-ap-northeast-1.aliyuncs.com"
// - "https://cos.ap-seoul.myqcloud.com"
//
// Default to "https://s3.amazonaws.com"
.endpoint("https://s3.amazonaws.com")
// Set the access_key_id and secret_access_key.
//
// OpenDAL will try load credential from the env.
// If credential not set and no valid credential in env, OpenDAL will
// send request without signing like anonymous user.
.access_key_id("access_key_id")
.secret_access_key("secret_access_key");
let op: Operator = Operator::new(builder)?;
Ok(())
}
```
### S3 with SSE-C
```rust,no_run
use log::info;
use opendal_core::Operator;
use opendal_core::Result;
use opendal_service_s3::S3;
#[tokio::main]
async fn main() -> Result<()> {
let mut builder = S3::default()
.root("/path/to/dir")
.bucket("test")
.region("us-east-1")
.endpoint("https://s3.amazonaws.com")
.access_key_id("access_key_id")
.secret_access_key("secret_access_key")
// Enable SSE-C
.server_side_encryption_with_customer_key("AES256", "customer_key".as_bytes());
let op = Operator::new(builder)?;
info!("operator: {:?}", op);
// Writing your testing code here.
Ok(())
}
```
### S3 with SSE-KMS and aws managed kms key
```rust,no_run
use log::info;
use opendal_core::Operator;
use opendal_core::Result;
use opendal_service_s3::S3;
#[tokio::main]
async fn main() -> Result<()> {
let mut builder = S3::default()
// Setup builders
.root("/path/to/dir")
.bucket("test")
.region("us-east-1")
.endpoint("https://s3.amazonaws.com")
.access_key_id("access_key_id")
.secret_access_key("secret_access_key")
// Enable SSE-KMS with aws managed kms key
.server_side_encryption_with_aws_managed_kms_key();
let op = Operator::new(builder)?;
info!("operator: {:?}", op);
// Writing your testing code here.
Ok(())
}
```
### S3 with SSE-KMS and customer managed kms key
```rust,no_run
use log::info;
use opendal_core::Operator;
use opendal_core::Result;
use opendal_service_s3::S3;
#[tokio::main]
async fn main() -> Result<()> {
let mut builder = S3::default()
// Setup builders
.root("/path/to/dir")
.bucket("test")
.region("us-east-1")
.endpoint("https://s3.amazonaws.com")
.access_key_id("access_key_id")
.secret_access_key("secret_access_key")
// Enable SSE-KMS with customer managed kms key
.server_side_encryption_with_customer_managed_kms_key("aws_kms_key_id");
let op = Operator::new(builder)?;
info!("operator: {:?}", op);
// Writing your testing code here.
Ok(())
}
```
### S3 with SSE-S3
```rust,no_run
use log::info;
use opendal_core::Operator;
use opendal_core::Result;
use opendal_service_s3::S3;
#[tokio::main]
async fn main() -> Result<()> {
let mut builder = S3::default()
// Setup builders
.root("/path/to/dir")
.bucket("test")
.region("us-east-1")
.endpoint("https://s3.amazonaws.com")
.access_key_id("access_key_id")
.secret_access_key("secret_access_key")
// Enable SSE-S3
.server_side_encryption_with_s3_key();
let op = Operator::new(builder)?;
info!("operator: {:?}", op);
// Writing your testing code here.
Ok(())
}
```
### S3 with default ACL
```rust,no_run
use log::info;
use opendal_core::Operator;
use opendal_core::Result;
use opendal_service_s3::S3;
#[tokio::main]
async fn main() -> Result<()> {
let mut builder = S3::default()
// Setup builders
.root("/path/to/dir")
.bucket("test")
.region("us-east-1")
.endpoint("https://s3.amazonaws.com")
.access_key_id("access_key_id")
.secret_access_key("secret_access_key")
// Enable public-read ACL
.default_acl("public-read");
let op = Operator::new(builder)?;
info!("operator: {:?}", op);
// New objects will be created with public-read ACL
Ok(())
}
```
lance-artifact/third-party/opendal-service-s3/src/lib.rs
Vendored
+46
View File
@@ -0,0 +1,46 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#![doc = include_str!("../README.md")]
#![cfg_attr(docsrs, feature(doc_cfg))]
#![cfg_attr(docsrs, doc(auto_cfg))]
#![deny(missing_docs)]
mod backend;
mod config;
mod copier;
mod core;
mod deleter;
mod lister;
mod reader;
mod writer;
pub use backend::S3Builder as S3;
pub use config::S3Config;
/// URI scheme used for service registration and scheme-driven construction.
pub const S3_SCHEME: &str = "s3";
/// Register this service's URI scheme or schemes with an operator registry.
///
/// Registration enables scheme-driven construction through
/// [`opendal_core::Operator::from_uri`] and
/// [`opendal_core::Operator::via_iter`]. Direct construction through
/// [`opendal_core::Operator::new`] does not require registration.
pub fn register_s3_service(registry: &opendal_core::OperatorRegistry) {
registry.register::<S3>(S3_SCHEME);
}
lance-artifact/third-party/opendal-service-s3/src/lister.rs
Vendored
+409
View File
@@ -0,0 +1,409 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::sync::Arc;
use bytes::Buf;
use quick_xml::de;
use crate::core::parse_error;
use crate::core::*;
use opendal_core::EntryMode;
use opendal_core::Error;
use opendal_core::Metadata;
use opendal_core::OperationContext;
use opendal_core::Result;
use opendal_core::raw::oio::PageContext;
use opendal_core::raw::*;
pub type S3Listers = ThreeWays<
oio::PageLister<S3ListerV1>,
oio::PageLister<S3ListerV2>,
oio::PageLister<S3ObjectVersionsLister>,
>;
/// S3ListerV1 implements ListObjectV1 for s3 backend.
pub struct S3ListerV1 {
core: Arc<S3Core>,
ctx: OperationContext,
path: String,
args: OpList,
delimiter: &'static str,
/// marker can also be used as `start-after` for list objects v1.
/// We will use it as `start-after` for the first page and then ignore
/// it in the following pages.
first_marker: String,
}
impl S3ListerV1 {
pub fn new(core: Arc<S3Core>, ctx: OperationContext, path: &str, args: OpList) -> Self {
let delimiter = if args.recursive() { "" } else { "/" };
let first_marker = args
.start_after()
.map(|start_after| build_abs_path(&core.root, start_after))
.unwrap_or_default();
Self {
core,
ctx,
path: path.to_string(),
args,
delimiter,
first_marker,
}
}
}
impl oio::PageList for S3ListerV1 {
async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
let resp = self
.core
.s3_list_objects_v1(
&self.ctx,
&self.path,
// `marker` is used as `start-after` for the first page.
if !ctx.token.is_empty() {
&ctx.token
} else {
&self.first_marker
},
self.delimiter,
self.args.limit(),
)
.await?;
if resp.status() != http::StatusCode::OK {
return Err(parse_error(resp));
}
let bs = resp.into_body();
let output: ListObjectsOutputV1 = de::from_reader(bs.reader())
.map_err(new_xml_deserialize_error)
// Allow S3 list to retry on XML deserialization errors.
//
// This is because the S3 list API may return incomplete XML data under high load.
// We are confident that our XML decoding logic is correct. When this error occurs,
// we allow retries to obtain the correct data.
.map_err(Error::set_temporary)?;
// Try our best to check whether this list is done.
//
// - Check `is_truncated`
// - Check the length of `common_prefixes` and `contents` (very rare case)
ctx.done = if let Some(is_truncated) = output.is_truncated {
!is_truncated
} else {
output.common_prefixes.is_empty() && output.contents.is_empty()
};
// Try out best to find the next marker.
//
// - Check `next-marker`
// - Check the last object key
// - Check the last common prefix
ctx.token = if let Some(next_marker) = &output.next_marker {
next_marker.clone()
} else if let Some(content) = output.contents.last() {
content.key.clone()
} else if let Some(prefix) = output.common_prefixes.last() {
prefix.prefix.clone()
} else {
"".to_string()
};
for prefix in output.common_prefixes {
let de = oio::Entry::new(
&build_rel_path(&self.core.root, &prefix.prefix),
Metadata::new(EntryMode::DIR),
);
ctx.entries.push_back(de);
}
for object in output.contents {
let mut path = build_rel_path(&self.core.root, &object.key);
if path.is_empty() {
path = "/".to_string();
}
let mut meta = Metadata::new(EntryMode::from_path(&path));
meta.set_is_current(true);
if let Some(etag) = &object.etag {
meta.set_etag(etag);
}
meta.set_content_length(object.size);
// object.last_modified provides more precise time that contains
// nanosecond, let's trim them.
meta.set_last_modified(object.last_modified.parse::<Timestamp>()?);
let de = oio::Entry::with(path, meta);
ctx.entries.push_back(de);
}
Ok(())
}
}
/// S3ListerV2 implements ListObjectV2 for s3 backend.
pub struct S3ListerV2 {
core: Arc<S3Core>,
ctx: OperationContext,
path: String,
args: OpList,
delimiter: &'static str,
abs_start_after: Option<String>,
}
impl S3ListerV2 {
pub fn new(core: Arc<S3Core>, ctx: OperationContext, path: &str, args: OpList) -> Self {
let delimiter = if args.recursive() { "" } else { "/" };
let abs_start_after = args
.start_after()
.map(|start_after| build_abs_path(&core.root, start_after));
Self {
core,
ctx,
path: path.to_string(),
args,
delimiter,
abs_start_after,
}
}
}
impl oio::PageList for S3ListerV2 {
async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
let resp = self
.core
.s3_list_objects_v2(
&self.ctx,
&self.path,
&ctx.token,
self.delimiter,
self.args.limit(),
// start after should only be set for the first page.
if ctx.token.is_empty() {
self.abs_start_after.clone()
} else {
None
},
)
.await?;
if resp.status() != http::StatusCode::OK {
return Err(parse_error(resp));
}
let bs = resp.into_body();
let output: ListObjectsOutputV2 = de::from_reader(bs.reader())
.map_err(new_xml_deserialize_error)
// Allow S3 list to retry on XML deserialization errors.
//
// This is because the S3 list API may return incomplete XML data under high load.
// We are confident that our XML decoding logic is correct. When this error occurs,
// we allow retries to obtain the correct data.
.map_err(Error::set_temporary)?;
// Try our best to check whether this list is done.
//
// - Check `is_truncated`
// - Check `next_continuation_token`
// - Check the length of `common_prefixes` and `contents` (very rare case)
ctx.done = if let Some(is_truncated) = output.is_truncated {
!is_truncated
} else if let Some(next_continuation_token) = output.next_continuation_token.as_ref() {
next_continuation_token.is_empty()
} else {
output.common_prefixes.is_empty() && output.contents.is_empty()
};
ctx.token = output.next_continuation_token.clone().unwrap_or_default();
for prefix in output.common_prefixes {
let de = oio::Entry::new(
&build_rel_path(&self.core.root, &prefix.prefix),
Metadata::new(EntryMode::DIR),
);
ctx.entries.push_back(de);
}
for object in output.contents {
let mut path = build_rel_path(&self.core.root, &object.key);
if path.is_empty() {
path = "/".to_string();
}
let mut meta = Metadata::new(EntryMode::from_path(&path));
meta.set_is_current(true);
if let Some(etag) = &object.etag {
meta.set_etag(etag);
}
meta.set_content_length(object.size);
// object.last_modified provides more precise time that contains
// nanosecond, let's trim them.
meta.set_last_modified(object.last_modified.parse::<Timestamp>()?);
let de = oio::Entry::with(path, meta);
ctx.entries.push_back(de);
}
Ok(())
}
}
/// refer: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html
pub struct S3ObjectVersionsLister {
core: Arc<S3Core>,
ctx: OperationContext,
prefix: String,
args: OpList,
delimiter: &'static str,
abs_start_after: Option<String>,
}
impl S3ObjectVersionsLister {
pub fn new(core: Arc<S3Core>, ctx: OperationContext, path: &str, args: OpList) -> Self {
let delimiter = if args.recursive() { "" } else { "/" };
let abs_start_after = args
.start_after()
.map(|start_after| build_abs_path(&core.root, start_after));
Self {
core,
ctx,
prefix: path.to_string(),
args,
delimiter,
abs_start_after,
}
}
}
impl oio::PageList for S3ObjectVersionsLister {
async fn next_page(&self, ctx: &mut PageContext) -> Result<()> {
let markers = ctx.token.rsplit_once(" ");
let (key_marker, version_id_marker) = if let Some(data) = markers {
data
} else if let Some(start_after) = &self.abs_start_after {
(start_after.as_str(), "")
} else {
("", "")
};
let resp = self
.core
.s3_list_object_versions(
&self.ctx,
&self.prefix,
self.delimiter,
self.args.limit(),
key_marker,
version_id_marker,
)
.await?;
if resp.status() != http::StatusCode::OK {
return Err(parse_error(resp));
}
let body = resp.into_body();
let output: ListObjectVersionsOutput = de::from_reader(body.reader())
.map_err(new_xml_deserialize_error)
// Allow S3 list to retry on XML deserialization errors.
//
// This is because the S3 list API may return incomplete XML data under high load.
// We are confident that our XML decoding logic is correct. When this error occurs,
// we allow retries to obtain the correct data.
.map_err(Error::set_temporary)?;
ctx.done = if let Some(is_truncated) = output.is_truncated {
!is_truncated
} else {
false
};
ctx.token = format!(
"{} {}",
output.next_key_marker.unwrap_or_default(),
output.next_version_id_marker.unwrap_or_default()
);
for prefix in output.common_prefixes {
let de = oio::Entry::new(
&build_rel_path(&self.core.root, &prefix.prefix),
Metadata::new(EntryMode::DIR),
);
ctx.entries.push_back(de);
}
for version_object in output.version {
// `list` must be additive, so we need to include the latest version object
// even if `versions` is not enabled.
//
// Here we skip all non-latest version objects if `versions` is not enabled.
if !(self.args.versions() || version_object.is_latest) {
continue;
}
let mut path = build_rel_path(&self.core.root, &version_object.key);
if path.is_empty() {
path = "/".to_owned();
}
let mut meta = Metadata::new(EntryMode::from_path(&path));
meta.set_version(&version_object.version_id);
meta.set_is_current(version_object.is_latest);
meta.set_content_length(version_object.size);
meta.set_last_modified(version_object.last_modified.parse::<Timestamp>()?);
if let Some(etag) = version_object.etag {
meta.set_etag(&etag);
}
let entry = oio::Entry::new(&path, meta);
ctx.entries.push_back(entry);
}
if self.args.deleted() {
for delete_marker in output.delete_marker {
let mut path = build_rel_path(&self.core.root, &delete_marker.key);
if path.is_empty() {
path = "/".to_owned();
}
let mut meta = Metadata::new(EntryMode::from_path(&path));
meta.set_version(&delete_marker.version_id);
meta.set_is_deleted(true);
meta.set_is_current(delete_marker.is_latest);
meta.set_last_modified(delete_marker.last_modified.parse::<Timestamp>()?);
let entry = oio::Entry::new(&path, meta);
ctx.entries.push_back(entry);
}
}
Ok(())
}
}
lance-artifact/third-party/opendal-service-s3/src/reader.rs
Vendored
+71
View File
@@ -0,0 +1,71 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use crate::backend::*;
use crate::core::parse_error;
use http::Response;
use http::StatusCode;
use opendal_core::raw::*;
use opendal_core::*;
/// Reader returned by this backend.
pub struct S3Reader {
backend: S3Backend,
// StreamRead::open can run after Service::read returns, so keep the
// per-operation context with the reader.
ctx: OperationContext,
path: String,
args: OpRead,
}
impl S3Reader {
pub(super) fn new(backend: S3Backend, ctx: OperationContext, path: &str, args: OpRead) -> Self {
Self {
backend,
ctx,
path: path.to_string(),
args,
}
}
}
impl oio::StreamRead for S3Reader {
async fn open(&self, range: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
let backend = &self.backend;
let path = self.path.as_str();
let args = self.args.clone();
let resp = backend
.core
.s3_get_object(&self.ctx, path, range, &args)
.await?;
let status = resp.status();
let (rp, stream) = match status {
StatusCode::OK | StatusCode::PARTIAL_CONTENT => (
RpRead::new(parse_into_metadata(path, resp.headers())?),
resp.into_body(),
),
_ => {
let (part, mut body) = resp.into_parts();
let buf = body.to_buffer().await?;
return Err(parse_error(Response::from_parts(part, buf)));
}
};
Ok((rp, Box::new(stream) as Box<dyn oio::ReadStreamDyn>))
}
}
lance-artifact/third-party/opendal-service-s3/src/writer.rs
Vendored
+264
View File
@@ -0,0 +1,264 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::sync::Arc;
use bytes::Buf;
use constants::X_AMZ_OBJECT_SIZE;
use constants::X_AMZ_VERSION_ID;
use http::StatusCode;
use crate::core::S3Error;
use crate::core::from_s3_error;
use crate::core::parse_error;
use crate::core::*;
use opendal_core::raw::*;
use opendal_core::*;
pub type S3Writers = TwoWays<oio::MultipartWriter<S3Writer>, oio::AppendWriter<S3Writer>>;
pub struct S3Writer {
core: Arc<S3Core>,
ctx: OperationContext,
op: OpWrite,
path: String,
}
impl S3Writer {
pub fn new(core: Arc<S3Core>, ctx: OperationContext, path: &str, op: OpWrite) -> Self {
S3Writer {
core,
ctx,
path: path.to_string(),
op,
}
}
fn parse_header_into_meta(path: &str, headers: &http::HeaderMap) -> Result<Metadata> {
let mut meta = Metadata::new(EntryMode::from_path(path));
if let Some(etag) = parse_etag(headers)? {
meta.set_etag(etag);
}
if let Some(version) = parse_header_to_str(headers, X_AMZ_VERSION_ID)? {
meta.set_version(version);
}
if let Some(value) =
parse_header_to_str(headers, X_AMZ_OBJECT_SIZE)?.and_then(|size| size.parse().ok())
{
meta.set_content_length(value);
}
Ok(meta)
}
}
impl oio::MultipartWrite for S3Writer {
async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
let req = self
.core
.s3_put_object_request(&self.path, Some(size), &self.op, body)?;
let resp = self.core.send(&self.ctx, req).await?;
let status = resp.status();
let meta = S3Writer::parse_header_into_meta(&self.path, resp.headers())?;
match status {
StatusCode::CREATED | StatusCode::OK => Ok(meta),
_ => Err(parse_error(resp)),
}
}
async fn initiate_part(&self) -> Result<String> {
let resp = self
.core
.s3_initiate_multipart_upload(&self.ctx, &self.path, &self.op)
.await?;
let status = resp.status();
match status {
StatusCode::OK => {
let bs = resp.into_body();
let result: InitiateMultipartUploadResult =
quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
Ok(result.upload_id)
}
_ => Err(parse_error(resp)),
}
}
async fn write_part(
&self,
upload_id: &str,
part_number: usize,
size: u64,
body: Buffer,
) -> Result<oio::MultipartPart> {
// AWS S3 requires part number must between [1..=10000]
let part_number = part_number + 1;
let checksum = self.core.calculate_checksum(&body);
let req = self.core.s3_upload_part_request(
&self.path,
upload_id,
part_number,
size,
body,
checksum.clone(),
)?;
let resp = self.core.send(&self.ctx, req).await?;
let status = resp.status();
match status {
StatusCode::OK => {
let etag = parse_etag(resp.headers())?
.ok_or_else(|| {
Error::new(
ErrorKind::Unexpected,
"ETag not present in returning response",
)
})?
.to_string();
Ok(oio::MultipartPart {
part_number,
etag,
checksum,
size: None,
})
}
_ => Err(parse_error(resp)),
}
}
async fn complete_part(
&self,
upload_id: &str,
parts: &[oio::MultipartPart],
) -> Result<Metadata> {
let parts = parts
.iter()
.map(|p| match &self.core.checksum_algorithm {
None => CompleteMultipartUploadRequestPart {
part_number: p.part_number,
etag: p.etag.clone(),
..Default::default()
},
Some(checksum_algorithm) => match checksum_algorithm {
ChecksumAlgorithm::Crc32c => CompleteMultipartUploadRequestPart {
part_number: p.part_number,
etag: p.etag.clone(),
checksum_crc32c: p.checksum.clone(),
},
ChecksumAlgorithm::Md5 => CompleteMultipartUploadRequestPart {
part_number: p.part_number,
etag: p.etag.clone(),
..Default::default()
},
},
})
.collect();
let resp = self
.core
.s3_complete_multipart_upload(&self.ctx, &self.path, upload_id, parts, &self.op)
.await?;
let status = resp.status();
let mut meta = S3Writer::parse_header_into_meta(&self.path, resp.headers())?;
match status {
StatusCode::OK => {
// still check if there is any error because S3 might return error for status code 200
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html#API_CompleteMultipartUpload_Example_4
let (parts, body) = resp.into_parts();
let ret: CompleteMultipartUploadResult =
quick_xml::de::from_reader(body.reader()).map_err(new_xml_deserialize_error)?;
if !ret.code.is_empty() {
return Err(from_s3_error(
S3Error {
code: ret.code,
message: ret.message,
resource: "".to_string(),
request_id: ret.request_id,
},
parts,
));
}
meta.set_etag(&ret.etag);
Ok(meta)
}
_ => Err(parse_error(resp)),
}
}
async fn abort_part(&self, upload_id: &str) -> Result<()> {
let resp = self
.core
.s3_abort_multipart_upload(&self.ctx, &self.path, upload_id)
.await?;
match resp.status() {
// s3 returns code 204 if abort succeeds.
StatusCode::NO_CONTENT => Ok(()),
_ => Err(parse_error(resp)),
}
}
}
impl oio::AppendWrite for S3Writer {
async fn offset(&self) -> Result<u64> {
let resp = self
.core
.s3_head_object(&self.ctx, &self.path, OpStat::default())
.await?;
let status = resp.status();
match status {
StatusCode::OK => Ok(parse_content_length(resp.headers())?.unwrap_or_default()),
StatusCode::NOT_FOUND => Ok(0),
_ => Err(parse_error(resp)),
}
}
async fn append(&self, offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
let req = self
.core
.s3_append_object_request(&self.path, offset, size, &self.op, body)?;
let resp = self.core.send(&self.ctx, req).await?;
let status = resp.status();
let meta = S3Writer::parse_header_into_meta(&self.path, resp.headers())?;
match status {
StatusCode::CREATED | StatusCode::OK => Ok(meta),
_ => Err(parse_error(resp)),
}
}
}
lance-artifact/third-party/opendal/.cargo_vcs_info.json
Vendored
+6
View File
@@ -0,0 +1,6 @@
{
"git": {
"sha1": "6f544a131be78e553d28e79f3cfe5ac948f2963e"
},
"path_in_vcs": "core"
}
lance-artifact/third-party/opendal/CHANGELOG.md
Vendored
+5834
View File
File diff suppressed because it is too large Load Diff
lance-artifact/third-party/opendal/CONTRIBUTING.md
Vendored
+61
View File
@@ -0,0 +1,61 @@
# Contributing
## Get Started
- `cargo check` to analyze the current package and report errors.
- `cargo build` to compile the current package.
- `cargo clippy` to catch common mistakes and improve code.
- `cargo test --features tests` to run unit tests.
- `cargo test` to run unit tests include doc tests.
- `cargo bench` to run benchmark tests.
Useful tips:
- Check/Build/Test/Clippy all code: `cargo <cmd> --tests --benches --examples`
- Test specific function: `cargo test tests::it::services::fs`
## Tests
We have unit tests and behavior tests.
### Unit Tests
Unit tests are placed under `src/tests`, organized by mod.
To run unit tests:
```shell
cargo test --features tests
```
To run unit tests include doc tests:
```shell
cargo test
```
### Behavior Tests
Behavior Tests are used to make sure every service works correctly.
```shell
# Setup env
cp .env.example .env
# Run tests
cargo test
```
Please visit [Behavior Test README](./tests/behavior/README.md) for more details.
## Benchmark
We use Ops Benchmark Tests to measure every operation's performance on the target platform.
```shell
# Setup env
cp .env.example .env
# Run benches
cargo bench
```
Please visit [Ops Benchmark README](./benches/ops/README.md) for more details.
lance-artifact/third-party/opendal/Cargo.lock
Generated Vendored
+12425
View File
File diff suppressed because it is too large Load Diff
lance-artifact/third-party/opendal/Cargo.toml
Vendored
+711
View File
@@ -0,0 +1,711 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2024"
rust-version = "1.91"
name = "opendal"
version = "0.58.1"
authors = ["Apache OpenDAL <dev@opendal.apache.org>"]
build = false
exclude = ["/tests/"]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "Apache OpenDAL™: One Layer, All Storage."
homepage = "https://opendal.apache.org/"
readme = "README.md"
keywords = [
"storage",
"fs",
"s3",
"azblob",
"gcs",
]
categories = ["filesystem"]
license = "Apache-2.0"
repository = "https://github.com/apache/opendal"
[package.metadata.docs.rs]
all-features = true
[features]
auto-register-services = ["dep:ctor"]
blocking = ["opendal-core/blocking"]
default = [
"auto-register-services",
"http-transport-reqwest",
"executors-tokio",
"layers-concurrent-limit",
"layers-logging",
"layers-retry",
"layers-timeout",
]
executors-tokio = ["opendal-core/executors-tokio"]
http-transport-reqwest = ["http-transport-reqwest-rustls"]
http-transport-reqwest-native-tls = [
"dep:opendal-http-transport-reqwest",
"opendal-http-transport-reqwest/native-tls",
]
http-transport-reqwest-rustls = [
"dep:opendal-http-transport-reqwest",
"opendal-http-transport-reqwest/rustls",
]
http-transport-reqwest-rustls-no-provider = [
"dep:opendal-http-transport-reqwest",
"opendal-http-transport-reqwest/rustls-no-provider",
]
internal-tokio-rt = ["opendal-core/internal-tokio-rt"]
layers-async-backtrace = ["dep:opendal-layer-async-backtrace"]
layers-await-tree = ["dep:opendal-layer-await-tree"]
layers-capability-check = ["dep:opendal-layer-capability-check"]
layers-chaos = ["dep:opendal-layer-chaos"]
layers-concurrent-limit = ["dep:opendal-layer-concurrent-limit"]
layers-dtrace = ["dep:opendal-layer-dtrace"]
layers-fastmetrics = ["dep:opendal-layer-fastmetrics"]
layers-fastrace = ["dep:opendal-layer-fastrace"]
layers-foyer = ["dep:opendal-layer-foyer"]
layers-hotpath = ["dep:opendal-layer-hotpath"]
layers-immutable-index = ["dep:opendal-layer-immutable-index"]
layers-logging = ["dep:opendal-layer-logging"]
layers-metrics = ["dep:opendal-layer-metrics"]
layers-mime-guess = ["dep:opendal-layer-mime-guess"]
layers-otel-metrics = ["dep:opendal-layer-otelmetrics"]
layers-otel-trace = ["dep:opendal-layer-oteltrace"]
layers-prometheus = ["dep:opendal-layer-prometheus"]
layers-prometheus-client = ["dep:opendal-layer-prometheus-client"]
layers-retry = ["dep:opendal-layer-retry"]
layers-route = ["dep:opendal-layer-route"]
layers-tail-cut = ["dep:opendal-layer-tail-cut"]
layers-throttle = ["dep:opendal-layer-throttle"]
layers-timeout = ["dep:opendal-layer-timeout"]
layers-tracing = ["dep:opendal-layer-tracing"]
reqwest-rustls-no-provider-tls = [
"http-transport-reqwest",
"http-transport-reqwest-rustls-no-provider",
]
reqwest-rustls-tls = [
"http-transport-reqwest",
"http-transport-reqwest-rustls",
]
services-aliyun-drive = ["dep:opendal-service-aliyun-drive"]
services-alluxio = ["dep:opendal-service-alluxio"]
services-azblob = ["dep:opendal-service-azblob"]
services-azdls = ["dep:opendal-service-azdls"]
services-azfile = ["dep:opendal-service-azfile"]
services-b2 = ["dep:opendal-service-b2"]
services-cacache = ["dep:opendal-service-cacache"]
services-cloudflare-kv = ["dep:opendal-service-cloudflare-kv"]
services-compfs = ["dep:opendal-service-compfs"]
services-cos = ["dep:opendal-service-cos"]
services-d1 = ["dep:opendal-service-d1"]
services-dashmap = ["dep:opendal-service-dashmap"]
services-dbfs = ["dep:opendal-service-dbfs"]
services-dropbox = ["dep:opendal-service-dropbox"]
services-etcd = ["dep:opendal-service-etcd"]
services-foundationdb = ["dep:opendal-service-foundationdb"]
services-foyer = ["dep:opendal-service-foyer"]
services-fs = ["dep:opendal-service-fs"]
services-ftp = ["dep:opendal-service-ftp"]
services-gcs = ["dep:opendal-service-gcs"]
services-gdrive = ["dep:opendal-service-gdrive"]
services-ghac = ["dep:opendal-service-ghac"]
services-github = ["dep:opendal-service-github"]
services-goosefs = ["dep:opendal-service-goosefs"]
services-gridfs = ["dep:opendal-service-gridfs"]
services-hdfs = ["dep:opendal-service-hdfs"]
services-hdfs-native = ["dep:opendal-service-hdfs-native"]
services-hf = ["dep:opendal-service-hf"]
services-http = ["dep:opendal-service-http"]
services-huggingface = ["services-hf"]
services-ipfs = ["dep:opendal-service-ipfs"]
services-ipmfs = ["dep:opendal-service-ipmfs"]
services-koofr = ["dep:opendal-service-koofr"]
services-lakefs = ["dep:opendal-service-lakefs"]
services-memcached = ["dep:opendal-service-memcached"]
services-memory = ["opendal-core/services-memory"]
services-mini-moka = ["dep:opendal-service-mini-moka"]
services-moka = ["dep:opendal-service-moka"]
services-mongodb = ["dep:opendal-service-mongodb"]
services-monoiofs = ["dep:opendal-service-monoiofs"]
services-mysql = ["dep:opendal-service-mysql"]
services-obs = ["dep:opendal-service-obs"]
services-onedrive = ["dep:opendal-service-onedrive"]
services-opfs = ["dep:opendal-service-opfs"]
services-oss = ["dep:opendal-service-oss"]
services-pcloud = ["dep:opendal-service-pcloud"]
services-persy = ["dep:opendal-service-persy"]
services-postgresql = ["dep:opendal-service-postgresql"]
services-redb = ["dep:opendal-service-redb"]
services-redis = [
"dep:opendal-service-redis",
"opendal-service-redis?/rustls",
]
services-redis-native-tls = [
"dep:opendal-service-redis",
"opendal-service-redis?/native-tls",
]
services-rocksdb = ["dep:opendal-service-rocksdb"]
services-s3 = ["dep:opendal-service-s3"]
services-seafile = ["dep:opendal-service-seafile"]
services-sftp = ["dep:opendal-service-sftp"]
services-sled = ["dep:opendal-service-sled"]
services-sqlite = ["dep:opendal-service-sqlite"]
services-surrealdb = ["dep:opendal-service-surrealdb"]
services-swift = ["dep:opendal-service-swift"]
services-tikv = ["dep:opendal-service-tikv"]
services-tos = ["dep:opendal-service-tos"]
services-upyun = ["dep:opendal-service-upyun"]
services-vercel-artifacts = ["dep:opendal-service-vercel-artifacts"]
services-vercel-blob = ["dep:opendal-service-vercel-blob"]
services-webdav = ["dep:opendal-service-webdav"]
services-webhdfs = ["dep:opendal-service-webhdfs"]
services-yandex-disk = ["dep:opendal-service-yandex-disk"]
tests = ["opendal-testkit"]
[lib]
name = "opendal"
path = "src/lib.rs"
bench = false
[[bench]]
name = "ops"
path = "benches/ops/main.rs"
harness = false
required-features = ["tests"]
[[bench]]
name = "types"
path = "benches/types/main.rs"
harness = false
[dependencies.ctor]
version = "1.0.7"
optional = true
[dependencies.opendal-core]
version = "0.58.1"
default-features = false
[dependencies.opendal-http-transport-reqwest]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-async-backtrace]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-await-tree]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-capability-check]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-chaos]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-concurrent-limit]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-fastmetrics]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-fastrace]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-foyer]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-hotpath]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-immutable-index]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-logging]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-metrics]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-mime-guess]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-observe-metrics-common]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-otelmetrics]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-oteltrace]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-prometheus]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-prometheus-client]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-retry]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-route]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-tail-cut]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-throttle]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-timeout]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-layer-tracing]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-aliyun-drive]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-alluxio]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-azblob]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-azdls]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-azfile]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-b2]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-cacache]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-cloudflare-kv]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-compfs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-cos]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-d1]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-dashmap]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-dbfs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-dropbox]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-etcd]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-foundationdb]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-foyer]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-fs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-ftp]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-gcs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-gdrive]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-ghac]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-github]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-goosefs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-gridfs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-hdfs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-hdfs-native]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-hf]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-http]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-ipfs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-ipmfs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-koofr]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-lakefs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-memcached]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-mini-moka]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-moka]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-mongodb]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-monoiofs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-mysql]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-obs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-onedrive]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-oss]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-pcloud]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-persy]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-postgresql]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-redb]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-redis]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-rocksdb]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-s3]
version = "0.58.1"
path = "../opendal-service-s3"
optional = true
default-features = false
[dependencies.opendal-service-seafile]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-sftp]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-sled]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-sqlite]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-surrealdb]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-swift]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-tikv]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-tos]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-upyun]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-vercel-artifacts]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-vercel-blob]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-webdav]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-webhdfs]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-service-yandex-disk]
version = "0.58.1"
optional = true
default-features = false
[dependencies.opendal-testkit]
version = "0.58.1"
optional = true
[dev-dependencies.anyhow]
version = "1.0"
[dev-dependencies.bytes]
version = "1.10"
[dev-dependencies.divan]
version = "0.1"
[dev-dependencies.futures]
version = "0.3"
default-features = true
[dev-dependencies.http]
version = "1.4"
[dev-dependencies.libtest-mimic]
version = "0.8"
[dev-dependencies.log]
version = "0.4.32"
[dev-dependencies.logforth]
version = "0.30.1"
features = ["starter-log"]
[dev-dependencies.rand]
version = "0.10.1"
[dev-dependencies.reqwest]
version = "0.13.2"
[dev-dependencies.sha2]
version = "0.11.0"
[dev-dependencies.size]
version = "0.5"
[dev-dependencies.tokio]
version = "1.52"
features = [
"fs",
"macros",
"rt-multi-thread",
]
default-features = false
[dev-dependencies.uuid]
version = "1"
features = [
"serde",
"v4",
]
default-features = false
[target.'cfg(target_arch = "wasm32")'.dependencies.opendal-service-opfs]
version = "0.58.1"
optional = true
default-features = false
[target.'cfg(target_os = "linux")'.dependencies.opendal-layer-dtrace]
version = "0.58.1"
optional = true
default-features = false
[lints.clippy]
unused_async = "warn"
lance-artifact/third-party/opendal/Cargo.toml.orig
Generated Vendored
+355
View File
@@ -0,0 +1,355 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[workspace]
default-members = [".", "core"]
members = [
".",
"benches/vs_*",
"core",
"edge/*",
"examples/*",
"fuzz",
"http-transports/*",
"layers/*",
"services/*",
"testkit",
]
[workspace.package]
authors = ["Apache OpenDAL <dev@opendal.apache.org>"]
categories = ["filesystem"]
description = "Apache OpenDAL™: One Layer, All Storage."
edition = "2024"
homepage = "https://opendal.apache.org/"
license = "Apache-2.0"
repository = "https://github.com/apache/opendal"
rust-version = "1.91"
version = "0.58.1"
[workspace.dependencies]
base64 = "0.23"
bytes = "1.10"
ctor = "1.0.7"
futures = { version = "0.3", default-features = false }
http = "1.4"
log = { version = "0.4.32" }
logforth = { version = "0.30.1", features = ["starter-log"] }
mea = "0.6"
quick-xml = { version = "0.41.0", default-features = false }
rand = "0.10.1"
serde = { version = "1", default-features = false }
serde_json = "1"
sha2 = "0.11.0"
tokio = { version = "1.52", default-features = false }
url = "2.5"
uuid = { version = "1", default-features = false }
[package]
exclude = ["/tests/"]
keywords = ["storage", "fs", "s3", "azblob", "gcs"]
name = "opendal"
readme = "README.md"
authors = { workspace = true }
categories = { workspace = true }
description = { workspace = true }
edition = { workspace = true }
homepage = { workspace = true }
license = { workspace = true }
repository = { workspace = true }
rust-version = { workspace = true }
version = { workspace = true }
[lints.clippy]
unused_async = "warn"
[package.metadata.docs.rs]
all-features = true
[features]
# Install facade defaults before `main`: register enabled services and, when
# configured, install the default HTTP transport. Disable this feature when the
# application needs to install its own process-wide HTTP transport.
auto-register-services = ["dep:ctor"]
blocking = ["opendal-core/blocking"]
default = [
"auto-register-services",
"http-transport-reqwest",
"executors-tokio",
"layers-concurrent-limit",
"layers-logging",
"layers-retry",
"layers-timeout",
]
executors-tokio = ["opendal-core/executors-tokio"]
# Default http-transport-reqwest transport feature, used for:
# - install default http transport
# - feature gate for various of bindings dependency
http-transport-reqwest = ["http-transport-reqwest-rustls"]
# Use reqwest with platform TLS via native-tls.
http-transport-reqwest-native-tls = [
"dep:opendal-http-transport-reqwest",
"opendal-http-transport-reqwest/native-tls",
]
# Use reqwest with Rustls, with default aws-lc-rs crypto provider.
http-transport-reqwest-rustls = [
"dep:opendal-http-transport-reqwest",
"opendal-http-transport-reqwest/rustls",
]
# Use reqwest with Rustls and no built-in crypto provider.
http-transport-reqwest-rustls-no-provider = [
"dep:opendal-http-transport-reqwest",
"opendal-http-transport-reqwest/rustls-no-provider",
]
internal-tokio-rt = ["opendal-core/internal-tokio-rt"]
layers-async-backtrace = ["dep:opendal-layer-async-backtrace"]
layers-await-tree = ["dep:opendal-layer-await-tree"]
layers-capability-check = ["dep:opendal-layer-capability-check"]
layers-chaos = ["dep:opendal-layer-chaos"]
layers-concurrent-limit = ["dep:opendal-layer-concurrent-limit"]
layers-dtrace = ["dep:opendal-layer-dtrace"]
layers-fastmetrics = ["dep:opendal-layer-fastmetrics"]
layers-fastrace = ["dep:opendal-layer-fastrace"]
layers-foyer = ["dep:opendal-layer-foyer"]
layers-hotpath = ["dep:opendal-layer-hotpath"]
layers-immutable-index = ["dep:opendal-layer-immutable-index"]
layers-logging = ["dep:opendal-layer-logging"]
layers-metrics = ["dep:opendal-layer-metrics"]
layers-mime-guess = ["dep:opendal-layer-mime-guess"]
layers-otel-metrics = ["dep:opendal-layer-otelmetrics"]
layers-otel-trace = ["dep:opendal-layer-oteltrace"]
layers-prometheus = ["dep:opendal-layer-prometheus"]
layers-prometheus-client = ["dep:opendal-layer-prometheus-client"]
layers-retry = ["dep:opendal-layer-retry"]
layers-route = ["dep:opendal-layer-route"]
layers-tail-cut = ["dep:opendal-layer-tail-cut"]
layers-throttle = ["dep:opendal-layer-throttle"]
layers-timeout = ["dep:opendal-layer-timeout"]
layers-tracing = ["dep:opendal-layer-tracing"]
# Deprecated since 0.58; Will delete in 0.59.
reqwest-rustls-no-provider-tls = [
"http-transport-reqwest",
"http-transport-reqwest-rustls-no-provider",
]
# Deprecated since 0.58; Will delete in 0.59.
reqwest-rustls-tls = ["http-transport-reqwest", "http-transport-reqwest-rustls"]
services-aliyun-drive = ["dep:opendal-service-aliyun-drive"]
services-alluxio = ["dep:opendal-service-alluxio"]
services-azblob = ["dep:opendal-service-azblob"]
services-azdls = ["dep:opendal-service-azdls"]
services-azfile = ["dep:opendal-service-azfile"]
services-b2 = ["dep:opendal-service-b2"]
services-cacache = ["dep:opendal-service-cacache"]
services-cloudflare-kv = ["dep:opendal-service-cloudflare-kv"]
services-compfs = ["dep:opendal-service-compfs"]
services-cos = ["dep:opendal-service-cos"]
services-d1 = ["dep:opendal-service-d1"]
services-dashmap = ["dep:opendal-service-dashmap"]
services-dbfs = ["dep:opendal-service-dbfs"]
services-dropbox = ["dep:opendal-service-dropbox"]
services-etcd = ["dep:opendal-service-etcd"]
services-foundationdb = ["dep:opendal-service-foundationdb"]
services-foyer = ["dep:opendal-service-foyer"]
services-fs = ["dep:opendal-service-fs"]
services-ftp = ["dep:opendal-service-ftp"]
services-gcs = ["dep:opendal-service-gcs"]
services-gdrive = ["dep:opendal-service-gdrive"]
services-ghac = ["dep:opendal-service-ghac"]
services-github = ["dep:opendal-service-github"]
services-goosefs = ["dep:opendal-service-goosefs"]
services-gridfs = ["dep:opendal-service-gridfs"]
services-hdfs = ["dep:opendal-service-hdfs"]
services-hdfs-native = ["dep:opendal-service-hdfs-native"]
services-hf = ["dep:opendal-service-hf"]
services-http = ["dep:opendal-service-http"]
services-huggingface = ["services-hf"]
services-ipfs = ["dep:opendal-service-ipfs"]
services-ipmfs = ["dep:opendal-service-ipmfs"]
services-koofr = ["dep:opendal-service-koofr"]
services-lakefs = ["dep:opendal-service-lakefs"]
services-memcached = ["dep:opendal-service-memcached"]
# Deprecated: memory service is always enabled.
services-memory = ["opendal-core/services-memory"]
services-mini-moka = ["dep:opendal-service-mini-moka"]
services-moka = ["dep:opendal-service-moka"]
services-mongodb = ["dep:opendal-service-mongodb"]
services-monoiofs = ["dep:opendal-service-monoiofs"]
services-mysql = ["dep:opendal-service-mysql"]
services-obs = ["dep:opendal-service-obs"]
services-onedrive = ["dep:opendal-service-onedrive"]
services-opfs = ["dep:opendal-service-opfs"]
services-oss = ["dep:opendal-service-oss"]
services-pcloud = ["dep:opendal-service-pcloud"]
services-persy = ["dep:opendal-service-persy"]
services-postgresql = ["dep:opendal-service-postgresql"]
services-redb = ["dep:opendal-service-redb"]
services-redis = ["dep:opendal-service-redis", "opendal-service-redis?/rustls"]
services-redis-native-tls = [
"dep:opendal-service-redis",
"opendal-service-redis?/native-tls",
]
services-rocksdb = ["dep:opendal-service-rocksdb"]
services-s3 = ["dep:opendal-service-s3"]
services-seafile = ["dep:opendal-service-seafile"]
services-sftp = ["dep:opendal-service-sftp"]
services-sled = ["dep:opendal-service-sled"]
services-sqlite = ["dep:opendal-service-sqlite"]
services-surrealdb = ["dep:opendal-service-surrealdb"]
services-swift = ["dep:opendal-service-swift"]
services-tikv = ["dep:opendal-service-tikv"]
services-tos = ["dep:opendal-service-tos"]
services-upyun = ["dep:opendal-service-upyun"]
services-vercel-artifacts = ["dep:opendal-service-vercel-artifacts"]
services-vercel-blob = ["dep:opendal-service-vercel-blob"]
services-webdav = ["dep:opendal-service-webdav"]
services-webhdfs = ["dep:opendal-service-webhdfs"]
services-yandex-disk = ["dep:opendal-service-yandex-disk"]
tests = ["opendal-testkit"]
[lib]
bench = false
[[bench]]
harness = false
name = "ops"
required-features = ["tests"]
[[bench]]
harness = false
name = "types"
[[test]]
harness = false
name = "behavior"
path = "tests/behavior/main.rs"
required-features = ["tests"]
[dependencies]
ctor = { workspace = true, optional = true }
opendal-core = { path = "core", version = "0.58.1", default-features = false }
opendal-http-transport-reqwest = { path = "http-transports/reqwest", version = "0.58.1", optional = true, default-features = false }
opendal-layer-async-backtrace = { path = "layers/async-backtrace", version = "0.58.1", optional = true, default-features = false }
opendal-layer-await-tree = { path = "layers/await-tree", version = "0.58.1", optional = true, default-features = false }
opendal-layer-capability-check = { path = "layers/capability-check", version = "0.58.1", optional = true, default-features = false }
opendal-layer-chaos = { path = "layers/chaos", version = "0.58.1", optional = true, default-features = false }
opendal-layer-concurrent-limit = { path = "layers/concurrent-limit", version = "0.58.1", optional = true, default-features = false }
opendal-layer-fastmetrics = { path = "layers/fastmetrics", version = "0.58.1", optional = true, default-features = false }
opendal-layer-fastrace = { path = "layers/fastrace", version = "0.58.1", optional = true, default-features = false }
opendal-layer-foyer = { path = "layers/foyer", version = "0.58.1", optional = true, default-features = false }
opendal-layer-hotpath = { path = "layers/hotpath", version = "0.58.1", optional = true, default-features = false }
opendal-layer-immutable-index = { path = "layers/immutable-index", version = "0.58.1", optional = true, default-features = false }
opendal-layer-logging = { path = "layers/logging", version = "0.58.1", optional = true, default-features = false }
opendal-layer-metrics = { path = "layers/metrics", version = "0.58.1", optional = true, default-features = false }
opendal-layer-mime-guess = { path = "layers/mime-guess", version = "0.58.1", optional = true, default-features = false }
opendal-layer-observe-metrics-common = { path = "layers/observe-metrics-common", version = "0.58.1", optional = true, default-features = false }
opendal-layer-otelmetrics = { path = "layers/otelmetrics", version = "0.58.1", optional = true, default-features = false }
opendal-layer-oteltrace = { path = "layers/oteltrace", version = "0.58.1", optional = true, default-features = false }
opendal-layer-prometheus = { path = "layers/prometheus", version = "0.58.1", optional = true, default-features = false }
opendal-layer-prometheus-client = { path = "layers/prometheus-client", version = "0.58.1", optional = true, default-features = false }
opendal-layer-retry = { path = "layers/retry", version = "0.58.1", optional = true, default-features = false }
opendal-layer-route = { path = "layers/route", version = "0.58.1", optional = true, default-features = false }
opendal-layer-tail-cut = { path = "layers/tail-cut", version = "0.58.1", optional = true, default-features = false }
opendal-layer-throttle = { path = "layers/throttle", version = "0.58.1", optional = true, default-features = false }
opendal-layer-timeout = { path = "layers/timeout", version = "0.58.1", optional = true, default-features = false }
opendal-layer-tracing = { path = "layers/tracing", version = "0.58.1", optional = true, default-features = false }
opendal-service-aliyun-drive = { path = "services/aliyun-drive", version = "0.58.1", optional = true, default-features = false }
opendal-service-alluxio = { path = "services/alluxio", version = "0.58.1", optional = true, default-features = false }
opendal-service-azblob = { path = "services/azblob", version = "0.58.1", optional = true, default-features = false }
opendal-service-azdls = { path = "services/azdls", version = "0.58.1", optional = true, default-features = false }
opendal-service-azfile = { path = "services/azfile", version = "0.58.1", optional = true, default-features = false }
opendal-service-b2 = { path = "services/b2", version = "0.58.1", optional = true, default-features = false }
opendal-service-cacache = { path = "services/cacache", version = "0.58.1", optional = true, default-features = false }
opendal-service-cloudflare-kv = { path = "services/cloudflare-kv", version = "0.58.1", optional = true, default-features = false }
opendal-service-compfs = { path = "services/compfs", version = "0.58.1", optional = true, default-features = false }
opendal-service-cos = { path = "services/cos", version = "0.58.1", optional = true, default-features = false }
opendal-service-d1 = { path = "services/d1", version = "0.58.1", optional = true, default-features = false }
opendal-service-dashmap = { path = "services/dashmap", version = "0.58.1", optional = true, default-features = false }
opendal-service-dbfs = { path = "services/dbfs", version = "0.58.1", optional = true, default-features = false }
opendal-service-dropbox = { path = "services/dropbox", version = "0.58.1", optional = true, default-features = false }
opendal-service-etcd = { path = "services/etcd", version = "0.58.1", optional = true, default-features = false }
opendal-service-foundationdb = { path = "services/foundationdb", version = "0.58.1", optional = true, default-features = false }
opendal-service-foyer = { path = "services/foyer", version = "0.58.1", optional = true, default-features = false }
opendal-service-fs = { path = "services/fs", version = "0.58.1", optional = true, default-features = false }
opendal-service-ftp = { path = "services/ftp", version = "0.58.1", optional = true, default-features = false }
opendal-service-gcs = { path = "services/gcs", version = "0.58.1", optional = true, default-features = false }
opendal-service-gdrive = { path = "services/gdrive", version = "0.58.1", optional = true, default-features = false }
opendal-service-ghac = { path = "services/ghac", version = "0.58.1", optional = true, default-features = false }
opendal-service-github = { path = "services/github", version = "0.58.1", optional = true, default-features = false }
opendal-service-goosefs = { path = "services/goosefs", version = "0.58.1", optional = true, default-features = false }
opendal-service-gridfs = { path = "services/gridfs", version = "0.58.1", optional = true, default-features = false }
opendal-service-hdfs = { path = "services/hdfs", version = "0.58.1", optional = true, default-features = false }
opendal-service-hdfs-native = { path = "services/hdfs-native", version = "0.58.1", optional = true, default-features = false }
opendal-service-hf = { path = "services/hf", version = "0.58.1", optional = true, default-features = false }
opendal-service-http = { path = "services/http", version = "0.58.1", optional = true, default-features = false }
opendal-service-ipfs = { path = "services/ipfs", version = "0.58.1", optional = true, default-features = false }
opendal-service-ipmfs = { path = "services/ipmfs", version = "0.58.1", optional = true, default-features = false }
opendal-service-koofr = { path = "services/koofr", version = "0.58.1", optional = true, default-features = false }
opendal-service-lakefs = { path = "services/lakefs", version = "0.58.1", optional = true, default-features = false }
opendal-service-memcached = { path = "services/memcached", version = "0.58.1", optional = true, default-features = false }
opendal-service-mini-moka = { path = "services/mini_moka", version = "0.58.1", optional = true, default-features = false }
opendal-service-moka = { path = "services/moka", version = "0.58.1", optional = true, default-features = false }
opendal-service-mongodb = { path = "services/mongodb", version = "0.58.1", optional = true, default-features = false }
opendal-service-monoiofs = { path = "services/monoiofs", version = "0.58.1", optional = true, default-features = false }
opendal-service-mysql = { path = "services/mysql", version = "0.58.1", optional = true, default-features = false }
opendal-service-obs = { path = "services/obs", version = "0.58.1", optional = true, default-features = false }
opendal-service-onedrive = { path = "services/onedrive", version = "0.58.1", optional = true, default-features = false }
opendal-service-oss = { path = "services/oss", version = "0.58.1", optional = true, default-features = false }
opendal-service-pcloud = { path = "services/pcloud", version = "0.58.1", optional = true, default-features = false }
opendal-service-persy = { path = "services/persy", version = "0.58.1", optional = true, default-features = false }
opendal-service-postgresql = { path = "services/postgresql", version = "0.58.1", optional = true, default-features = false }
opendal-service-redb = { path = "services/redb", version = "0.58.1", optional = true, default-features = false }
opendal-service-redis = { path = "services/redis", version = "0.58.1", optional = true, default-features = false }
opendal-service-rocksdb = { path = "services/rocksdb", version = "0.58.1", optional = true, default-features = false }
opendal-service-s3 = { path = "services/s3", version = "0.58.1", optional = true, default-features = false }
opendal-service-seafile = { path = "services/seafile", version = "0.58.1", optional = true, default-features = false }
opendal-service-sftp = { path = "services/sftp", version = "0.58.1", optional = true, default-features = false }
opendal-service-sled = { path = "services/sled", version = "0.58.1", optional = true, default-features = false }
opendal-service-sqlite = { path = "services/sqlite", version = "0.58.1", optional = true, default-features = false }
opendal-service-surrealdb = { path = "services/surrealdb", version = "0.58.1", optional = true, default-features = false }
opendal-service-swift = { path = "services/swift", version = "0.58.1", optional = true, default-features = false }
opendal-service-tikv = { path = "services/tikv", version = "0.58.1", optional = true, default-features = false }
opendal-service-tos = { path = "services/tos", version = "0.58.1", optional = true, default-features = false }
opendal-service-upyun = { path = "services/upyun", version = "0.58.1", optional = true, default-features = false }
opendal-service-vercel-artifacts = { path = "services/vercel-artifacts", version = "0.58.1", optional = true, default-features = false }
opendal-service-vercel-blob = { path = "services/vercel-blob", version = "0.58.1", optional = true, default-features = false }
opendal-service-webdav = { path = "services/webdav", version = "0.58.1", optional = true, default-features = false }
opendal-service-webhdfs = { path = "services/webhdfs", version = "0.58.1", optional = true, default-features = false }
opendal-service-yandex-disk = { path = "services/yandex-disk", version = "0.58.1", optional = true, default-features = false }
opendal-testkit = { path = "testkit", version = "0.58.1", optional = true }
[target.'cfg(target_os = "linux")'.dependencies]
opendal-layer-dtrace = { path = "layers/dtrace", version = "0.58.1", optional = true, default-features = false }
[target.'cfg(target_arch = "wasm32")'.dependencies]
opendal-service-opfs = { path = "services/opfs", version = "0.58.1", optional = true, default-features = false }
[dev-dependencies]
anyhow = "1.0"
bytes = { workspace = true }
divan = "0.1"
futures = { workspace = true, default-features = true }
http = { workspace = true }
libtest-mimic = "0.8"
log = { workspace = true }
logforth = { workspace = true }
rand = { workspace = true }
reqwest = "0.13.2"
sha2 = { workspace = true }
size = "0.5"
tokio = { workspace = true, features = ["fs", "macros", "rt-multi-thread"] }
uuid = { workspace = true, features = ["serde", "v4"] }
lance-artifact/third-party/opendal/DEPENDENCIES.md
Vendored
+3
View File
@@ -0,0 +1,3 @@
# Dependencies
Refer to [DEPENDENCIES.rust.tsv](DEPENDENCIES.rust.tsv) for full list.
lance-artifact/third-party/opendal/LICENSE
Vendored
+201
View File
@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
lance-artifact/third-party/opendal/NOTICE
Vendored
+5
View File
@@ -0,0 +1,5 @@
Apache OpenDAL
Copyright 2023-2026 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
lance-artifact/third-party/opendal/README.md
Vendored
+117
View File
@@ -0,0 +1,117 @@
# Apache OpenDAL™ Rust Core: One Layer, All Storage.
[![Build Status]][actions] [![Latest Version]][crates.io] [![Crate Downloads]][crates.io] [![chat]][discord]
[build status]: https://img.shields.io/github/actions/workflow/status/apache/opendal/ci_core.yml?branch=main
[actions]: https://github.com/apache/opendal/actions?query=branch%3Amain
[latest version]: https://img.shields.io/crates/v/opendal.svg
[crates.io]: https://crates.io/crates/opendal
[crate downloads]: https://img.shields.io/crates/d/opendal.svg
[chat]: https://img.shields.io/discord/1081052318650339399
[discord]: https://opendal.apache.org/discord
Apache OpenDAL™ is an Open Data Access Layer that enables seamless interaction with diverse storage services.
<img src="https://opendal.apache.org/img/architectural.png" alt="OpenDAL Architectural" width="61.8%" />
## Useful Links
- **User guide**: [opendal.apache.org/docs/core](https://opendal.apache.org/docs/core) — install, operations, layers, idioms, and extending the core.
- **API reference**: [docs.rs/opendal](https://docs.rs/opendal/) (release) | [dev](https://opendal.apache.org/docs/rust/opendal/)
- **Services & configuration**: [opendal.apache.org/services](https://opendal.apache.org/services)
- **Concepts**: [opendal.apache.org/docs/concepts](https://opendal.apache.org/docs/concepts)
- [Upgrade Guide](https://github.com/apache/opendal/blob/main/core/core/src/docs/upgrade.md) | [Release Notes](https://github.com/apache/opendal/blob/main/CHANGELOG.md) | [RFCs](https://github.com/apache/opendal/tree/main/core/core/src/docs/rfcs)
- [Examples](https://github.com/apache/opendal/tree/main/core/examples)
## Installation
```shell
cargo add opendal
```
Each service is a feature flag. The in-memory service is always available; enable
any other service with its `services-*` feature, e.g. `features = ["services-s3"]`.
## Quickstart
```rust
use opendal::services;
use opendal::Operator;
use opendal::Result;
#[tokio::main]
async fn main() -> Result<()> {
// Configure a service, then build an operator from it.
let op = Operator::new(services::Memory::default())?;
// The same verbs work on every service.
op.write("hello.txt", "Hello, World!").await?;
let bytes = op.read("hello.txt").await?;
let meta = op.stat("hello.txt").await?;
op.delete("hello.txt").await?;
println!("read {} bytes", meta.content_length());
Ok(())
}
```
To use a real backend, swap `Memory` for another service and configure it — the
operations stay identical. See [Getting started](https://opendal.apache.org/docs/core/getting-started)
and [Connecting to your storage](https://opendal.apache.org/docs/core/connecting).
## Services
OpenDAL talks to 50+ backends through one API. A selection by category:
| Type | Services |
|----------------------------|-----------------------------------------------------------------|
| Standard Protocols | ftp, http, sftp, webdav |
| Object Storage | s3, gcs, azblob, oss, cos, obs, b2, vercel-blob, … |
| File Storage | fs, hdfs, azdls, azfile, webhdfs, ipfs, … |
| Consumer Cloud Storage | gdrive, onedrive, dropbox, aliyun-drive, koofr, … |
| Key-Value & Database | redis, etcd, tikv, rocksdb, sqlite, postgresql, mongodb, … |
| Cache | memcached, moka, mini-moka, ghac, vercel-artifacts |
See [Services](https://opendal.apache.org/services) for the full list and each
service's configuration keys.
## Layers
Wrap an operator with layers to add retry, logging, timeout, metrics, and other
cross-cutting behavior without touching your storage code. Retry, logging,
timeout, and concurrency limit are built in:
```rust
use opendal::layers::RetryLayer;
use opendal::services::Memory;
use opendal::{Operator, Result};
fn build_operator() -> Result<Operator> {
let op = Operator::new(Memory::default())?;
Ok(op.layer(RetryLayer::new()))
}
```
See [Going to production](https://opendal.apache.org/docs/core/production) and the
[`layers` module](https://docs.rs/opendal/latest/opendal/layers/index.html).
## Contributing
Check out the [CONTRIBUTING](https://github.com/apache/opendal/blob/main/CONTRIBUTING.md) guide for building, testing, and
submitting changes to the core.
## Used by
Check out the [users](https://github.com/apache/opendal/blob/main/core/users.md) list for more details on who is using OpenDAL.
## Branding
The first and most prominent mentions must use the full form: **Apache OpenDAL™** of the name for any individual usage (webpage, handout, slides, etc.) Depending on the context and writing style, you should use the full form of the name sufficiently often to ensure that readers clearly understand the association of both the OpenDAL project and the OpenDAL software product to the ASF as the parent organization.
For more details, see the [Apache Product Name Usage Guide](https://www.apache.org/foundation/marks/guide).
## License and Trademarks
Licensed under the Apache License, Version 2.0: <http://www.apache.org/licenses/LICENSE-2.0>
Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.
lance-artifact/third-party/opendal/benches/README.md
Vendored
+18
View File
@@ -0,0 +1,18 @@
# OpenDAL Benchmarks
Running benchmark
```shell
OPENDAL_TEST=memory cargo bench
```
Build flamegraph:
```shell
cargo flamegraph --bench io -- seek --bench
```
- `--bench io` pick the `io` target.
- `-- seek --bench`: chose the benches with `seek` and `--bench` is required by `criterion`
After `flamegraph.svg` has been generated, use browser to visit it.
lance-artifact/third-party/opendal/benches/ops/README.md
Vendored
+26
View File
@@ -0,0 +1,26 @@
# Ops Benchmark Tests
Ops Benchmark Tests measure every operation's performance on the target platform.
To support benching different backends simultaneously, we use `environment value` to carry the backend config.
## Setup
Please copy `.env.example` to `.env` and change the values on need.
Take `fs` for example, we add config on `fs` on `/tmp`.
```dotenv
OPENDAL_FS_ROOT=/tmp
```
Notice: The default will skip benches if the env is not set.
## Run
Test specific backend, take s3 for example, first set the corresponding environment variables of s3, then:
```shell
OPENDAL_TEST=s3
cargo bench ops --features tests
```
lance-artifact/third-party/opendal/benches/ops/main.rs
Vendored
+25
View File
@@ -0,0 +1,25 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
mod read;
mod write;
mod utils;
fn main() {
divan::main();
}
lance-artifact/third-party/opendal/benches/ops/read.rs
Vendored
+100
View File
@@ -0,0 +1,100 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use divan::Bencher;
use divan::counter::BytesCount;
use opendal::tests::TEST_RUNTIME;
use opendal::tests::init_test_service;
use rand::rng;
use size::Size;
use super::utils::*;
#[divan::bench(
args = [
Size::from_kib(4),
Size::from_kib(64),
Size::from_mib(1),
Size::from_mib(16),
],
ignore = std::env::var("OPENDAL_TEST").is_err()
)]
fn whole(b: Bencher, size: Size) {
let op = init_test_service().unwrap().unwrap();
let mut rng = rng();
let content = gen_bytes(&mut rng, size.bytes() as usize);
let path = uuid::Uuid::new_v4().to_string();
let _temp_data = TempData::generate(op.clone(), &path, content.clone());
b.counter(BytesCount::from(size.bytes() as u64)).bench(|| {
let op = op.clone();
let path = path.clone();
TEST_RUNTIME.block_on(async move {
let _ = op.read(&path).await.unwrap();
})
})
}
mod concurrent {
use super::*;
#[divan::bench(
ignore = std::env::var("OPENDAL_TEST").is_err()
)]
fn baseline(b: Bencher) {
let op = init_test_service().unwrap().unwrap();
let mut rng = rng();
let content = gen_bytes(&mut rng, 1024 * 1024 * 1024);
let path = uuid::Uuid::new_v4().to_string();
let _temp_data = TempData::generate(op.clone(), &path, content.clone());
b.counter(BytesCount::from(1024usize * 1024 * 1024))
.bench(|| {
let op = op.clone();
let path = path.clone();
TEST_RUNTIME.block_on(async move {
let _ = op.read(&path).await.unwrap();
})
})
}
#[divan::bench(
args = [1, 2, 4, 8, 16],
ignore = std::env::var("OPENDAL_TEST").is_err()
)]
fn concurrent(b: Bencher, concurrent: usize) {
let op = init_test_service().unwrap().unwrap();
let mut rng = rng();
let content = gen_bytes(&mut rng, 1024 * 1024 * 1024);
let path = uuid::Uuid::new_v4().to_string();
let _temp_data = TempData::generate(op.clone(), &path, content.clone());
b.counter(BytesCount::from(1024usize * 1024 * 1024))
.bench(|| {
let op = op.clone();
let path = path.clone();
TEST_RUNTIME.block_on(async move {
let _ = op
.read_with(&path)
.chunk(16 * 1024 * 1024)
.concurrent(concurrent)
.await
.unwrap();
})
})
}
}
lance-artifact/third-party/opendal/benches/ops/utils.rs
Vendored
+59
View File
@@ -0,0 +1,59 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use bytes::Bytes;
use opendal::tests::TEST_RUNTIME;
use opendal::*;
use rand::prelude::*;
pub fn gen_bytes(rng: &mut ThreadRng, size: usize) -> Bytes {
let mut content = vec![0; size];
rng.fill_bytes(&mut content);
content.into()
}
pub struct TempData {
op: Operator,
path: String,
}
impl TempData {
pub fn existing(op: Operator, path: &str) -> Self {
Self {
op,
path: path.to_string(),
}
}
pub fn generate(op: Operator, path: &str, content: Bytes) -> Self {
TEST_RUNTIME.block_on(async { op.write(path, content).await.expect("create test data") });
Self {
op,
path: path.to_string(),
}
}
}
impl Drop for TempData {
fn drop(&mut self) {
TEST_RUNTIME.block_on(async {
self.op.delete(&self.path).await.expect("cleanup test data");
})
}
}
lance-artifact/third-party/opendal/benches/ops/write.rs
Vendored
+106
View File
@@ -0,0 +1,106 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use divan::Bencher;
use divan::counter::BytesCount;
use opendal::tests::TEST_RUNTIME;
use opendal::tests::init_test_service;
use rand::rng;
use size::Size;
use super::utils::*;
#[divan::bench(
args = [
Size::from_kib(4),
Size::from_kib(64),
Size::from_mib(1),
Size::from_mib(16),
],
ignore = std::env::var("OPENDAL_TEST").is_err()
)]
fn whole(b: Bencher, size: Size) {
let op = init_test_service().unwrap().unwrap();
let path = uuid::Uuid::new_v4().to_string();
let _temp_data = TempData::existing(op.clone(), &path);
b.counter(BytesCount::from(size.bytes() as u64))
.with_inputs(|| {
let mut rng = rng();
gen_bytes(&mut rng, size.bytes() as usize)
})
.bench_values(|content| {
let op = op.clone();
let path = path.clone();
TEST_RUNTIME.block_on(async move {
let _ = op.write(&path, content).await.unwrap();
})
})
}
mod concurrent {
use super::*;
/// Size of the data to be written.
const SIZE: usize = 64 * 1024 * 1024;
#[divan::bench(
ignore = std::env::var("OPENDAL_TEST").is_err()
)]
fn baseline(b: Bencher) {
let op = init_test_service().unwrap().unwrap();
let path = uuid::Uuid::new_v4().to_string();
let _temp_data = TempData::existing(op.clone(), &path);
let mut rng = rng();
let content = gen_bytes(&mut rng, SIZE);
b.counter(BytesCount::from(SIZE)).bench(|| {
let op = op.clone();
let path = path.clone();
let content = content.clone();
TEST_RUNTIME.block_on(async move {
let _ = op.write(&path, content).await.unwrap();
})
})
}
#[divan::bench(
args = [1, 2, 4, 8],
ignore = std::env::var("OPENDAL_TEST").is_err()
)]
fn concurrent(b: Bencher, concurrent: usize) {
let op = init_test_service().unwrap().unwrap();
let path = uuid::Uuid::new_v4().to_string();
let _temp_data = TempData::existing(op.clone(), &path);
let mut rng = rng();
let content = gen_bytes(&mut rng, SIZE);
b.counter(BytesCount::from(SIZE)).bench(|| {
let op = op.clone();
let path = path.clone();
let content = content.clone();
TEST_RUNTIME.block_on(async move {
let _ = op
.write_with(&path, content)
.chunk(8 * 1024 * 1024)
.concurrent(concurrent)
.await
.unwrap();
})
})
}
}
lance-artifact/third-party/opendal/benches/types/README.md
Vendored
+9
View File
@@ -0,0 +1,9 @@
# Types Benchmark Tests
This benchmark contains performance testing of critical data structures in opendal types, currently including performance testing of Buffer.
## Run
```shell
cargo bench types --features tests
```
lance-artifact/third-party/opendal/benches/types/buffer.rs
Vendored
+110
View File
@@ -0,0 +1,110 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use bytes::Buf;
use bytes::Bytes;
use divan::Bencher;
use divan::black_box;
use opendal::Buffer;
mod chunk {
use super::*;
#[divan::bench]
fn bytes(b: Bencher) {
b.with_inputs(|| Bytes::from(vec![1; 10]))
.bench_refs(|buffer| {
black_box(buffer.chunk());
});
}
#[divan::bench]
fn contiguous(b: Bencher) {
b.with_inputs(|| Buffer::from(vec![1; 10]))
.bench_refs(|buffer| {
black_box(buffer.chunk());
});
}
#[divan::bench(args = [10, 1_000, 1_000_000])]
fn non_contiguous(b: Bencher, parts: usize) {
b.with_inputs(|| Buffer::from([1; 1].repeat(parts)))
.bench_refs(|buffer| {
black_box(buffer.chunk());
});
}
}
mod advance {
use super::*;
#[divan::bench]
fn bytes(b: Bencher) {
b.with_inputs(|| Bytes::from(vec![1; 10]))
.bench_refs(|buffer| buffer.advance(4));
}
#[divan::bench]
fn contiguous(b: Bencher) {
b.with_inputs(|| Buffer::from(vec![1; 10]))
.bench_refs(|buffer| buffer.advance(4));
}
#[divan::bench(args = [10, 1_000, 1_000_000])]
fn non_contiguous(b: Bencher, parts: usize) {
b.with_inputs(|| Buffer::from([1; 1].repeat(parts)))
.bench_refs(|buffer| buffer.advance(4));
}
}
mod truncate {
use super::*;
#[divan::bench]
fn bytes(b: Bencher) {
b.with_inputs(|| Bytes::from(vec![1; 10]))
.bench_refs(|buffer| buffer.truncate(5));
}
#[divan::bench]
fn contiguous(b: Bencher) {
b.with_inputs(|| Buffer::from(vec![1; 10]))
.bench_refs(|buffer| buffer.truncate(5));
}
#[divan::bench(args = [10, 1_000, 1_000_000])]
fn non_contiguous(b: Bencher, parts: usize) {
b.with_inputs(|| Buffer::from([1; 1].repeat(parts)))
.bench_refs(|buffer| buffer.truncate(4));
}
}
mod iterator {
use super::*;
#[divan::bench]
fn contiguous(b: Bencher) {
b.with_inputs(|| Buffer::from(vec![1; 1_000_000]))
.bench_refs(|buffer| for _ in buffer {});
}
#[divan::bench(args = [10, 1_000, 1_000_000])]
fn non_contiguous(b: Bencher, parts: usize) {
b.with_inputs(|| Buffer::from(vec![1; 1_000_000 / parts].repeat(parts)))
.bench_refs(|buffer| for _ in buffer {});
}
}
lance-artifact/third-party/opendal/benches/types/main.rs
Vendored
+23
View File
@@ -0,0 +1,23 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
mod buffer;
mod tasks;
fn main() {
divan::main();
}
lance-artifact/third-party/opendal/benches/types/tasks.rs
Vendored
+64
View File
@@ -0,0 +1,64 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::sync::LazyLock;
use std::time::Duration;
use divan::Bencher;
use opendal::Executor;
use opendal::raw::ConcurrentTasks;
pub static TOKIO: LazyLock<tokio::runtime::Runtime> =
LazyLock::new(|| tokio::runtime::Runtime::new().expect("build tokio runtime"));
pub static TOTAL_TASKS: usize = 128;
#[divan::bench]
fn baseline(b: Bencher) {
b.bench(|| {
TOKIO.block_on(async move {
for _ in 0..TOTAL_TASKS {
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
});
}
#[divan::bench(args = [1, 2, 8, 32])]
fn concurrent(b: Bencher, concurrent: usize) {
b.with_inputs(|| {
ConcurrentTasks::new(Executor::new(), concurrent, 8, |()| {
Box::pin(async {
tokio::time::sleep(Duration::from_millis(1)).await;
((), Ok(()))
})
})
})
.bench_refs(|tasks| {
TOKIO.block_on(async move {
for _ in 0..TOTAL_TASKS {
let _ = tasks.execute(()).await;
}
loop {
if tasks.next().await.is_none() {
break;
}
}
})
});
}
lance-artifact/third-party/opendal/edge/README.md
Vendored
+3
View File
@@ -0,0 +1,3 @@
# OpenDAL Edge Tests
OpenDAL edge tests served as edge tests for the OpenDAL project. They will have pre-set data and will test the functionality of the OpenDAL project.
lance-artifact/third-party/opendal/examples/README.md
Vendored
+25
View File
@@ -0,0 +1,25 @@
# Apache OpenDAL™ Rust Core Examples
Thank you for using OpenDAL!
Those examples are designed to help you to understand how to use OpenDAL Rust Core.
## Setup
All examples following the same setup steps:
To run this example, please copy the `.env.example`, which is at project root, to `.env` and change the values under each type of service on need.
Take `fs` for example, we need to change to enable behavior test on `fs` on `/tmp`.
```dotenv
OPENDAL_FS_ROOT=/path/to/dir
OPENDAL_FS_ATOMIC_WRITE_DIR=/path/to/dir
```
into
```dotenv
OPENDAL_FS_ROOT=/tmp/
OPENDAL_FS_ATOMIC_WRITE_DIR=/tmp/
```
lance-artifact/third-party/opendal/src/lib.rs
Vendored
+470
View File
@@ -0,0 +1,470 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#![doc(
html_logo_url = "https://raw.githubusercontent.com/apache/opendal/main/website/static/img/logo.svg"
)]
#![doc = include_str!("../README.md")]
#![cfg_attr(docsrs, feature(doc_cfg))]
#![cfg_attr(docsrs, doc(auto_cfg))]
#![deny(missing_docs)]
pub use opendal_core::*;
#[cfg(feature = "tests")]
pub extern crate opendal_testkit as tests;
/// Install the global defaults provided by the facade crate.
///
/// This function is safe to call multiple times. It registers enabled services
/// and installs the default HTTP transport when the corresponding feature is
/// enabled.
///
/// # Process-wide HTTP transport
///
/// When `auto-register-services` is enabled, a process constructor calls this
/// function before `main`. If a default HTTP transport feature is also enabled,
/// that transport occupies the process-wide default because
/// [`HttpTransporter::install_default`] uses first-installed-wins semantics.
///
/// Applications that need to install their own process-wide transport should
/// disable `auto-register-services`, call [`init_default_registry`] explicitly
/// when URI-based construction is needed, and install the transport themselves.
/// Per-operator transports can be configured regardless of this setting.
pub fn install_default() {
init_default_registry();
#[cfg(feature = "http-transport-reqwest")]
opendal_http_transport_reqwest::install_default();
}
/// Initialize the global [`OperatorRegistry`] with enabled services.
///
/// This function is safe to call multiple times and will only perform
/// initialization once.
///
/// # Notes
///
/// Some bindings link `opendal` as a `staticlib`, where `#[ctor::ctor]`
/// initializers may not be executed due to linker behavior. Those bindings
/// should call this function explicitly before using `Operator::from_uri` or
/// `Operator::via_iter`.
pub fn init_default_registry() {
static DEFAULT_REGISTRY_INIT: std::sync::Once = std::sync::Once::new();
DEFAULT_REGISTRY_INIT.call_once(|| init_default_registry_inner(OperatorRegistry::get()));
}
fn init_default_registry_inner(registry: &OperatorRegistry) {
opendal_core::services::register_memory_service(registry);
#[cfg(feature = "services-aliyun-drive")]
opendal_service_aliyun_drive::register_aliyun_drive_service(registry);
#[cfg(feature = "services-alluxio")]
opendal_service_alluxio::register_alluxio_service(registry);
#[cfg(feature = "services-azblob")]
opendal_service_azblob::register_azblob_service(registry);
#[cfg(feature = "services-azdls")]
opendal_service_azdls::register_azdls_service(registry);
#[cfg(feature = "services-azfile")]
opendal_service_azfile::register_azfile_service(registry);
#[cfg(feature = "services-b2")]
opendal_service_b2::register_b2_service(registry);
#[cfg(feature = "services-cacache")]
opendal_service_cacache::register_cacache_service(registry);
#[cfg(feature = "services-cloudflare-kv")]
opendal_service_cloudflare_kv::register_cloudflare_kv_service(registry);
#[cfg(feature = "services-compfs")]
opendal_service_compfs::register_compfs_service(registry);
#[cfg(feature = "services-cos")]
opendal_service_cos::register_cos_service(registry);
#[cfg(feature = "services-d1")]
opendal_service_d1::register_d1_service(registry);
#[cfg(feature = "services-dashmap")]
opendal_service_dashmap::register_dashmap_service(registry);
#[cfg(feature = "services-dbfs")]
opendal_service_dbfs::register_dbfs_service(registry);
#[cfg(feature = "services-dropbox")]
opendal_service_dropbox::register_dropbox_service(registry);
#[cfg(feature = "services-etcd")]
opendal_service_etcd::register_etcd_service(registry);
#[cfg(feature = "services-foundationdb")]
opendal_service_foundationdb::register_foundationdb_service(registry);
#[cfg(feature = "services-foyer")]
opendal_service_foyer::register_foyer_service(registry);
#[cfg(feature = "services-fs")]
opendal_service_fs::register_fs_service(registry);
#[cfg(feature = "services-ftp")]
opendal_service_ftp::register_ftp_service(registry);
#[cfg(feature = "services-gcs")]
opendal_service_gcs::register_gcs_service(registry);
#[cfg(feature = "services-gdrive")]
opendal_service_gdrive::register_gdrive_service(registry);
#[cfg(feature = "services-ghac")]
opendal_service_ghac::register_ghac_service(registry);
#[cfg(feature = "services-github")]
opendal_service_github::register_github_service(registry);
#[cfg(feature = "services-goosefs")]
opendal_service_goosefs::register_goosefs_service(registry);
#[cfg(feature = "services-gridfs")]
opendal_service_gridfs::register_gridfs_service(registry);
#[cfg(feature = "services-hdfs")]
opendal_service_hdfs::register_hdfs_service(registry);
#[cfg(feature = "services-hdfs-native")]
opendal_service_hdfs_native::register_hdfs_native_service(registry);
#[cfg(feature = "services-http")]
opendal_service_http::register_http_service(registry);
#[cfg(feature = "services-hf")]
opendal_service_hf::register_hf_service(registry);
#[cfg(feature = "services-ipfs")]
opendal_service_ipfs::register_ipfs_service(registry);
#[cfg(feature = "services-ipmfs")]
opendal_service_ipmfs::register_ipmfs_service(registry);
#[cfg(feature = "services-koofr")]
opendal_service_koofr::register_koofr_service(registry);
#[cfg(feature = "services-lakefs")]
opendal_service_lakefs::register_lakefs_service(registry);
#[cfg(feature = "services-memcached")]
opendal_service_memcached::register_memcached_service(registry);
#[cfg(feature = "services-mini-moka")]
opendal_service_mini_moka::register_mini_moka_service(registry);
#[cfg(feature = "services-moka")]
opendal_service_moka::register_moka_service(registry);
#[cfg(feature = "services-mongodb")]
opendal_service_mongodb::register_mongodb_service(registry);
#[cfg(feature = "services-monoiofs")]
opendal_service_monoiofs::register_monoiofs_service(registry);
#[cfg(feature = "services-mysql")]
opendal_service_mysql::register_mysql_service(registry);
#[cfg(feature = "services-obs")]
opendal_service_obs::register_obs_service(registry);
#[cfg(feature = "services-onedrive")]
opendal_service_onedrive::register_onedrive_service(registry);
#[cfg(feature = "services-oss")]
opendal_service_oss::register_oss_service(registry);
#[cfg(feature = "services-pcloud")]
opendal_service_pcloud::register_pcloud_service(registry);
#[cfg(feature = "services-persy")]
opendal_service_persy::register_persy_service(registry);
#[cfg(feature = "services-postgresql")]
opendal_service_postgresql::register_postgresql_service(registry);
#[cfg(feature = "services-redb")]
opendal_service_redb::register_redb_service(registry);
#[cfg(feature = "services-rocksdb")]
opendal_service_rocksdb::register_rocksdb_service(registry);
#[cfg(feature = "services-s3")]
opendal_service_s3::register_s3_service(registry);
#[cfg(feature = "services-seafile")]
opendal_service_seafile::register_seafile_service(registry);
#[cfg(feature = "services-sftp")]
opendal_service_sftp::register_sftp_service(registry);
#[cfg(feature = "services-sled")]
opendal_service_sled::register_sled_service(registry);
#[cfg(feature = "services-sqlite")]
opendal_service_sqlite::register_sqlite_service(registry);
#[cfg(feature = "services-surrealdb")]
opendal_service_surrealdb::register_surrealdb_service(registry);
#[cfg(feature = "services-swift")]
opendal_service_swift::register_swift_service(registry);
#[cfg(feature = "services-tikv")]
opendal_service_tikv::register_tikv_service(registry);
#[cfg(feature = "services-tos")]
opendal_service_tos::register_tos_service(registry);
#[cfg(feature = "services-upyun")]
opendal_service_upyun::register_upyun_service(registry);
#[cfg(feature = "services-vercel-artifacts")]
opendal_service_vercel_artifacts::register_vercel_artifacts_service(registry);
#[cfg(feature = "services-vercel-blob")]
opendal_service_vercel_blob::register_vercel_blob_service(registry);
#[cfg(feature = "services-webdav")]
opendal_service_webdav::register_webdav_service(registry);
#[cfg(feature = "services-webhdfs")]
opendal_service_webhdfs::register_webhdfs_service(registry);
#[cfg(feature = "services-yandex-disk")]
opendal_service_yandex_disk::register_yandex_disk_service(registry);
#[cfg(all(target_arch = "wasm32", feature = "services-opfs"))]
opendal_service_opfs::register_opfs_service(registry);
#[cfg(any(feature = "services-redis", feature = "services-redis-native-tls"))]
opendal_service_redis::register_redis_service(registry);
}
#[cfg(feature = "auto-register-services")]
#[ctor::ctor(unsafe)]
fn register_default_operator_registry() {
install_default();
}
/// Service builders enabled through `services-*` Cargo features.
///
/// [`Memory`](services::Memory) is always available. Other builders are
/// re-exported when their matching facade feature is enabled; for example,
/// `services-s3` enables [`S3`](services::S3).
///
/// Pass a configured builder to [`Operator::new`]. The facade automatically
/// registers enabled services for [`Operator::from_uri`] and
/// [`Operator::via_iter`] when the `auto-register-services` feature is enabled.
pub mod services {
pub use opendal_core::services::*;
#[cfg(feature = "services-aliyun-drive")]
pub use opendal_service_aliyun_drive::*;
#[cfg(feature = "services-alluxio")]
pub use opendal_service_alluxio::*;
#[cfg(feature = "services-azblob")]
pub use opendal_service_azblob::*;
#[cfg(feature = "services-azdls")]
pub use opendal_service_azdls::*;
#[cfg(feature = "services-azfile")]
pub use opendal_service_azfile::*;
#[cfg(feature = "services-b2")]
pub use opendal_service_b2::*;
#[cfg(feature = "services-cacache")]
pub use opendal_service_cacache::*;
#[cfg(feature = "services-cloudflare-kv")]
pub use opendal_service_cloudflare_kv::*;
#[cfg(feature = "services-compfs")]
pub use opendal_service_compfs::*;
#[cfg(feature = "services-cos")]
pub use opendal_service_cos::*;
#[cfg(feature = "services-d1")]
pub use opendal_service_d1::*;
#[cfg(feature = "services-dashmap")]
pub use opendal_service_dashmap::*;
#[cfg(feature = "services-dbfs")]
pub use opendal_service_dbfs::*;
#[cfg(feature = "services-dropbox")]
pub use opendal_service_dropbox::*;
#[cfg(feature = "services-etcd")]
pub use opendal_service_etcd::*;
#[cfg(feature = "services-foundationdb")]
pub use opendal_service_foundationdb::*;
#[cfg(feature = "services-foyer")]
pub use opendal_service_foyer::*;
#[cfg(feature = "services-fs")]
pub use opendal_service_fs::*;
#[cfg(feature = "services-ftp")]
pub use opendal_service_ftp::*;
#[cfg(feature = "services-gcs")]
pub use opendal_service_gcs::*;
#[cfg(feature = "services-gdrive")]
pub use opendal_service_gdrive::*;
#[cfg(feature = "services-ghac")]
pub use opendal_service_ghac::*;
#[cfg(feature = "services-github")]
pub use opendal_service_github::*;
#[cfg(feature = "services-goosefs")]
pub use opendal_service_goosefs::*;
#[cfg(feature = "services-gridfs")]
pub use opendal_service_gridfs::*;
#[cfg(feature = "services-hdfs")]
pub use opendal_service_hdfs::*;
#[cfg(feature = "services-hdfs-native")]
pub use opendal_service_hdfs_native::*;
#[cfg(feature = "services-hf")]
pub use opendal_service_hf::*;
#[cfg(feature = "services-http")]
pub use opendal_service_http::*;
#[cfg(feature = "services-ipfs")]
pub use opendal_service_ipfs::*;
#[cfg(feature = "services-ipmfs")]
pub use opendal_service_ipmfs::*;
#[cfg(feature = "services-koofr")]
pub use opendal_service_koofr::*;
#[cfg(feature = "services-lakefs")]
pub use opendal_service_lakefs::*;
#[cfg(feature = "services-memcached")]
pub use opendal_service_memcached::*;
#[cfg(feature = "services-mini-moka")]
pub use opendal_service_mini_moka::*;
#[cfg(feature = "services-moka")]
pub use opendal_service_moka::*;
#[cfg(feature = "services-mongodb")]
pub use opendal_service_mongodb::*;
#[cfg(feature = "services-monoiofs")]
pub use opendal_service_monoiofs::*;
#[cfg(feature = "services-mysql")]
pub use opendal_service_mysql::*;
#[cfg(feature = "services-obs")]
pub use opendal_service_obs::*;
#[cfg(feature = "services-onedrive")]
pub use opendal_service_onedrive::*;
#[cfg(all(target_arch = "wasm32", feature = "services-opfs"))]
pub use opendal_service_opfs::*;
#[cfg(feature = "services-oss")]
pub use opendal_service_oss::*;
#[cfg(feature = "services-pcloud")]
pub use opendal_service_pcloud::*;
#[cfg(feature = "services-persy")]
pub use opendal_service_persy::*;
#[cfg(feature = "services-postgresql")]
pub use opendal_service_postgresql::*;
#[cfg(feature = "services-redb")]
pub use opendal_service_redb::*;
#[cfg(feature = "services-redis")]
pub use opendal_service_redis::*;
#[cfg(feature = "services-rocksdb")]
pub use opendal_service_rocksdb::*;
#[cfg(feature = "services-s3")]
pub use opendal_service_s3::*;
#[cfg(feature = "services-seafile")]
pub use opendal_service_seafile::*;
#[cfg(feature = "services-sftp")]
pub use opendal_service_sftp::*;
#[cfg(feature = "services-sled")]
pub use opendal_service_sled::*;
#[cfg(feature = "services-sqlite")]
pub use opendal_service_sqlite::*;
#[cfg(feature = "services-surrealdb")]
pub use opendal_service_surrealdb::*;
#[cfg(feature = "services-swift")]
pub use opendal_service_swift::*;
#[cfg(feature = "services-tikv")]
pub use opendal_service_tikv::*;
#[cfg(feature = "services-tos")]
pub use opendal_service_tos::*;
#[cfg(feature = "services-upyun")]
pub use opendal_service_upyun::*;
#[cfg(feature = "services-vercel-artifacts")]
pub use opendal_service_vercel_artifacts::*;
#[cfg(feature = "services-vercel-blob")]
pub use opendal_service_vercel_blob::*;
#[cfg(feature = "services-webdav")]
pub use opendal_service_webdav::*;
#[cfg(feature = "services-webhdfs")]
pub use opendal_service_webhdfs::*;
#[cfg(feature = "services-yandex-disk")]
pub use opendal_service_yandex_disk::*;
}
/// Layers enabled through `layers-*` Cargo features.
///
/// A layer adds cross-service behavior to an [`Operator`]. Enable the matching
/// facade feature, construct the layer, and pass it to [`Operator::layer`].
/// Layer order is observable, so review each layer's composition notes when
/// combining retries, timeouts, caching, or observability.
pub mod layers {
pub use opendal_core::layers::*;
#[cfg(feature = "layers-async-backtrace")]
pub use opendal_layer_async_backtrace::*;
#[cfg(feature = "layers-await-tree")]
pub use opendal_layer_await_tree::*;
#[cfg(feature = "layers-capability-check")]
pub use opendal_layer_capability_check::*;
#[cfg(feature = "layers-chaos")]
pub use opendal_layer_chaos::*;
#[cfg(feature = "layers-concurrent-limit")]
pub use opendal_layer_concurrent_limit::*;
#[cfg(all(target_os = "linux", feature = "layers-dtrace"))]
pub use opendal_layer_dtrace::*;
#[cfg(feature = "layers-fastmetrics")]
pub use opendal_layer_fastmetrics::*;
#[cfg(feature = "layers-fastrace")]
pub use opendal_layer_fastrace::*;
#[cfg(feature = "layers-foyer")]
pub use opendal_layer_foyer::*;
#[cfg(feature = "layers-hotpath")]
pub use opendal_layer_hotpath::*;
#[cfg(feature = "layers-immutable-index")]
pub use opendal_layer_immutable_index::*;
#[cfg(feature = "layers-logging")]
pub use opendal_layer_logging::*;
#[cfg(feature = "layers-metrics")]
pub use opendal_layer_metrics::*;
#[cfg(feature = "layers-mime-guess")]
pub use opendal_layer_mime_guess::*;
#[cfg(feature = "layers-otel-metrics")]
pub use opendal_layer_otelmetrics::*;
#[cfg(feature = "layers-otel-trace")]
pub use opendal_layer_oteltrace::*;
#[cfg(feature = "layers-prometheus")]
pub use opendal_layer_prometheus::*;
#[cfg(feature = "layers-prometheus-client")]
pub use opendal_layer_prometheus_client::*;
#[cfg(feature = "layers-retry")]
pub use opendal_layer_retry::*;
#[cfg(feature = "layers-route")]
pub use opendal_layer_route::*;
#[cfg(feature = "layers-tail-cut")]
pub use opendal_layer_tail_cut::*;
#[cfg(feature = "layers-throttle")]
pub use opendal_layer_throttle::*;
#[cfg(feature = "layers-timeout")]
pub use opendal_layer_timeout::*;
#[cfg(feature = "layers-tracing")]
pub use opendal_layer_tracing::*;
}
lance-artifact/third-party/opendal/upgrade.md
Vendored
+1717
View File
File diff suppressed because it is too large Load Diff
lance-artifact/third-party/opendal/users.md
Vendored
+17
View File
@@ -0,0 +1,17 @@
# Users
- [apache/iceberg-rust](https://github.com/apache/iceberg-rust/): Native Rust implementation of [Apache Iceberg](https://iceberg.apache.org/), the open table format for analytic datasets.
- [apache/paimon-rust](https://github.com/apache/paimon-rust): The Rust implementation of [Apache Paimon](https://paimon.apache.org/).
- [Databend](https://github.com/datafuselabs/databend/): A modern Elasticity and Performance cloud data warehouse.
- [deepeth/mars](https://github.com/deepeth/mars): The powerful analysis platform to explore and visualize data from blockchain.
- [GreptimeDB](https://github.com/GreptimeTeam/greptimedb): An open-source, cloud-native, distributed time-series database.
- [kunobi-kache](https://github.com/kunobi-ninja/kache): Zero-copy, content-addressed Rust build cache for Rust, C/C++ and more.
- [kvcache-ai/AgentENV](https://github.com/kvcache-ai/AgentENV): A distributed platform for running agent environments at scale.
- [mozilla/sccache](https://github.com/mozilla/sccache/): `sccache` is [`ccache`](https://github.com/ccache/ccache) with cloud storage
- [Noorle](https://noorle.com/): A managed runtime for AI agents; OpenDAL backs agents' files and memories on different storage providers.
- [OctoBase](https://github.com/toeverything/OctoBase): the open-source database behind [AFFiNE](https://github.com/toeverything/affine), local-first, yet collaborative.
- [Pants](https://github.com/pantsbuild/pants): A fast, scalable, user-friendly build system for codebases of all sizes.
- [QuestDB](https://github.com/questdb/questdb): An open-source time-series database for high throughput ingestion and fast SQL queries with operational simplicity.
- [RisingWave](https://github.com/risingwavelabs/risingwave): A Distributed SQL Database for Stream Processing
- [Vector](https://github.com/vectordotdev/vector): A high-performance observability data pipeline.
- [rustic](https://github.com/rustic-rs/rustic): Fast, encrypted, and deduplicated backups powered by Rust