From 291e9e37befd1202ff4358973a012d6c1eabc1c9 Mon Sep 17 00:00:00 2001 From: ForwardXu Date: Thu, 9 Jul 2026 05:14:39 +0800 Subject: [PATCH] feat: add Tencent COS and GooseFS object store support via new feature flags (#3526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Closes #3525 This PR wires up two new optional object-store backends at the LanceDB layer, exposing capabilities that already exist upstream in `lance` / `lance-io`: | Backend | Cargo feature | Default in Rust crate | Default in Python wheel | Default in Node binding | | --- | --- | --- | --- | --- | | **Tencent COS** | `cos` | ❌ off | ✅ on | ❌ off | | **GooseFS** | `goosefs` | ❌ off | ✅ on | ✅ on | Both backends are additive and do not affect existing users who don't opt in. ## Motivation - **Tencent COS** is the dominant object storage in the China region. Tencent Cloud users currently need an S3-compatible proxy or a private fork to use LanceDB against COS buckets. - **GooseFS** is Tencent Cloud's distributed cache acceleration layer that sits in front of COS/S3, a common pattern for vector search / AI training where the same hot dataset is read repeatedly. - This brings COS / GooseFS to feature parity with the existing first-class backends (`aws`, `gcs`, `azure`, `oss`, `huggingface`). See the linked issue #3525 for the full discussion. ## Changes ### `rust/lancedb/Cargo.toml` Add two new optional features that pull through the corresponding upstream feature flags: ```toml cos = ["lance/tencent", "lance-io/tencent"] goosefs = [ "lance/goosefs", "lance-io/goosefs", "lance-namespace-impls/dir-goosefs", ] ``` ### `python/Cargo.toml` Enable both `cos` and `goosefs` by default for the Python wheels, so `pip install lancedb` works against COS / GooseFS out of the box (consistent with how `aws` / `gcs` / `azure` / `oss` are bundled today): ```diff -default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"] +default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs"] ``` ### `nodejs/Cargo.toml` Enable `goosefs` by default for the Node binding (COS kept opt-in to limit the default native binary size; can be revisited based on demand): ```diff -default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"] +default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/goosefs"] ``` ### `Cargo.lock` Regenerated to reflect the transitive dependencies brought in by the new upstream features. No manual edits. ## Example Usage ### Rust ```toml # Cargo.toml lancedb = { version = "0.30", features = ["cos", "goosefs"] } ``` ```rust // Tencent COS let db = lancedb::connect("cos://my-bucket/my-db").execute().await?; // GooseFS let db = lancedb::connect("goosefs://my-namespace/my-db").execute().await?; ``` ### Python ```python import lancedb db = lancedb.connect( "cos://my-bucket/my-db", storage_options={ "secret_id": "...", "secret_key": "...", "region": "ap-guangzhou", }, ) ``` ## Backwards Compatibility - All new features are **opt-in** at the Rust crate level (`default = []` for `lancedb` itself is unchanged). - The Python wheel gains both backends by default, increasing wheel size slightly but matching the existing pattern of bundling all major cloud backends. - Node binding only adds `goosefs` to defaults; existing users see no behavior change. ## Testing - `cargo check --all-features` ✅ - `cargo check -p lancedb --features cos` ✅ - `cargo check -p lancedb --features goosefs` ✅ - End-to-end COS / GooseFS smoke tests require Tencent Cloud credentials and are intentionally not added to CI in this PR (same approach used for `s3-test`). Happy to add a gated test feature in a follow-up if reviewers prefer. ## Checklist - [x] Added `cos` and `goosefs` features to `rust/lancedb/Cargo.toml` - [x] Updated `python/Cargo.toml` default features - [x] Updated `nodejs/Cargo.toml` default features - [x] Regenerated `Cargo.lock` - [x] Verified build with `--all-features` - [ ] Documentation update (can be done in a follow-up PR once API stabilizes) ## Related - Issue: #3525 - Upstream support: [`lance/tencent`](https://github.com/lance-format/lance), [`lance/goosefs`](https://github.com/lance-format/lance) --- Cargo.lock | 138 ++++++++++++++++++++++++++++++++++++++++ nodejs/Cargo.toml | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 6 ++ 4 files changed, 146 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c693e34a2..f8533441d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3779,6 +3779,30 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "goosefs-sdk" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +dependencies = [ + "async-trait", + "bytes", + "dashmap", + "hostname", + "prost", + "prost-types", + "rand 0.9.4", + "reqwest 0.12.28", + "serde", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tracing", + "uuid", +] + [[package]] name = "group" version = "0.12.1" @@ -3990,6 +4014,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if 1.0.4", + "libc", + "windows-link", +] + [[package]] name = "http" version = "0.2.12" @@ -4150,6 +4185,19 @@ dependencies = [ "webpki-roots 1.0.7", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.9.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -6343,7 +6391,9 @@ dependencies = [ "opendal-layer-timeout", "opendal-service-azblob", "opendal-service-azdls", + "opendal-service-cos", "opendal-service-gcs", + "opendal-service-goosefs", "opendal-service-hf", "opendal-service-oss", "opendal-service-s3", @@ -6471,6 +6521,23 @@ dependencies = [ "opendal-core", ] +[[package]] +name = "opendal-service-cos" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +dependencies = [ + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "quick-xml 0.39.4", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-tencent-cos", + "serde", +] + [[package]] name = "opendal-service-gcs" version = "0.57.0" @@ -6492,6 +6559,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-goosefs" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +dependencies = [ + "bytes", + "goosefs-sdk", + "log", + "opendal-core", + "serde", + "tokio", +] + [[package]] name = "opendal-service-hf" version = "0.57.0" @@ -8127,6 +8208,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "reqsign-tencent-cos" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" +dependencies = [ + "anyhow", + "http 1.4.2", + "log", + "percent-encoding", + "reqsign-core", + "serde", + "serde_json", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -9812,6 +9908,45 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "h2 0.4.14", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "hyper 1.9.0", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2 0.6.3", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -9820,9 +9955,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap 2.14.0", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 7793b694e..31d62c10c 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -44,6 +44,6 @@ aws-lc-rs = "=1.16.3" napi-build = "2.3.1" [features] -default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"] +default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/goosefs"] fp16kernels = ["lancedb/fp16kernels"] remote = ["lancedb/remote"] diff --git a/python/Cargo.toml b/python/Cargo.toml index 9f14be7fc..f8cf9b666 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -47,6 +47,6 @@ pyo3-build-config = { version = "0.28", features = [ ] } [features] -default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"] +default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs"] fp16kernels = ["lancedb/fp16kernels"] remote = ["lancedb/remote"] diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 761d6e55d..59c1ec6e4 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -128,6 +128,12 @@ azure = [ "lance-namespace-impls/dir-azure", "lance-namespace-impls/credential-vendor-azure", ] +cos = ["lance/tencent", "lance-io/tencent"] +goosefs = [ + "lance/goosefs", + "lance-io/goosefs", + "lance-namespace-impls/dir-goosefs", +] huggingface = [ "lance/huggingface", "lance-io/huggingface",