update

Cargo.lock

@@ -3044,8 +3044,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow-array",
  "rand 0.9.2",
@@ -4229,8 +4229,8 @@ dependencies = [
 
 [[package]]
 name = "lance"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4293,8 +4293,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4312,8 +4312,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrayref",
  "paste",
@@ -4322,8 +4322,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4359,8 +4359,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4389,8 +4389,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4407,8 +4407,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4445,8 +4445,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4479,8 +4479,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4542,8 +4542,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4583,8 +4583,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4607,8 +4607,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow",
  "async-trait",
@@ -4620,8 +4620,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -4630,9 +4630,8 @@ dependencies = [
  "bytes",
  "lance",
  "lance-core",
- "lance-io",
  "lance-namespace",
- "object_store",
+ "opendal",
  "reqwest",
  "serde_json",
  "snafu",
@@ -4654,8 +4653,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4693,8 +4692,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "0.38.3-beta.11"
+version = "0.38.3-beta.9"
-source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
+source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5182,9 +5181,12 @@ dependencies = [
 
 [[package]]
 name = "mock_instant"
-version = "0.6.0"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6"
+checksum = "9366861eb2a2c436c20b12c8dbec5f798cea6b47ad99216be0282942e2c81ea0"
+dependencies = [
+ "once_cell",
+]
 
 [[package]]
 name = "moka"

Cargo.toml

@@ -15,20 +15,20 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=0.38.3-beta.11", default-features = false, "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance = { "version" = "=0.38.3-beta.9", default-features = false, "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-core = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-core = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-datagen = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-datagen = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-file = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-file = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-io = { "version" = "=0.38.3-beta.11", default-features = false, "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-io = { "version" = "=0.38.3-beta.9", default-features = false, "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-index = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-index = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-linalg = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-linalg = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-namespace = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-namespace = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-namespace-impls = { "version" = "=0.38.3-beta.11", "features" = ["dir-aws", "dir-gcp", "dir-azure", "dir-oss", "rest"], "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-namespace-impls = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-table = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-table = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-testing = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-testing = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-datafusion = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-datafusion = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-encoding = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-encoding = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
-lance-arrow = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
+lance-arrow = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "56.2", optional = false }
@@ -65,4 +65,4 @@ semver = "1.0.25"
 crunchy = "0.2.4"
 chrono = "0.4"
 # Workaround for: https://github.com/Lokathor/bytemuck/issues/306
-bytemuck_derive = ">=1.8.1, <1.9.0"
+bytemuck_derive = ">=1.8.1, <1.9.0"

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.25.3-beta.3"
+current_version = "0.25.3-beta.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.25.3-beta.3"
+version = "0.25.3-beta.2"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true

python/python/lancedb/embeddings/gemini_text.py

@@ -4,7 +4,7 @@
 
 import os
 from functools import cached_property
-from typing import List, Union
+from typing import List, Optional, Union
 
 import numpy as np
 
@@ -46,10 +46,11 @@ class GeminiText(TextEmbeddingFunction):
 
     Parameters
     ----------
-    name: str, default "models/embedding-001"
+    name: str, default "models/text-embedding-004"
         The name of the model to use. See the Gemini documentation for a list of
         available models.
-
+    dims: int, optional
+        The dimension of the embedding, otherwise it will be inferred.
     query_task_type: str, default "retrieval_query"
         Sets the task type for the queries.
     source_task_type: str, default "retrieval_document"
@@ -77,9 +78,10 @@ class GeminiText(TextEmbeddingFunction):
 
     """
 
-    name: str = "models/embedding-001"
+    name: str = "models/text-embedding-004"
     query_task_type: str = "retrieval_query"
     source_task_type: str = "retrieval_document"
+    dims: Optional[int] = None
 
     if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
 
@@ -89,9 +91,18 @@ class GeminiText(TextEmbeddingFunction):
         model_config = dict()
         model_config["ignored_types"] = (cached_property,)
 
-    def ndims(self):
+    @cached_property
-        # TODO: fix hardcoding
+    def _model(self):
-        return 768
+        return self.client.get_model(self.name)
+
+    def ndims(self) -> int:
+        if self.dims:
+            return self.dims
+        if hasattr(self._model, "output_dimensionality"):
+            return self._model.output_dimensionality
+        # Fallback for older versions of the library
+        # or models that don't have the attribute
+        return len(self.generate_embeddings(["lancedb"])[0])
 
     def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
         return self.compute_source_embeddings(query, task_type=self.query_task_type)
@@ -119,6 +130,8 @@ class GeminiText(TextEmbeddingFunction):
         ):  # Provide a title to use existing API design
             title = "Embedding of a document"
             kwargs["title"] = title
+        if self.dims:
+            kwargs["output_dimensionality"] = self.dims
 
         return [
             self.client.embed_content(model=self.name, content=text, **kwargs)[
@@ -131,6 +144,8 @@ class GeminiText(TextEmbeddingFunction):
     def client(self):
         genai = attempt_import_or_raise("google.generativeai", "google.generativeai")
 
-        if not os.environ.get("GOOGLE_API_KEY"):
+        api_key = os.environ.get("GOOGLE_API_KEY")
+        if not api_key:
             api_key_not_found_help("google")
+        genai.configure(api_key=api_key)
         return genai

python/python/tests/test_embeddings_slow.py

@@ -308,7 +308,7 @@ def test_instructor_embedding(tmp_path):
     os.environ.get("GOOGLE_API_KEY") is None, reason="GOOGLE_API_KEY not set"
 )
 def test_gemini_embedding(tmp_path):
-    model = get_registry().get("gemini-text").create(max_retries=0)
+    model = get_registry().get("gemini-text").create(max_retries=0, dims=512)
 
     class TextModel(LanceModel):
         text: str = model.SourceField()
@@ -319,7 +319,7 @@ def test_gemini_embedding(tmp_path):
     tbl = db.create_table("test", schema=TextModel, mode="overwrite")
 
     tbl.add(df)
-    assert len(tbl.to_pandas()["vector"][0]) == model.ndims()
+    assert len(tbl.to_pandas()["vector"][0]) == model.ndims() == 512
     assert tbl.search("hello").limit(1).to_pandas()["text"][0] == "hello world"
 
 

python/python/tests/test_namespace.py

@@ -59,14 +59,6 @@ class TempNamespace(LanceNamespace):
             root
         ]  # Reference to shared namespaces
 
-    def namespace_id(self) -> str:
-        """Return a human-readable unique identifier for this namespace instance.
-
-        Returns:
-            A unique identifier string based on the root directory
-        """
-        return f"TempNamespace {{ root: '{self.config.root}' }}"
-
     def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
         """List all tables in the namespace."""
         if not request.id:

rust/lancedb/Cargo.toml

@@ -44,7 +44,7 @@ lance-testing = { workspace = true }
 lance-encoding = { workspace = true }
 lance-arrow = { workspace = true }
 lance-namespace = { workspace = true }
-lance-namespace-impls = { workspace = true }
+lance-namespace-impls = { workspace = true, features = ["dir", "rest"] }
 moka = { workspace = true }
 pin-project = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }

rust/lancedb/src/database/namespace.rs

@@ -14,7 +14,7 @@ use lance_namespace::{
     },
     LanceNamespace,
 };
-use lance_namespace_impls::ConnectBuilder;
+use lance_namespace_impls::connect::connect as connect_namespace;
 
 use crate::database::listing::ListingDatabase;
 use crate::error::{Error, Result};
@@ -48,16 +48,11 @@ impl LanceNamespaceDatabase {
         read_consistency_interval: Option<std::time::Duration>,
         session: Option<Arc<lance::session::Session>>,
     ) -> Result<Self> {
-        let mut builder = ConnectBuilder::new(ns_impl);
+        let namespace = connect_namespace(ns_impl, ns_properties.clone())
-        for (key, value) in ns_properties.clone() {
+            .await
-            builder = builder.property(key, value);
+            .map_err(|e| Error::InvalidInput {
-        }
+                message: format!("Failed to connect to namespace: {:?}", e),
-        if let Some(ref sess) = session {
+            })?;
-            builder = builder.session(sess.clone());
-        }
-        let namespace = builder.connect().await.map_err(|e| Error::InvalidInput {
-            message: format!("Failed to connect to namespace: {:?}", e),
-        })?;
 
         Ok(Self {
             namespace,