Compare commits

..

1 Commits

Author SHA1 Message Date
ayush chaurasia
5904aec34b update 2025-10-27 14:39:22 +05:30
9 changed files with 87 additions and 83 deletions

74
Cargo.lock generated
View File

@@ -3044,8 +3044,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow-array",
"rand 0.9.2",
@@ -4229,8 +4229,8 @@ dependencies = [
[[package]]
name = "lance"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow",
"arrow-arith",
@@ -4293,8 +4293,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4312,8 +4312,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrayref",
"paste",
@@ -4322,8 +4322,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4359,8 +4359,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow",
"arrow-array",
@@ -4389,8 +4389,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow",
"arrow-array",
@@ -4407,8 +4407,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -4445,8 +4445,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -4479,8 +4479,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow",
"arrow-arith",
@@ -4542,8 +4542,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow",
"arrow-arith",
@@ -4583,8 +4583,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4607,8 +4607,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow",
"async-trait",
@@ -4620,8 +4620,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow",
"arrow-ipc",
@@ -4630,9 +4630,8 @@ dependencies = [
"bytes",
"lance",
"lance-core",
"lance-io",
"lance-namespace",
"object_store",
"opendal",
"reqwest",
"serde_json",
"snafu",
@@ -4654,8 +4653,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow",
"arrow-array",
@@ -4693,8 +4692,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "0.38.3-beta.11"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.11#db497cb2373156679aa0d6a1f2087880a8579bc6"
version = "0.38.3-beta.9"
source = "git+https://github.com/lancedb/lance.git?tag=v0.38.3-beta.9#5f603515786cdc3b5aadb3313131bf686d5e932b"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5182,9 +5181,12 @@ dependencies = [
[[package]]
name = "mock_instant"
version = "0.6.0"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6"
checksum = "9366861eb2a2c436c20b12c8dbec5f798cea6b47ad99216be0282942e2c81ea0"
dependencies = [
"once_cell",
]
[[package]]
name = "moka"

View File

@@ -15,20 +15,20 @@ categories = ["database-implementations"]
rust-version = "1.78.0"
[workspace.dependencies]
lance = { "version" = "=0.38.3-beta.11", default-features = false, "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-core = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-datagen = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-file = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-io = { "version" = "=0.38.3-beta.11", default-features = false, "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-index = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-linalg = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-namespace = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-namespace-impls = { "version" = "=0.38.3-beta.11", "features" = ["dir-aws", "dir-gcp", "dir-azure", "dir-oss", "rest"], "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-table = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-testing = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-datafusion = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-encoding = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance-arrow = { "version" = "=0.38.3-beta.11", "tag" = "v0.38.3-beta.11", "git" = "https://github.com/lancedb/lance.git" }
lance = { "version" = "=0.38.3-beta.9", default-features = false, "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-core = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-datagen = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-file = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-io = { "version" = "=0.38.3-beta.9", default-features = false, "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-index = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-linalg = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-namespace = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-namespace-impls = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-table = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-testing = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-datafusion = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-encoding = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
lance-arrow = { "version" = "=0.38.3-beta.9", "tag" = "v0.38.3-beta.9", "git" = "https://github.com/lancedb/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "56.2", optional = false }
@@ -65,4 +65,4 @@ semver = "1.0.25"
crunchy = "0.2.4"
chrono = "0.4"
# Workaround for: https://github.com/Lokathor/bytemuck/issues/306
bytemuck_derive = ">=1.8.1, <1.9.0"
bytemuck_derive = ">=1.8.1, <1.9.0"

View File

@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.25.3-beta.3"
current_version = "0.25.3-beta.2"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.

View File

@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.25.3-beta.3"
version = "0.25.3-beta.2"
edition.workspace = true
description = "Python bindings for LanceDB"
license.workspace = true

View File

@@ -4,7 +4,7 @@
import os
from functools import cached_property
from typing import List, Union
from typing import List, Optional, Union
import numpy as np
@@ -46,10 +46,11 @@ class GeminiText(TextEmbeddingFunction):
Parameters
----------
name: str, default "models/embedding-001"
name: str, default "models/text-embedding-004"
The name of the model to use. See the Gemini documentation for a list of
available models.
dims: int, optional
The dimension of the embedding, otherwise it will be inferred.
query_task_type: str, default "retrieval_query"
Sets the task type for the queries.
source_task_type: str, default "retrieval_document"
@@ -77,9 +78,10 @@ class GeminiText(TextEmbeddingFunction):
"""
name: str = "models/embedding-001"
name: str = "models/text-embedding-004"
query_task_type: str = "retrieval_query"
source_task_type: str = "retrieval_document"
dims: Optional[int] = None
if PYDANTIC_VERSION.major < 2: # Pydantic 1.x compat
@@ -89,9 +91,18 @@ class GeminiText(TextEmbeddingFunction):
model_config = dict()
model_config["ignored_types"] = (cached_property,)
def ndims(self):
# TODO: fix hardcoding
return 768
@cached_property
def _model(self):
return self.client.get_model(self.name)
def ndims(self) -> int:
if self.dims:
return self.dims
if hasattr(self._model, "output_dimensionality"):
return self._model.output_dimensionality
# Fallback for older versions of the library
# or models that don't have the attribute
return len(self.generate_embeddings(["lancedb"])[0])
def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
return self.compute_source_embeddings(query, task_type=self.query_task_type)
@@ -119,6 +130,8 @@ class GeminiText(TextEmbeddingFunction):
): # Provide a title to use existing API design
title = "Embedding of a document"
kwargs["title"] = title
if self.dims:
kwargs["output_dimensionality"] = self.dims
return [
self.client.embed_content(model=self.name, content=text, **kwargs)[
@@ -131,6 +144,8 @@ class GeminiText(TextEmbeddingFunction):
def client(self):
genai = attempt_import_or_raise("google.generativeai", "google.generativeai")
if not os.environ.get("GOOGLE_API_KEY"):
api_key = os.environ.get("GOOGLE_API_KEY")
if not api_key:
api_key_not_found_help("google")
genai.configure(api_key=api_key)
return genai

View File

@@ -308,7 +308,7 @@ def test_instructor_embedding(tmp_path):
os.environ.get("GOOGLE_API_KEY") is None, reason="GOOGLE_API_KEY not set"
)
def test_gemini_embedding(tmp_path):
model = get_registry().get("gemini-text").create(max_retries=0)
model = get_registry().get("gemini-text").create(max_retries=0, dims=512)
class TextModel(LanceModel):
text: str = model.SourceField()
@@ -319,7 +319,7 @@ def test_gemini_embedding(tmp_path):
tbl = db.create_table("test", schema=TextModel, mode="overwrite")
tbl.add(df)
assert len(tbl.to_pandas()["vector"][0]) == model.ndims()
assert len(tbl.to_pandas()["vector"][0]) == model.ndims() == 512
assert tbl.search("hello").limit(1).to_pandas()["text"][0] == "hello world"

View File

@@ -59,14 +59,6 @@ class TempNamespace(LanceNamespace):
root
] # Reference to shared namespaces
def namespace_id(self) -> str:
"""Return a human-readable unique identifier for this namespace instance.
Returns:
A unique identifier string based on the root directory
"""
return f"TempNamespace {{ root: '{self.config.root}' }}"
def list_tables(self, request: ListTablesRequest) -> ListTablesResponse:
"""List all tables in the namespace."""
if not request.id:

View File

@@ -44,7 +44,7 @@ lance-testing = { workspace = true }
lance-encoding = { workspace = true }
lance-arrow = { workspace = true }
lance-namespace = { workspace = true }
lance-namespace-impls = { workspace = true }
lance-namespace-impls = { workspace = true, features = ["dir", "rest"] }
moka = { workspace = true }
pin-project = { workspace = true }
tokio = { version = "1.23", features = ["rt-multi-thread"] }

View File

@@ -14,7 +14,7 @@ use lance_namespace::{
},
LanceNamespace,
};
use lance_namespace_impls::ConnectBuilder;
use lance_namespace_impls::connect::connect as connect_namespace;
use crate::database::listing::ListingDatabase;
use crate::error::{Error, Result};
@@ -48,16 +48,11 @@ impl LanceNamespaceDatabase {
read_consistency_interval: Option<std::time::Duration>,
session: Option<Arc<lance::session::Session>>,
) -> Result<Self> {
let mut builder = ConnectBuilder::new(ns_impl);
for (key, value) in ns_properties.clone() {
builder = builder.property(key, value);
}
if let Some(ref sess) = session {
builder = builder.session(sess.clone());
}
let namespace = builder.connect().await.map_err(|e| Error::InvalidInput {
message: format!("Failed to connect to namespace: {:?}", e),
})?;
let namespace = connect_namespace(ns_impl, ns_properties.clone())
.await
.map_err(|e| Error::InvalidInput {
message: format!("Failed to connect to namespace: {:?}", e),
})?;
Ok(Self {
namespace,