mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-02 11:38:49 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Gatefixer
|
88d8a69a99 | ||
|
Gatefixer
|
bd779bb7d5 | ||
|
lancedb-gatefixer[bot]
|
7357d63e87 | ||
|
lancedb-gatefixer[bot]
|
624a75edf7 |
Generated
-297
@@ -3081,16 +3081,6 @@ version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9"
|
||||
|
||||
[[package]]
|
||||
name = "earcutr"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79127ed59a85d7687c409e9978547cffb7dc79675355ed22da6b66fd5f6ead01"
|
||||
dependencies = [
|
||||
"itertools 0.11.0",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ecdsa"
|
||||
version = "0.14.8"
|
||||
@@ -3390,12 +3380,6 @@ dependencies = [
|
||||
"rand_distr 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "float_next_after"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8"
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
@@ -3706,129 +3690,6 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geo"
|
||||
version = "0.31.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fc1a1678e54befc9b4bcab6cd43b8e7f834ae8ea121118b0fd8c42747675b4a"
|
||||
dependencies = [
|
||||
"earcutr",
|
||||
"float_next_after",
|
||||
"geo-types",
|
||||
"geographiclib-rs",
|
||||
"i_overlay",
|
||||
"log",
|
||||
"num-traits",
|
||||
"robust",
|
||||
"rstar",
|
||||
"spade",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geo-traits"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206"
|
||||
dependencies = [
|
||||
"geo-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geo-types"
|
||||
version = "0.7.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5"
|
||||
dependencies = [
|
||||
"approx",
|
||||
"num-traits",
|
||||
"rayon",
|
||||
"rstar",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geoarrow-array"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dafe7b7de3fab1a8b7099fd6a6434ca955fa65065f9c19f0f8a133693f3c2b0e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
"geo-traits",
|
||||
"geoarrow-schema",
|
||||
"num-traits",
|
||||
"wkb",
|
||||
"wkt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geoarrow-expr-geo"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e4a62ac19c86827c6ec81ea584594b3ee96db5a8119b9774d3466c6b373c434"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"geo",
|
||||
"geo-traits",
|
||||
"geoarrow-array",
|
||||
"geoarrow-schema",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geoarrow-schema"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d4a7edb2a1d87024a93805332a9c8184a0354836271d42c0d18cf628a5e3cd0"
|
||||
dependencies = [
|
||||
"arrow-schema",
|
||||
"geo-traits",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geodatafusion"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
"datafusion",
|
||||
"geo",
|
||||
"geo-traits",
|
||||
"geoarrow-array",
|
||||
"geoarrow-expr-geo",
|
||||
"geoarrow-schema",
|
||||
"geohash",
|
||||
"thiserror 1.0.69",
|
||||
"wkt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geographiclib-rs"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c5a7f08910fd98737a6eda7568e7c5e645093e073328eeef49758cfe8b0489c7"
|
||||
dependencies = [
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "geohash"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f58890382f70caccc5fa388981f7ac80c913795042afce9f3e065695d8f7464"
|
||||
dependencies = [
|
||||
"geo-types",
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
@@ -4004,15 +3865,6 @@ dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hash32"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
@@ -4069,16 +3921,6 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e"
|
||||
|
||||
[[package]]
|
||||
name = "heapless"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad"
|
||||
dependencies = [
|
||||
"hash32",
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
@@ -4393,49 +4235,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "i_float"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "010025c2c532c8d82e42d0b8bb5184afa449fa6f06c709ea9adcb16c49ae405b"
|
||||
dependencies = [
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "i_key_sort"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9190f86706ca38ac8add223b2aed8b1330002b5cdbbce28fb58b10914d38fc27"
|
||||
|
||||
[[package]]
|
||||
name = "i_overlay"
|
||||
version = "4.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "413183068e6e0289e18d7d0a1f661b81546e6918d5453a44570b9ab30cbed1b3"
|
||||
dependencies = [
|
||||
"i_float",
|
||||
"i_key_sort",
|
||||
"i_shape",
|
||||
"i_tree",
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "i_shape"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ea154b742f7d43dae2897fcd5ead86bc7b5eefcedd305a7ebf9f69d44d61082"
|
||||
dependencies = [
|
||||
"i_float",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "i_tree"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35e6d558e6d4c7b82bc51d9c771e7a927862a161a7d87bf2b0541450e0e20915"
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.65"
|
||||
@@ -5172,7 +4971,6 @@ dependencies = [
|
||||
"lance-arrow",
|
||||
"lance-core",
|
||||
"lance-datagen",
|
||||
"lance-geo",
|
||||
"log",
|
||||
"pin-project",
|
||||
"prost",
|
||||
@@ -5277,21 +5075,6 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-geo"
|
||||
version = "10.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.1.0-beta.1#68f4d4c1d0c4871b067557c61fc405078f1ab3b7"
|
||||
dependencies = [
|
||||
"datafusion",
|
||||
"geo-traits",
|
||||
"geo-types",
|
||||
"geoarrow-array",
|
||||
"geoarrow-schema",
|
||||
"geodatafusion",
|
||||
"lance-core",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "10.1.0-beta.1"
|
||||
@@ -5319,9 +5102,6 @@ dependencies = [
|
||||
"dirs",
|
||||
"fst",
|
||||
"futures",
|
||||
"geo-types",
|
||||
"geoarrow-array",
|
||||
"geoarrow-schema",
|
||||
"half",
|
||||
"itertools 0.14.0",
|
||||
"jieba-rs",
|
||||
@@ -5334,7 +5114,6 @@ dependencies = [
|
||||
"lance-datagen",
|
||||
"lance-encoding",
|
||||
"lance-file",
|
||||
"lance-geo",
|
||||
"lance-index-core",
|
||||
"lance-io",
|
||||
"lance-linalg",
|
||||
@@ -6520,28 +6299,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
|
||||
dependencies = [
|
||||
"num_enum_derive",
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum_derive"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "number_prefix"
|
||||
version = "0.4.0"
|
||||
@@ -8779,12 +8536,6 @@ dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "robust"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839"
|
||||
|
||||
[[package]]
|
||||
name = "rsa"
|
||||
version = "0.9.10"
|
||||
@@ -8806,17 +8557,6 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rstar"
|
||||
version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb"
|
||||
dependencies = [
|
||||
"heapless",
|
||||
"num-traits",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rstest"
|
||||
version = "0.23.0"
|
||||
@@ -9586,18 +9326,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spade"
|
||||
version = "2.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9699399fd9349b00b184f5635b074f9ec93afffef30c853f8c875b32c0f8c7fa"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"num-traits",
|
||||
"robust",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.9"
|
||||
@@ -11399,31 +11127,6 @@ dependencies = [
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wkb"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a120b336c7ad17749026d50427c23d838ecb50cd64aaea6254b5030152f890a9"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"geo-traits",
|
||||
"num_enum",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wkt"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "efb2b923ccc882312e559ffaa832a055ba9d1ac0cc8e86b3e25453247e4b81d7"
|
||||
dependencies = [
|
||||
"geo-traits",
|
||||
"geo-types",
|
||||
"log",
|
||||
"num-traits",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.3"
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=10.1.0-beta.1", default-features = false, features = ["geo"], "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=10.1.0-beta.1", default-features = false, "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=10.1.0-beta.1", "tag" = "v10.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
|
||||
@@ -707,6 +707,9 @@ class LanceDBConnection(DBConnection):
|
||||
self._namespace_client_properties = namespace_client_properties
|
||||
if _inner is not None:
|
||||
self._conn = _inner
|
||||
# Native-derived wrappers resolve this in their async reconstruction
|
||||
# path so construction never synchronously re-enters LOOP.
|
||||
self._read_consistency_interval = read_consistency_interval
|
||||
self._cached_namespace_client = None
|
||||
return
|
||||
|
||||
@@ -756,11 +759,14 @@ class LanceDBConnection(DBConnection):
|
||||
# storage_options. Also, this class really shouldn't be holding any state
|
||||
# beyond _conn.
|
||||
self._conn = AsyncConnection(LOOP.run(do_connect()))
|
||||
# Keep property access synchronous so debugger introspection cannot wait on
|
||||
# the background loop while that thread is suspended at a breakpoint.
|
||||
self._read_consistency_interval = read_consistency_interval
|
||||
self._cached_namespace_client: Optional[LanceNamespace] = None
|
||||
|
||||
@property
|
||||
def read_consistency_interval(self) -> Optional[timedelta]:
|
||||
return LOOP.run(self._conn.get_read_consistency_interval())
|
||||
return self._read_consistency_interval
|
||||
|
||||
@property
|
||||
def session(self) -> Optional[Session]:
|
||||
@@ -771,8 +777,16 @@ class LanceDBConnection(DBConnection):
|
||||
return self._conn.uri
|
||||
|
||||
@classmethod
|
||||
def from_inner(cls, inner: LanceDbConnection):
|
||||
return cls(None, _inner=inner)
|
||||
def from_inner(
|
||||
cls,
|
||||
inner: LanceDbConnection,
|
||||
read_consistency_interval: Optional[timedelta],
|
||||
):
|
||||
return cls(
|
||||
None,
|
||||
read_consistency_interval=read_consistency_interval,
|
||||
_inner=inner,
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(uri={self._conn.uri!r})"
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
|
||||
from typing import List
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -125,9 +126,20 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
|
||||
|
||||
@weak_lru(maxsize=1)
|
||||
def get_model(self):
|
||||
instructor_embedding = attempt_import_or_raise(
|
||||
"InstructorEmbedding", "InstructorEmbedding"
|
||||
)
|
||||
huggingface_hub = attempt_import_or_raise("huggingface_hub", "huggingface-hub")
|
||||
missing = object()
|
||||
original_cached_download = getattr(huggingface_hub, "cached_download", missing)
|
||||
if original_cached_download is missing:
|
||||
huggingface_hub.cached_download = _cached_download(huggingface_hub)
|
||||
|
||||
try:
|
||||
instructor_embedding = attempt_import_or_raise(
|
||||
"InstructorEmbedding", "InstructorEmbedding"
|
||||
)
|
||||
finally:
|
||||
if original_cached_download is missing:
|
||||
del huggingface_hub.cached_download
|
||||
|
||||
torch = attempt_import_or_raise("torch", "torch")
|
||||
|
||||
model = instructor_embedding.INSTRUCTOR(self.name)
|
||||
@@ -140,3 +152,44 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
|
||||
model, {torch.nn.Linear}, dtype=torch.qint8
|
||||
)
|
||||
return model
|
||||
|
||||
|
||||
def _cached_download(huggingface_hub):
|
||||
"""Provide the legacy download API used by sentence-transformers 2.2.x."""
|
||||
|
||||
def cached_download(
|
||||
*,
|
||||
url,
|
||||
cache_dir=None,
|
||||
force_filename=None,
|
||||
library_name=None,
|
||||
library_version=None,
|
||||
user_agent=None,
|
||||
use_auth_token=None,
|
||||
**_,
|
||||
):
|
||||
path = urlparse(url).path.lstrip("/")
|
||||
try:
|
||||
repo_id, resolved_path = path.split("/resolve/", maxsplit=1)
|
||||
revision, filename = resolved_path.split("/", maxsplit=1)
|
||||
except ValueError as err:
|
||||
raise ValueError(f"Unsupported Hugging Face Hub URL: {url}") from err
|
||||
|
||||
repo_id = unquote(repo_id)
|
||||
revision = unquote(revision)
|
||||
filename = unquote(filename)
|
||||
# sentence-transformers derives force_filename from this Hub path with
|
||||
# os.path.join. Using the URL path beneath local_dir produces the same
|
||||
# local destination without sending Windows separators to the Hub.
|
||||
return huggingface_hub.hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
filename=filename,
|
||||
revision=revision,
|
||||
local_dir=cache_dir,
|
||||
library_name=library_name,
|
||||
library_version=library_version,
|
||||
user_agent=user_agent,
|
||||
token=use_auth_token,
|
||||
)
|
||||
|
||||
return cached_download
|
||||
|
||||
@@ -226,7 +226,7 @@ class PermutationBuilder:
|
||||
|
||||
async def do_execute():
|
||||
inner_tbl = await self._async.execute()
|
||||
return LanceTable.from_inner(inner_tbl)
|
||||
return await LanceTable.from_inner(inner_tbl)
|
||||
|
||||
return LOOP.run(do_execute())
|
||||
|
||||
|
||||
@@ -2182,11 +2182,15 @@ class LanceTable(Table):
|
||||
return self.name
|
||||
|
||||
@classmethod
|
||||
def from_inner(cls, tbl: LanceDBTable):
|
||||
from .db import LanceDBConnection
|
||||
async def from_inner(cls, tbl: LanceDBTable):
|
||||
from .db import AsyncConnection, LanceDBConnection
|
||||
|
||||
async_tbl = AsyncTable(tbl)
|
||||
conn = LanceDBConnection.from_inner(tbl.database())
|
||||
inner_conn = tbl.database()
|
||||
read_consistency_interval = await AsyncConnection(
|
||||
inner_conn
|
||||
).get_read_consistency_interval()
|
||||
conn = LanceDBConnection.from_inner(inner_conn, read_consistency_interval)
|
||||
return cls(
|
||||
conn,
|
||||
async_tbl.name,
|
||||
|
||||
@@ -77,6 +77,23 @@ def test_sync_repr_does_not_use_background_loop(tmp_path, monkeypatch):
|
||||
assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
|
||||
|
||||
|
||||
def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkeypatch):
|
||||
from lancedb.background_loop import LOOP
|
||||
from lancedb.db import LanceDBConnection
|
||||
|
||||
consistency_interval = timedelta(seconds=5)
|
||||
db = lancedb.connect(tmp_path, read_consistency_interval=consistency_interval)
|
||||
db_from_inner = LanceDBConnection.from_inner(db._inner, consistency_interval)
|
||||
|
||||
def fail_run(*args, **kwargs):
|
||||
raise AssertionError("properties should not use the Python background loop")
|
||||
|
||||
monkeypatch.setattr(LOOP, "run", fail_run)
|
||||
|
||||
assert db.read_consistency_interval == consistency_interval
|
||||
assert db_from_inner.read_consistency_interval == consistency_interval
|
||||
|
||||
|
||||
def test_ingest_pd(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import ntpath
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
from types import ModuleType
|
||||
from typing import List, Optional, Union
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -522,6 +525,59 @@ def test_embedding_function_safe_model_dump(embedding_type):
|
||||
)
|
||||
|
||||
|
||||
def test_instructor_embedding_supports_huggingface_hub_without_cached_download(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
from lancedb.embeddings.instructor import InstructorEmbeddingFunction
|
||||
|
||||
hub_download = MagicMock(return_value="/cache/1_Pooling/config.json")
|
||||
huggingface_hub = ModuleType("huggingface_hub")
|
||||
huggingface_hub.hf_hub_download = hub_download
|
||||
torch = ModuleType("torch")
|
||||
monkeypatch.setitem(sys.modules, "huggingface_hub", huggingface_hub)
|
||||
monkeypatch.setitem(sys.modules, "torch", torch)
|
||||
monkeypatch.delitem(sys.modules, "InstructorEmbedding", raising=False)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
|
||||
(tmp_path / "InstructorEmbedding.py").write_text(
|
||||
"from huggingface_hub import cached_download\n\n"
|
||||
"class INSTRUCTOR:\n"
|
||||
" def __init__(self, name):\n"
|
||||
" self.name = name\n"
|
||||
)
|
||||
|
||||
embedding = InstructorEmbeddingFunction.create(show_progress_bar=False)
|
||||
instructor_model = embedding.get_model()
|
||||
|
||||
assert instructor_model.name == "hkunlp/instructor-base"
|
||||
assert not hasattr(huggingface_hub, "cached_download")
|
||||
|
||||
instructor_embedding = sys.modules["InstructorEmbedding"]
|
||||
path = instructor_embedding.cached_download(
|
||||
url=(
|
||||
"https://huggingface.co/hkunlp/instructor-base/resolve/abc123/"
|
||||
"1_Pooling/config.json"
|
||||
),
|
||||
cache_dir="/cache",
|
||||
force_filename=ntpath.join("1_Pooling", "config.json"),
|
||||
library_name="sentence-transformers",
|
||||
library_version="2.2.2",
|
||||
use_auth_token="token",
|
||||
)
|
||||
|
||||
assert path == "/cache/1_Pooling/config.json"
|
||||
hub_download.assert_called_once_with(
|
||||
repo_id="hkunlp/instructor-base",
|
||||
filename="1_Pooling/config.json",
|
||||
revision="abc123",
|
||||
local_dir="/cache",
|
||||
library_name="sentence-transformers",
|
||||
library_version="2.2.2",
|
||||
user_agent=None,
|
||||
token="token",
|
||||
)
|
||||
|
||||
|
||||
@patch("time.sleep")
|
||||
def test_retry(mock_sleep):
|
||||
test_function = MagicMock(side_effect=[Exception] * 9 + ["result"])
|
||||
|
||||
@@ -6,6 +6,7 @@ import math
|
||||
import pytest
|
||||
|
||||
from lancedb import DBConnection, Table, connect
|
||||
from lancedb.background_loop import LOOP
|
||||
from lancedb.permutation import Permutation, Permutations, permutation_builder
|
||||
|
||||
|
||||
@@ -31,6 +32,25 @@ def test_split_random_ratios(mem_db):
|
||||
assert 65 <= split_1_count <= 75 # ~70% ± tolerance
|
||||
|
||||
|
||||
def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch):
|
||||
import threading
|
||||
|
||||
db = connect(tmp_path)
|
||||
tbl = db.create_table("test_table", pa.table({"x": range(10)}))
|
||||
original_run = LOOP.run
|
||||
|
||||
def fail_on_reentry(future):
|
||||
assert threading.current_thread() is not LOOP.thread
|
||||
return original_run(future)
|
||||
|
||||
monkeypatch.setattr(LOOP, "run", fail_on_reentry)
|
||||
|
||||
permutation_tbl = permutation_builder(tbl).execute()
|
||||
|
||||
assert permutation_tbl.count_rows() == 10
|
||||
assert permutation_tbl._conn.read_consistency_interval is None
|
||||
|
||||
|
||||
def test_split_random_counts(mem_db):
|
||||
"""Test random splitting with absolute counts."""
|
||||
tbl = mem_db.create_table(
|
||||
|
||||
@@ -6,6 +6,7 @@ import os
|
||||
import sys
|
||||
import threading
|
||||
import warnings
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, datetime, timedelta
|
||||
from time import sleep
|
||||
from typing import List
|
||||
@@ -2124,6 +2125,27 @@ def test_delete(mem_db: DBConnection):
|
||||
assert table.to_arrow()["id"].to_pylist() == [1]
|
||||
|
||||
|
||||
def test_concurrent_deletes_are_thread_safe(mem_db: DBConnection):
|
||||
num_workers = 8
|
||||
table = mem_db.create_table(
|
||||
"my_table", data=[{"id": row_id} for row_id in range(num_workers)]
|
||||
)
|
||||
barrier = threading.Barrier(num_workers)
|
||||
|
||||
def delete(row_id: int):
|
||||
barrier.wait()
|
||||
return table.delete(f"id = {row_id}")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=num_workers) as pool:
|
||||
results = list(pool.map(delete, range(num_workers)))
|
||||
|
||||
assert all(result.num_deleted_rows == 1 for result in results)
|
||||
assert sorted(result.version for result in results) == list(
|
||||
range(2, num_workers + 2)
|
||||
)
|
||||
assert table.count_rows() == 0
|
||||
|
||||
|
||||
def test_delete_expr(mem_db: DBConnection):
|
||||
table = mem_db.create_table(
|
||||
"my_table",
|
||||
|
||||
@@ -745,6 +745,9 @@ impl Table {
|
||||
|
||||
#[allow(private_interfaces)]
|
||||
pub fn delete(self_: PyRef<'_, Self>, condition: PredicateArg) -> PyResult<Bound<'_, PyAny>> {
|
||||
// Do not hold the Python borrow across the await. The cloned Rust table
|
||||
// handle is thread-safe and allows deletes on the same Python table to
|
||||
// run concurrently without PyO3 reporting "Already borrowed".
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let result = match &condition {
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::{
|
||||
};
|
||||
|
||||
mod hybrid;
|
||||
mod sql;
|
||||
|
||||
pub(crate) const DEFAULT_TOP_K: usize = 10;
|
||||
|
||||
@@ -562,8 +561,8 @@ impl<T: HasQuery> QueryBase for T {
|
||||
}
|
||||
|
||||
fn only_if(mut self, filter: impl AsRef<str>) -> Self {
|
||||
let filter = sql::rewrite_st_dwithin(filter.as_ref());
|
||||
self.mut_query().add_filter(QueryFilter::Sql(filter));
|
||||
self.mut_query()
|
||||
.add_filter(QueryFilter::Sql(filter.as_ref().to_string()));
|
||||
self
|
||||
}
|
||||
|
||||
@@ -1649,8 +1648,8 @@ mod tests {
|
||||
use super::*;
|
||||
use arrow::{array::downcast_array, compute::concat_batches, datatypes::Int32Type};
|
||||
use arrow_array::{
|
||||
FixedSizeListArray, Float32Array, Float64Array, Int32Array, RecordBatch, StringArray,
|
||||
cast::AsArray, types::Float32Type,
|
||||
FixedSizeListArray, Float32Array, Int32Array, RecordBatch, StringArray, cast::AsArray,
|
||||
types::Float32Type,
|
||||
};
|
||||
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
@@ -2025,46 +2024,6 @@ mod tests {
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_st_dwithin_filter() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let uri = tmp_dir.path().to_str().unwrap();
|
||||
let batch = RecordBatch::try_from_iter([
|
||||
("id", Arc::new(Int32Array::from_iter_values([1, 2, 3])) as _),
|
||||
(
|
||||
"x",
|
||||
Arc::new(Float64Array::from_iter_values([0.0, 1.0, 3.0])) as _,
|
||||
),
|
||||
(
|
||||
"y",
|
||||
Arc::new(Float64Array::from_iter_values([0.0, 1.0, 4.0])) as _,
|
||||
),
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let table = connect(uri)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.create_table("points", batch)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let batches = table
|
||||
.query()
|
||||
.only_if("ST_DWithin(ST_Point(x, y), ST_Point(0.0, 0.0), 2.0)")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
let batch = concat_batches(&batches[0].schema(), &batches).unwrap();
|
||||
let ids = batch["id"].as_primitive::<Int32Type>();
|
||||
|
||||
assert_eq!(ids.values(), &[1, 2]);
|
||||
}
|
||||
|
||||
fn make_non_empty_batches() -> Box<dyn arrow_array::RecordBatchReader + Send> {
|
||||
let vec = Box::new(RandomVector::new().named("vector".to_string()));
|
||||
let id = Box::new(IncrementingInt32::new().named("id".to_string()));
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! SQL compatibility helpers for query filters.
|
||||
|
||||
use std::{any::TypeId, ops::ControlFlow};
|
||||
|
||||
use datafusion_sql::sqlparser::{
|
||||
ast::{
|
||||
BinaryOperator, Expr, FunctionArg, FunctionArgExpr, FunctionArguments, Ident, ObjectName,
|
||||
SelectItem, SetExpr, Statement, visit_expressions_mut,
|
||||
},
|
||||
dialect::{Dialect, GenericDialect},
|
||||
parser::Parser,
|
||||
tokenizer::{Token, Tokenizer},
|
||||
};
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct LanceDialect(GenericDialect);
|
||||
|
||||
impl Dialect for LanceDialect {
|
||||
fn dialect(&self) -> TypeId {
|
||||
self.0.dialect()
|
||||
}
|
||||
|
||||
fn is_identifier_start(&self, ch: char) -> bool {
|
||||
self.0.is_identifier_start(ch)
|
||||
}
|
||||
|
||||
fn is_identifier_part(&self, ch: char) -> bool {
|
||||
self.0.is_identifier_part(ch)
|
||||
}
|
||||
|
||||
fn is_delimited_identifier_start(&self, ch: char) -> bool {
|
||||
ch == '`'
|
||||
}
|
||||
}
|
||||
|
||||
/// Rewrite `ST_DWithin(a, b, distance)` as `ST_Distance(a, b) <= distance`.
|
||||
///
|
||||
/// GeoDataFusion does not currently expose `ST_DWithin`, but its definition is
|
||||
/// exactly this distance comparison. Invalid SQL and unsupported function
|
||||
/// shapes are left untouched so the query planner can report the usual error.
|
||||
pub(super) fn rewrite_st_dwithin(filter: &str) -> String {
|
||||
if !filter.to_ascii_lowercase().contains("st_dwithin") {
|
||||
return filter.to_string();
|
||||
}
|
||||
|
||||
let Some(mut expr) = parse_filter(filter) else {
|
||||
return filter.to_string();
|
||||
};
|
||||
let mut rewritten = false;
|
||||
let _ = visit_expressions_mut(&mut expr, |expr| {
|
||||
let Expr::Function(function) = expr else {
|
||||
return ControlFlow::<()>::Continue(());
|
||||
};
|
||||
if !function.name.to_string().eq_ignore_ascii_case("st_dwithin") {
|
||||
return ControlFlow::Continue(());
|
||||
}
|
||||
let FunctionArguments::List(arguments) = &function.args else {
|
||||
return ControlFlow::Continue(());
|
||||
};
|
||||
let [
|
||||
FunctionArg::Unnamed(FunctionArgExpr::Expr(first)),
|
||||
FunctionArg::Unnamed(FunctionArgExpr::Expr(second)),
|
||||
FunctionArg::Unnamed(FunctionArgExpr::Expr(distance)),
|
||||
] = arguments.args.as_slice()
|
||||
else {
|
||||
return ControlFlow::Continue(());
|
||||
};
|
||||
|
||||
let mut st_distance = function.clone();
|
||||
st_distance.name = ObjectName::from(vec![Ident::new("ST_Distance")]);
|
||||
if let FunctionArguments::List(arguments) = &mut st_distance.args {
|
||||
arguments.args = vec![
|
||||
FunctionArg::Unnamed(FunctionArgExpr::Expr(first.clone())),
|
||||
FunctionArg::Unnamed(FunctionArgExpr::Expr(second.clone())),
|
||||
];
|
||||
}
|
||||
*expr = Expr::BinaryOp {
|
||||
left: Box::new(Expr::Function(st_distance)),
|
||||
op: BinaryOperator::LtEq,
|
||||
right: Box::new(distance.clone()),
|
||||
};
|
||||
rewritten = true;
|
||||
ControlFlow::Continue(())
|
||||
});
|
||||
|
||||
if rewritten {
|
||||
expr.to_string()
|
||||
} else {
|
||||
filter.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_filter(filter: &str) -> Option<Expr> {
|
||||
let statement = format!("SELECT 1 FROM t WHERE {filter}");
|
||||
let dialect = LanceDialect::default();
|
||||
let mut token_iter = Tokenizer::new(&dialect, &statement)
|
||||
.tokenize()
|
||||
.ok()?
|
||||
.into_iter();
|
||||
let mut previous = token_iter.next()?;
|
||||
let mut tokens = Vec::new();
|
||||
|
||||
// Match Lance's support for `==` as an equality operator.
|
||||
for next in token_iter {
|
||||
if let (Token::Eq, Token::Eq) = (&previous, &next) {
|
||||
continue;
|
||||
}
|
||||
tokens.push(std::mem::replace(&mut previous, next));
|
||||
}
|
||||
tokens.push(previous);
|
||||
|
||||
let statement = Parser::new(&dialect)
|
||||
.with_tokens(tokens)
|
||||
.parse_statement()
|
||||
.ok()?;
|
||||
if let Statement::Query(query) = statement
|
||||
&& let SetExpr::Select(select) = *query.body
|
||||
&& let Some(expr) = select.selection
|
||||
&& matches!(select.projection.as_slice(), [SelectItem::UnnamedExpr(_)])
|
||||
{
|
||||
Some(expr)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rewrites_nested_and_case_insensitive_dwithin() {
|
||||
assert_eq!(
|
||||
rewrite_st_dwithin("id == 1 AND st_dwithin(ST_Point(x, y), ST_Point(0, 0), radius)"),
|
||||
"id = 1 AND ST_Distance(ST_Point(x, y), ST_Point(0, 0)) <= radius"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_other_filters_unchanged() {
|
||||
assert_eq!(rewrite_st_dwithin("id == 1"), "id == 1");
|
||||
assert_eq!(
|
||||
rewrite_st_dwithin("ST_DWithin(point, origin)"),
|
||||
"ST_DWithin(point, origin)"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user