fix(python): scope Instructor compatibility shim

Cargo.lock

@@ -5442,7 +5442,6 @@ dependencies = [
  "pprof 0.14.1",
  "rand 0.9.5",
  "random_word",
- "rayon",
  "regex",
  "reqwest 0.12.28",
  "rstest",

Cargo.toml

@@ -60,7 +60,6 @@ moka = { version = "0.12", features = ["future"] }
 object_store = "0.13.2"
 pin-project = "1.0.7"
 rand = "0.9"
-rayon = "1"
 snafu = "0.8"
 url = "2"
 num-traits = "0.2"

python/python/lancedb/embeddings/instructor.py

@@ -3,6 +3,7 @@
 
 
 from typing import List
+from urllib.parse import unquote, urlparse
 
 import numpy as np
 
@@ -125,9 +126,20 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
 
     @weak_lru(maxsize=1)
     def get_model(self):
-        instructor_embedding = attempt_import_or_raise(
-            "InstructorEmbedding", "InstructorEmbedding"
-        )
+        huggingface_hub = attempt_import_or_raise("huggingface_hub", "huggingface-hub")
+        missing = object()
+        original_cached_download = getattr(huggingface_hub, "cached_download", missing)
+        if original_cached_download is missing:
+            huggingface_hub.cached_download = _cached_download(huggingface_hub)
+
+        try:
+            instructor_embedding = attempt_import_or_raise(
+                "InstructorEmbedding", "InstructorEmbedding"
+            )
+        finally:
+            if original_cached_download is missing:
+                del huggingface_hub.cached_download
+
         torch = attempt_import_or_raise("torch", "torch")
 
         model = instructor_embedding.INSTRUCTOR(self.name)
@@ -140,3 +152,44 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
                 model, {torch.nn.Linear}, dtype=torch.qint8
             )
         return model
+
+
+def _cached_download(huggingface_hub):
+    """Provide the legacy download API used by sentence-transformers 2.2.x."""
+
+    def cached_download(
+        *,
+        url,
+        cache_dir=None,
+        force_filename=None,
+        library_name=None,
+        library_version=None,
+        user_agent=None,
+        use_auth_token=None,
+        **_,
+    ):
+        path = urlparse(url).path.lstrip("/")
+        try:
+            repo_id, resolved_path = path.split("/resolve/", maxsplit=1)
+            revision, filename = resolved_path.split("/", maxsplit=1)
+        except ValueError as err:
+            raise ValueError(f"Unsupported Hugging Face Hub URL: {url}") from err
+
+        repo_id = unquote(repo_id)
+        revision = unquote(revision)
+        filename = unquote(filename)
+        # sentence-transformers derives force_filename from this Hub path with
+        # os.path.join. Using the URL path beneath local_dir produces the same
+        # local destination without sending Windows separators to the Hub.
+        return huggingface_hub.hf_hub_download(
+            repo_id=repo_id,
+            filename=filename,
+            revision=revision,
+            local_dir=cache_dir,
+            library_name=library_name,
+            library_version=library_version,
+            user_agent=user_agent,
+            token=use_auth_token,
+        )
+
+    return cached_download

python/python/lancedb/index.py

@@ -769,13 +769,6 @@ class IvfPq:
 
         The default value is 256.
 
-    seed: int, optional
-        Seed used for deterministic sampling and training. Given identical data in
-        the same row order and identical index parameters, the same seed produces
-        the same IVF centroids and PQ codebook. This option is supported for local
-        CPU index builds; remote and accelerator builds reject it explicitly. If
-        omitted, training remains random.
-
     target_partition_size: int, default is 8192
 
         The target size of each partition.
@@ -790,18 +783,11 @@ class IvfPq:
     num_bits: int = 8
     max_iterations: int = 50
     sample_rate: int = 256
-    seed: Optional[int] = None
     target_partition_size: Optional[int] = None
     # Name of the accelerator (e.g. "cuda") to use for IVF training. When set,
     # create_index() dispatches to pylance to build the index on the accelerator.
     accelerator: Optional[str] = None
 
-    def __post_init__(self):
-        if self.seed is not None and self.accelerator is not None:
-            raise ValueError(
-                "IvfPq seed is not supported with accelerator-based index training"
-            )
-
 
 @dataclass
 class IvfRq:

python/python/lancedb/table.py

@@ -2762,11 +2762,6 @@ class LanceTable(Table):
             if config is not None and hasattr(config, "accelerator"):
                 acc = getattr(config, "accelerator", None)
                 if acc is not None:
-                    if isinstance(config, IvfPq) and config.seed is not None:
-                        raise ValueError(
-                            "IvfPq seed is not supported with accelerator-based "
-                            "index training"
-                        )
                     # Dispatch to pylance for GPU acceleration
                     index_type_map = {
                         "IvfFlat": "IVF_FLAT",

python/python/tests/test_embeddings.py

@@ -1,8 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+import ntpath
 import os
 import pickle
+import sys
+from types import ModuleType
 from typing import List, Optional, Union
 from unittest.mock import MagicMock, patch
 
@@ -522,6 +525,59 @@ def test_embedding_function_safe_model_dump(embedding_type):
             )
 
 
+def test_instructor_embedding_supports_huggingface_hub_without_cached_download(
+    tmp_path, monkeypatch
+):
+    from lancedb.embeddings.instructor import InstructorEmbeddingFunction
+
+    hub_download = MagicMock(return_value="/cache/1_Pooling/config.json")
+    huggingface_hub = ModuleType("huggingface_hub")
+    huggingface_hub.hf_hub_download = hub_download
+    torch = ModuleType("torch")
+    monkeypatch.setitem(sys.modules, "huggingface_hub", huggingface_hub)
+    monkeypatch.setitem(sys.modules, "torch", torch)
+    monkeypatch.delitem(sys.modules, "InstructorEmbedding", raising=False)
+    monkeypatch.syspath_prepend(str(tmp_path))
+
+    (tmp_path / "InstructorEmbedding.py").write_text(
+        "from huggingface_hub import cached_download\n\n"
+        "class INSTRUCTOR:\n"
+        "    def __init__(self, name):\n"
+        "        self.name = name\n"
+    )
+
+    embedding = InstructorEmbeddingFunction.create(show_progress_bar=False)
+    instructor_model = embedding.get_model()
+
+    assert instructor_model.name == "hkunlp/instructor-base"
+    assert not hasattr(huggingface_hub, "cached_download")
+
+    instructor_embedding = sys.modules["InstructorEmbedding"]
+    path = instructor_embedding.cached_download(
+        url=(
+            "https://huggingface.co/hkunlp/instructor-base/resolve/abc123/"
+            "1_Pooling/config.json"
+        ),
+        cache_dir="/cache",
+        force_filename=ntpath.join("1_Pooling", "config.json"),
+        library_name="sentence-transformers",
+        library_version="2.2.2",
+        use_auth_token="token",
+    )
+
+    assert path == "/cache/1_Pooling/config.json"
+    hub_download.assert_called_once_with(
+        repo_id="hkunlp/instructor-base",
+        filename="1_Pooling/config.json",
+        revision="abc123",
+        local_dir="/cache",
+        library_name="sentence-transformers",
+        library_version="2.2.2",
+        user_agent=None,
+        token="token",
+    )
+
+
 @patch("time.sleep")
 def test_retry(mock_sleep):
     test_function = MagicMock(side_effect=[Exception] * 9 + ["result"])

python/python/tests/test_index.py

@@ -26,7 +26,7 @@ from lancedb.index import (
     HnswFlat,
     FTS,
 )
-from lancedb.table import IndexStatistics, LanceTable
+from lancedb.table import IndexStatistics
 
 
 @pytest_asyncio.fixture
@@ -375,7 +375,7 @@ async def test_create_vector_index(some_table: AsyncTable):
 @pytest.mark.asyncio
 async def test_create_4bit_ivfpq_index(some_table: AsyncTable):
     # Can create
-    await some_table.create_index("vector", config=IvfPq(num_bits=4, seed=42))
+    await some_table.create_index("vector", config=IvfPq(num_bits=4))
     # Can recreate if replace=True
     await some_table.create_index("vector", config=IvfPq(num_bits=4), replace=True)
     # Can't recreate if replace=False
@@ -395,16 +395,6 @@ async def test_create_4bit_ivfpq_index(some_table: AsyncTable):
     assert stats.num_indices == 1
 
 
-def test_seeded_ivfpq_rejects_accelerator():
-    with pytest.raises(ValueError, match="seed is not supported with accelerator"):
-        IvfPq(seed=42, accelerator="cuda")
-
-    config = IvfPq(seed=42)
-    config.accelerator = "cuda"
-    with pytest.raises(ValueError, match="seed is not supported with accelerator"):
-        object.__new__(LanceTable).create_index("vector", config=config)
-
-
 @pytest.mark.asyncio
 async def test_create_ivfrq_index(some_table: AsyncTable):
     await some_table.create_index("vector", config=IvfRq(num_bits=1))

python/src/index.rs

@@ -90,9 +90,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .max_iterations(params.max_iterations)
                     .sample_rate(params.sample_rate)
                     .num_bits(params.num_bits);
-                if let Some(seed) = params.seed {
-                    ivf_pq_builder = ivf_pq_builder.seed(seed);
-                }
                 if let Some(num_partitions) = params.num_partitions {
                     ivf_pq_builder = ivf_pq_builder.num_partitions(num_partitions);
                 }
@@ -235,7 +232,6 @@ struct IvfPqParams {
     num_bits: u32,
     max_iterations: u32,
     sample_rate: u32,
-    seed: Option<u64>,
     target_partition_size: Option<u32>,
 }
 

rust/lancedb/Cargo.toml

@@ -61,7 +61,6 @@ futures.workspace = true
 num-traits.workspace = true
 url.workspace = true
 rand.workspace = true
-rayon.workspace = true
 regex.workspace = true
 serde = { version = "^1" }
 serde_json = { version = "1" }

rust/lancedb/src/index/vector.rs

@@ -274,8 +274,6 @@ pub struct IvfPqIndexBuilder {
     pub(crate) sample_rate: u32,
     pub(crate) max_iterations: u32,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub(crate) seed: Option<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) target_partition_size: Option<u32>,
 
     // PQ
@@ -294,7 +292,6 @@ impl Default for IvfPqIndexBuilder {
             num_bits: None,
             sample_rate: 256,
             max_iterations: 50,
-            seed: None,
             target_partition_size: None,
         }
     }
@@ -304,30 +301,6 @@ impl IvfPqIndexBuilder {
     impl_distance_type_setter!();
     impl_ivf_params_setter!();
     impl_pq_params_setter!();
-
-    /// Use a deterministic seed when sampling and training the IVF and PQ models.
-    ///
-    /// Given identical data in the same row order and identical index parameters,
-    /// using the same seed produces the same IVF centroids and PQ codebook. This is
-    /// useful when independently-built tables need reproducible approximate-search
-    /// results.
-    ///
-    /// Seeded training is supported by native tables. Remote backends reject this
-    /// option unless they can provide the same deterministic training contract.
-    ///
-    /// If no seed is provided, index training uses random sampling and initialization.
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// use lancedb::index::vector::IvfPqIndexBuilder;
-    ///
-    /// let index = IvfPqIndexBuilder::default().seed(42);
-    /// ```
-    pub fn seed(mut self, seed: u64) -> Self {
-        self.seed = Some(seed);
-        self
-    }
 }
 
 pub(crate) fn suggested_num_sub_vectors(dim: u32) -> u32 {

rust/lancedb/src/remote/table.rs

@@ -339,12 +339,6 @@ impl<S: HttpSend> RemoteTable<S> {
         // Auto is special-cased since it needs schema inspection.
         let (index_type_str, params) = match &index.index {
             Index::IvfFlat(p) => ("IVF_FLAT", Some(to_json(p)?)),
-            Index::IvfPq(p) if p.seed.is_some() => {
-                return Err(Error::NotSupported {
-                    message: "Deterministic IVF PQ training is not supported on remote tables"
-                        .to_string(),
-                });
-            }
             Index::IvfPq(p) => ("IVF_PQ", Some(to_json(p)?)),
             Index::IvfSq(p) => ("IVF_SQ", Some(to_json(p)?)),
             Index::IvfHnswSq(p) => ("IVF_HNSW_SQ", Some(to_json(p)?)),
@@ -5163,41 +5157,6 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn test_seeded_ivf_pq_is_rejected_for_remote_tables() {
-        let table = Table::new_with_handler("my_table", |request| match request.url().path() {
-            "/v1/table/my_table/describe/" => {
-                let schema = Schema::new(vec![Field::new(
-                    "vector",
-                    DataType::FixedSizeList(
-                        Arc::new(Field::new("item", DataType::Float32, true)),
-                        8,
-                    ),
-                    false,
-                )]);
-                http::Response::builder()
-                    .status(200)
-                    .body(describe_response(&schema))
-                    .unwrap()
-            }
-            path => panic!("Unexpected request for unsupported seeded index: {path}"),
-        });
-
-        let error = table
-            .create_index(
-                &["vector"],
-                Index::IvfPq(IvfPqIndexBuilder::default().seed(42)),
-            )
-            .execute()
-            .await
-            .unwrap_err();
-        assert!(matches!(
-            error,
-            Error::NotSupported { message }
-                if message.contains("not supported on remote tables")
-        ));
-    }
-
     #[tokio::test]
     async fn test_create_index_returns_job() {
         let describe_calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));