From 22fc2a2087c1ed78033eadb056b1aef63abd1b69 Mon Sep 17 00:00:00 2001 From: devjoinedthechat Date: Mon, 26 Oct 2026 11:17:14 +0100 Subject: [PATCH] fix(python)!: put OpenCLIP models in eval mode before embedding (#4536) `OpenClipEmbeddings` loads its model with `open_clip.create_model_and_transforms()`, which hands the model back in training mode (the open_clip README calls `model.eval()` right after it for that reason). `OpenClipEmbeddings` never switches it out of training mode, so layers that behave differently during training stay active while we embed: stochastic depth in the timm-based image towers (ViTamin, ConvNeXt, MobileCLIP2-L-14), dropout in the Hugging Face text towers (`xlm-roberta-*`, `roberta-ViT-B-32`, `nllb-clip-*`), and batch norm in the ResNet towers loaded with non-OpenAI weights. The `torch.no_grad()` blocks turn off autograd but don't change how these layers behave. The default `ViT-B-32` has none of those layers, which is probably why nobody has run into this. With dropout or stochastic depth active, the same input gives a different vector on every call, so the vector stored at ingestion and the one computed for an identical query don't match. With batch norm in training mode, an image's vector depends on the other images in its batch: ```python import numpy as np from PIL import Image from lancedb.embeddings import get_registry func = get_registry().get("open-clip").create(name="ViTamin-S", pretrained="datacomp1b") image = Image.new("RGB", (256, 256), "red") a = func.generate_image_embedding(image) b = func.generate_image_embedding(image) print(func._model.training, np.array_equal(a, b)) # before: True False # after: False True ``` The fix calls `model.eval()` once after loading, same as the SigLIP, ColPali and ImageBind embedding functions already do. The new test swaps in a stand-in for the `open_clip` module, so it runs without torch or a model download, and checks that the model handed back by `create_model_and_transforms` ends up in eval mode. BREAKING CHANGE: Embeddings change for OpenCLIP models with training-sensitive layers, including ViTamin, ConvNeXt, MobileCLIP2, Hugging Face RoBERTa/XLM-RoBERTa/NLLB text towers, and ResNet models loaded with non-OpenAI weights. For tables embedded with affected models, regenerate the stored embeddings in eval mode and rebuild their vector indexes before querying with the new embeddings. Rebuilding an index alone does not regenerate embeddings. The default ViT-B-32 model is unaffected. Co-authored-by: devjoinedthechat <235875317+devjoinedthechat@users.noreply.github.com> Co-authored-by: Xuanwo --- python/python/lancedb/embeddings/open_clip.py | 3 ++ python/python/tests/test_embeddings.py | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/python/python/lancedb/embeddings/open_clip.py b/python/python/lancedb/embeddings/open_clip.py index 8b3f0eb06..6199ef271 100644 --- a/python/python/lancedb/embeddings/open_clip.py +++ b/python/python/lancedb/embeddings/open_clip.py @@ -49,6 +49,9 @@ class OpenClipEmbeddings(EmbeddingFunction): self.name, pretrained=self.pretrained ) model.to(self.device) + # open_clip returns the model in training mode, which keeps dropout, + # stochastic depth and batch norm in their training behaviour. + model.eval() self._model, self._preprocess = model, preprocess self._tokenizer = open_clip.get_tokenizer(self.name) self._ndims = None diff --git a/python/python/tests/test_embeddings.py b/python/python/tests/test_embeddings.py index 207a3d253..3c968a821 100644 --- a/python/python/tests/test_embeddings.py +++ b/python/python/tests/test_embeddings.py @@ -1039,6 +1039,39 @@ def test_open_clip_opens_percent_encoded_file_uri(tmp_path): assert image.size == (4, 4) +def test_open_clip_model_in_eval_mode(): + """ + open_clip.create_model_and_transforms returns the model in training mode, + which leaves dropout, stochastic depth and batch norm in their training + behaviour. OpenCLIP must switch the model to eval mode before embedding. + """ + from lancedb.embeddings.open_clip import OpenClipEmbeddings + + class FakeModel: + training = True + + def to(self, device): + return self + + def eval(self): + self.training = False + return self + + open_clip = MagicMock() + open_clip.create_model_and_transforms.return_value = (FakeModel(), None, None) + + with patch( + "lancedb.embeddings.open_clip.attempt_import_or_raise", + return_value=open_clip, + ): + func = OpenClipEmbeddings(name="ViTamin-B", pretrained="datacomp1b") + + open_clip.create_model_and_transforms.assert_called_once_with( + "ViTamin-B", pretrained="datacomp1b" + ) + assert func._model.training is False + + def test_jina_generate_image_input_dict_local_path(tmp_path): """ JinaEmbeddings._generate_image_input_dict must accept a local image path