From f2c6aa35c236e04b012c150ce94dcc6fc09fd2f7 Mon Sep 17 00:00:00 2001 From: Captain <42566883+po-et@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:24:04 +0800 Subject: [PATCH] fix(python): normalize VoyageAI image modes before JPEG encoding (#4576) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PIL images, PNG bytes and local image paths in VoyageAI source embeddings are saved directly as JPEG. RGBA, grayscale-with-alpha (LA) and palette (P) images therefore raise `OSError: cannot write mode ... as JPEG` before the provider is called. Convert to RGB at all three JPEG encoding sites. The caller’s original PIL image remains unchanged. Document that alpha is discarded and callers should composite onto a chosen background when it matters. Validation: - New image regressions + existing VoyageAI tests: **18 failed / 26 passed before → 44 passed after**. Tests cover both multimodal models, PIL/bytes/Path, RGBA/LA/P/RGB, decoded JPEG pixels and invalid images. - Independent local LanceDB table ingestion smoke: three PNG modes stored with 1024-dimensional vectors, original bytes preserved. Uses the freshly built current native extension. - Adjacent local embedding tests: 15 passed, 1 skipped (no OpenAI key), 9 deselected. The separate Flickr download test failed with `RemoteDisconnected`; it is excluded from the local result. - Pyright: 0 errors; CI-pinned Ruff 0.9.9 lint and Python formatting passed. External VoyageAI calls are stubbed; real Pillow conversion and LanceDB ingestion execute locally. No live provider or full-repository test result is claimed. AI assistance was used for implementation and validation. --- docs/web/integrations/embedding/voyageai.mdx | 7 +- python/python/lancedb/embeddings/voyageai.py | 6 +- .../tests/test_voyageai_image_inputs.py | 88 +++++++++++++++++++ 3 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 python/python/tests/test_voyageai_image_inputs.py diff --git a/docs/web/integrations/embedding/voyageai.mdx b/docs/web/integrations/embedding/voyageai.mdx index c0741b39b..957ab2b4d 100644 --- a/docs/web/integrations/embedding/voyageai.mdx +++ b/docs/web/integrations/embedding/voyageai.mdx @@ -57,7 +57,12 @@ Usage Example: ### Multimodal Example {#multimodal-example} -The `voyage-multimodal-3.5` model can embed text alongside images. You can use image URLs, file paths, or PIL Image objects: +The `voyage-multimodal-3.5` model can embed text alongside images. You can use image URLs, local `pathlib.Path` objects, raw image bytes, or PIL Image objects. + +For source embeddings, PIL images and images loaded from bytes or local paths are +converted to RGB and encoded as JPEG. This supports RGBA, grayscale with alpha, +and palette images; transparency is discarded. Composite transparent images onto +your chosen background before embedding if the background color matters. diff --git a/python/python/lancedb/embeddings/voyageai.py b/python/python/lancedb/embeddings/voyageai.py index 95525e07b..bcb2fd5a3 100644 --- a/python/python/lancedb/embeddings/voyageai.py +++ b/python/python/lancedb/embeddings/voyageai.py @@ -75,7 +75,7 @@ def transform_input(input_data: Union[str, bytes, Path]): content = {"type": "text", "text": input_data} elif isinstance(input_data, PIL_Image.Image): buffered = BytesIO() - input_data.save(buffered, format="JPEG") + input_data.convert("RGB").save(buffered, format="JPEG") img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") content = { "type": "image_base64", @@ -84,7 +84,7 @@ def transform_input(input_data: Union[str, bytes, Path]): elif isinstance(input_data, bytes): img = PIL_Image.open(BytesIO(input_data)) buffered = BytesIO() - img.save(buffered, format="JPEG") + img.convert("RGB").save(buffered, format="JPEG") img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") content = { "type": "image_base64", @@ -103,7 +103,7 @@ def transform_input(input_data: Union[str, bytes, Path]): else: img = PIL_Image.open(input_data) buffered = BytesIO() - img.save(buffered, format="JPEG") + img.convert("RGB").save(buffered, format="JPEG") img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") content = { "type": "image_base64", diff --git a/python/python/tests/test_voyageai_image_inputs.py b/python/python/tests/test_voyageai_image_inputs.py new file mode 100644 index 000000000..885f94821 --- /dev/null +++ b/python/python/tests/test_voyageai_image_inputs.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import base64 +from io import BytesIO +from unittest.mock import MagicMock + +import pytest + +from lancedb.embeddings import get_registry +from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction + + +@pytest.mark.parametrize("model_name", ["voyage-multimodal-3", "voyage-multimodal-3.5"]) +@pytest.mark.parametrize("input_kind", ["pil", "bytes", "path"]) +@pytest.mark.parametrize( + "mode,color,expected_rgb", + [ + ("RGBA", (64, 128, 192, 128), (64, 128, 192)), + ("LA", (96, 128), (96, 96, 96)), + ("P", 0, (64, 128, 192)), + ("RGB", (64, 128, 192), (64, 128, 192)), + ], +) +def test_multimodal_source_embeddings_encode_image_modes( + model_name, input_kind, mode, color, expected_rgb, tmp_path, monkeypatch +): + Image = pytest.importorskip("PIL.Image") + image = Image.new(mode, (8, 6), color) + if mode == "P": + image.putpalette([64, 128, 192] + [0, 0, 0] * 255) + image.info["transparency"] = 0 + original_pixels = image.tobytes() + image_input = image + if input_kind != "pil": + png = BytesIO() + image.save(png, format="PNG") + if input_kind == "bytes": + image_input = png.getvalue() + else: + image_input = tmp_path / "image.png" + image_input.write_bytes(png.getvalue()) + + client = MagicMock() + client.multimodal_embed.return_value.embeddings = [[0.1, 0.2]] + monkeypatch.setattr(VoyageAIEmbeddingFunction, "client", client) + func = get_registry().get("voyageai").create(name=model_name) + + assert func.compute_source_embeddings(image_input) == [[0.1, 0.2]] + + client.multimodal_embed.assert_called_once() + request = client.multimodal_embed.call_args.kwargs + assert request["model"] == model_name + assert request["input_type"] == "document" + assert len(request["inputs"]) == 1 + (content,) = request["inputs"][0]["content"] + assert content["type"] == "image_base64" + prefix, encoded = content["image_base64"].split(",", 1) + assert prefix == "data:image/jpeg;base64" + with Image.open(BytesIO(base64.b64decode(encoded, validate=True))) as decoded: + assert decoded.format == "JPEG" + assert decoded.mode == "RGB" + assert decoded.size == (8, 6) + assert decoded.getpixel((0, 0)) == pytest.approx(expected_rgb, abs=3) + assert image.mode == mode + assert image.tobytes() == original_pixels + client.embed.assert_not_called() + + +@pytest.mark.parametrize("input_kind", ["bytes", "path"]) +def test_multimodal_source_embeddings_reject_invalid_image( + input_kind, tmp_path, monkeypatch +): + PIL = pytest.importorskip("PIL") + image_input = b"not an image" + if input_kind == "path": + image_input = tmp_path / "invalid.png" + image_input.write_bytes(b"not an image") + + client = MagicMock() + monkeypatch.setattr(VoyageAIEmbeddingFunction, "client", client) + func = get_registry().get("voyageai").create(name="voyage-multimodal-3") + + with pytest.raises(PIL.UnidentifiedImageError): + func.compute_source_embeddings(image_input) + + client.multimodal_embed.assert_not_called() + client.embed.assert_not_called()