mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-24 04:32:27 +00:00
fix(python): normalize VoyageAI image modes before JPEG encoding (#4576)
PIL images, PNG bytes and local image paths in VoyageAI source embeddings are saved directly as JPEG. RGBA, grayscale-with-alpha (LA) and palette (P) images therefore raise `OSError: cannot write mode ... as JPEG` before the provider is called. Convert to RGB at all three JPEG encoding sites. The caller’s original PIL image remains unchanged. Document that alpha is discarded and callers should composite onto a chosen background when it matters. Validation: - New image regressions + existing VoyageAI tests: **18 failed / 26 passed before → 44 passed after**. Tests cover both multimodal models, PIL/bytes/Path, RGBA/LA/P/RGB, decoded JPEG pixels and invalid images. - Independent local LanceDB table ingestion smoke: three PNG modes stored with 1024-dimensional vectors, original bytes preserved. Uses the freshly built current native extension. - Adjacent local embedding tests: 15 passed, 1 skipped (no OpenAI key), 9 deselected. The separate Flickr download test failed with `RemoteDisconnected`; it is excluded from the local result. - Pyright: 0 errors; CI-pinned Ruff 0.9.9 lint and Python formatting passed. External VoyageAI calls are stubbed; real Pillow conversion and LanceDB ingestion execute locally. No live provider or full-repository test result is claimed. AI assistance was used for implementation and validation.
This commit is contained in:
@@ -57,7 +57,12 @@ Usage Example:
|
||||
|
||||
### Multimodal Example {#multimodal-example}
|
||||
|
||||
The `voyage-multimodal-3.5` model can embed text alongside images. You can use image URLs, file paths, or PIL Image objects:
|
||||
The `voyage-multimodal-3.5` model can embed text alongside images. You can use image URLs, local `pathlib.Path` objects, raw image bytes, or PIL Image objects.
|
||||
|
||||
For source embeddings, PIL images and images loaded from bytes or local paths are
|
||||
converted to RGB and encoded as JPEG. This supports RGBA, grayscale with alpha,
|
||||
and palette images; transparency is discarded. Composite transparent images onto
|
||||
your chosen background before embedding if the background color matters.
|
||||
|
||||
<CodeGroup>
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
|
||||
@@ -75,7 +75,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
|
||||
content = {"type": "text", "text": input_data}
|
||||
elif isinstance(input_data, PIL_Image.Image):
|
||||
buffered = BytesIO()
|
||||
input_data.save(buffered, format="JPEG")
|
||||
input_data.convert("RGB").save(buffered, format="JPEG")
|
||||
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
content = {
|
||||
"type": "image_base64",
|
||||
@@ -84,7 +84,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
|
||||
elif isinstance(input_data, bytes):
|
||||
img = PIL_Image.open(BytesIO(input_data))
|
||||
buffered = BytesIO()
|
||||
img.save(buffered, format="JPEG")
|
||||
img.convert("RGB").save(buffered, format="JPEG")
|
||||
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
content = {
|
||||
"type": "image_base64",
|
||||
@@ -103,7 +103,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
|
||||
else:
|
||||
img = PIL_Image.open(input_data)
|
||||
buffered = BytesIO()
|
||||
img.save(buffered, format="JPEG")
|
||||
img.convert("RGB").save(buffered, format="JPEG")
|
||||
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
content = {
|
||||
"type": "image_base64",
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from lancedb.embeddings import get_registry
|
||||
from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", ["voyage-multimodal-3", "voyage-multimodal-3.5"])
|
||||
@pytest.mark.parametrize("input_kind", ["pil", "bytes", "path"])
|
||||
@pytest.mark.parametrize(
|
||||
"mode,color,expected_rgb",
|
||||
[
|
||||
("RGBA", (64, 128, 192, 128), (64, 128, 192)),
|
||||
("LA", (96, 128), (96, 96, 96)),
|
||||
("P", 0, (64, 128, 192)),
|
||||
("RGB", (64, 128, 192), (64, 128, 192)),
|
||||
],
|
||||
)
|
||||
def test_multimodal_source_embeddings_encode_image_modes(
|
||||
model_name, input_kind, mode, color, expected_rgb, tmp_path, monkeypatch
|
||||
):
|
||||
Image = pytest.importorskip("PIL.Image")
|
||||
image = Image.new(mode, (8, 6), color)
|
||||
if mode == "P":
|
||||
image.putpalette([64, 128, 192] + [0, 0, 0] * 255)
|
||||
image.info["transparency"] = 0
|
||||
original_pixels = image.tobytes()
|
||||
image_input = image
|
||||
if input_kind != "pil":
|
||||
png = BytesIO()
|
||||
image.save(png, format="PNG")
|
||||
if input_kind == "bytes":
|
||||
image_input = png.getvalue()
|
||||
else:
|
||||
image_input = tmp_path / "image.png"
|
||||
image_input.write_bytes(png.getvalue())
|
||||
|
||||
client = MagicMock()
|
||||
client.multimodal_embed.return_value.embeddings = [[0.1, 0.2]]
|
||||
monkeypatch.setattr(VoyageAIEmbeddingFunction, "client", client)
|
||||
func = get_registry().get("voyageai").create(name=model_name)
|
||||
|
||||
assert func.compute_source_embeddings(image_input) == [[0.1, 0.2]]
|
||||
|
||||
client.multimodal_embed.assert_called_once()
|
||||
request = client.multimodal_embed.call_args.kwargs
|
||||
assert request["model"] == model_name
|
||||
assert request["input_type"] == "document"
|
||||
assert len(request["inputs"]) == 1
|
||||
(content,) = request["inputs"][0]["content"]
|
||||
assert content["type"] == "image_base64"
|
||||
prefix, encoded = content["image_base64"].split(",", 1)
|
||||
assert prefix == "data:image/jpeg;base64"
|
||||
with Image.open(BytesIO(base64.b64decode(encoded, validate=True))) as decoded:
|
||||
assert decoded.format == "JPEG"
|
||||
assert decoded.mode == "RGB"
|
||||
assert decoded.size == (8, 6)
|
||||
assert decoded.getpixel((0, 0)) == pytest.approx(expected_rgb, abs=3)
|
||||
assert image.mode == mode
|
||||
assert image.tobytes() == original_pixels
|
||||
client.embed.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("input_kind", ["bytes", "path"])
|
||||
def test_multimodal_source_embeddings_reject_invalid_image(
|
||||
input_kind, tmp_path, monkeypatch
|
||||
):
|
||||
PIL = pytest.importorskip("PIL")
|
||||
image_input = b"not an image"
|
||||
if input_kind == "path":
|
||||
image_input = tmp_path / "invalid.png"
|
||||
image_input.write_bytes(b"not an image")
|
||||
|
||||
client = MagicMock()
|
||||
monkeypatch.setattr(VoyageAIEmbeddingFunction, "client", client)
|
||||
func = get_registry().get("voyageai").create(name="voyage-multimodal-3")
|
||||
|
||||
with pytest.raises(PIL.UnidentifiedImageError):
|
||||
func.compute_source_embeddings(image_input)
|
||||
|
||||
client.multimodal_embed.assert_not_called()
|
||||
client.embed.assert_not_called()
|
||||
Reference in New Issue
Block a user