mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-29 23:06:51 +00:00
fix(embeddings): default Cohere to embed-multilingual-v3.0, the v2.0 models are retired (#4698)
Cohere retired `embed-english-v2.0`, `embed-english-light-v2.0` and `embed-multilingual-v2.0` on April 4, 2026 ([deprecations](https://docs.cohere.com/docs/deprecations), which lists `embed-english-v3.0`, `embed-multilingual-v3.0` and `embed-v4.0` as the embedding alternatives). `CohereEmbeddingFunction` still defaulted to `embed-multilingual-v2.0`, so every call without an explicit `name` fails. The default is now `embed-multilingual-v3.0`, the multilingual v3 model, and the retired models are removed from the docstring and the Cohere integration page (the page also listed `embed-english-v2.0` as the default, which did not match the code). The docs example, its generated snippet, the docs test and the slow Cohere test use the new name. This changes the default vector size: `ndims()` returns 1024 for `embed-multilingual-v3.0`, where the old default gave 768. Tables created from a schema that uses `cohere.ndims()` get 1024-dimension vectors from now on. A table that already holds 768-dimension vectors from the retired model needs a re-embed with a current model, since the old model can no longer be called. The `ndims()` entries for the v2 names are left as they are. Test: `test_cohere_default_model_is_not_a_retired_v2_model` in `python/python/tests/test_embeddings.py` checks the default name and that `ndims()` is 1024. The repository's pytest session could not be run for this change (its `conftest.py` needs a newer native module than the released wheel); the body of that test was run against the released `lancedb` 0.40.0 with this version of `cohere.py` in place, where it passes, and it returns `embed-multilingual-v2.0` / 768 without the change. ruff check and format are clean on the touched Python files. No Cohere request was made. This change was made with Claude Code. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
co-authored by
Claude Sonnet 5.5
Xuanwo
parent
4359688f32
commit
659ea11ac4
@@ -93,7 +93,7 @@ def test_embedding_cohere_usage() -> None:
|
||||
cohere = (
|
||||
EmbeddingFunctionRegistry.get_instance()
|
||||
.get("cohere")
|
||||
.create(name="embed-multilingual-v2.0")
|
||||
.create(name="embed-multilingual-v3.0")
|
||||
)
|
||||
|
||||
class TextModel(LanceModel):
|
||||
|
||||
@@ -181,7 +181,7 @@ LanceDB supports most popular embedding providers.
|
||||
| OpenAI | `openai` | `text-embedding-ada-002` |
|
||||
| Sentence Transformers | `sentence-transformers` | `all-MiniLM-L6-v2` |
|
||||
| Hugging Face | `huggingface` | `colbert-ir/colbertv2.0` |
|
||||
| Cohere | `cohere` | `embed-english-v3.0` |
|
||||
| Cohere | `cohere` | `embed-multilingual-v3.0` |
|
||||
| ... | ... | ... |
|
||||
|
||||
### Multimodal embedding {#multimodal-embedding}
|
||||
|
||||
@@ -18,16 +18,20 @@ Supported models are:
|
||||
- embed-multilingual-v3.0
|
||||
- embed-english-light-v3.0
|
||||
- embed-multilingual-light-v3.0
|
||||
- embed-english-v2.0
|
||||
- embed-english-light-v2.0
|
||||
- embed-multilingual-v2.0
|
||||
|
||||
The v2.0 models (`embed-english-v2.0`, `embed-english-light-v2.0` and `embed-multilingual-v2.0`) were retired by Cohere on April 4, 2026.
|
||||
|
||||
The previous default, `embed-multilingual-v2.0`, produced 768-dimensional vectors.
|
||||
The new default, `embed-multilingual-v3.0`, produces 1024-dimensional vectors.
|
||||
Existing tables containing vectors from a v2.0 model must be re-embedded with an
|
||||
active model into a table whose vector column has the matching dimension.
|
||||
|
||||
|
||||
Supported parameters (to be passed in `create` method) are:
|
||||
|
||||
| Parameter | Type | Default Value | Description |
|
||||
|---|---|--------|---------|
|
||||
| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
|
||||
| `name` | `str` | `"embed-multilingual-v3.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0 |
|
||||
| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
|
||||
| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
|
||||
|
||||
@@ -52,4 +56,4 @@ Usage Example:
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyEmbeddingCohereUsage}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
</CodeGroup>
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
export const PyEmbeddingAwsUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, Vector\n\nmodel = get_registry().get(\"bedrock-text\").create()\n\nclass TextModel(LanceModel):\n text: str = model.SourceField()\n vector: Vector(model.ndims()) = model.VectorField()\n\ndf = pd.DataFrame({\"text\": [\"hello world\", \"goodbye world\"]})\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"bedrock-demo\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\n\ntbl.add(df)\nrs = tbl.search(\"hello\").limit(1).to_pandas()\nprint(rs.head())\n";
|
||||
|
||||
export const PyEmbeddingCohereUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\ncohere = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"cohere\")\n .create(name=\"embed-multilingual-v2.0\")\n)\n\nclass TextModel(LanceModel):\n text: str = cohere.SourceField()\n vector: Vector(cohere.ndims()) = cohere.VectorField()\n\ndata = [{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cohere-demo\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\ntbl.add(data)\n";
|
||||
export const PyEmbeddingCohereUsage = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nfrom lancedb.embeddings import EmbeddingFunctionRegistry\nfrom lancedb.pydantic import LanceModel, Vector\n\ncohere = (\n EmbeddingFunctionRegistry.get_instance()\n .get(\"cohere\")\n .create(name=\"embed-multilingual-v3.0\")\n)\n\nclass TextModel(LanceModel):\n text: str = cohere.SourceField()\n vector: Vector(cohere.ndims()) = cohere.VectorField()\n\ndata = [{\"text\": \"hello world\"}, {\"text\": \"goodbye world\"}]\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"cohere-demo\"))\ntbl = db.create_table(\"test\", schema=TextModel, mode=\"overwrite\")\ntbl.add(data)\n";
|
||||
|
||||
export const PyEmbeddingColpaliSetup = "import tempfile\nfrom pathlib import Path\n\nimport lancedb\nimport pandas as pd\nimport requests\nfrom lancedb.embeddings import get_registry\nfrom lancedb.pydantic import LanceModel, MultiVector\n\ndb = lancedb.connect(str(Path(tempfile.mkdtemp()) / \"colpali-demo\"))\nfunc = get_registry().get(\"colpali\").create()\n\nclass Images(LanceModel):\n label: str\n image_uri: str = func.SourceField()\n image_bytes: bytes = func.SourceField()\n vector: MultiVector(func.ndims()) = func.VectorField()\n vec_from_bytes: MultiVector(func.ndims()) = func.VectorField()\n\ntable = db.create_table(\"images\", schema=Images)\nlabels = [\"cat\", \"dog\", \"horse\"]\nuris = [\n \"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg\",\n \"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg\",\n \"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg\",\n]\nimage_bytes = [requests.get(uri).content for uri in uris]\ntable.add(\n pd.DataFrame({\"label\": labels, \"image_uri\": uris, \"image_bytes\": image_bytes})\n)\n";
|
||||
|
||||
|
||||
@@ -22,16 +22,20 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str, default "embed-multilingual-v2.0"
|
||||
name : str, default "embed-multilingual-v3.0"
|
||||
The name of the model to use. List of acceptable models:
|
||||
|
||||
* embed-english-v3.0
|
||||
* embed-multilingual-v3.0
|
||||
* embed-english-light-v3.0
|
||||
* embed-multilingual-light-v3.0
|
||||
* embed-english-v2.0
|
||||
* embed-english-light-v2.0
|
||||
* embed-multilingual-v2.0
|
||||
|
||||
The v2.0 models (embed-english-v2.0, embed-english-light-v2.0 and
|
||||
embed-multilingual-v2.0) were retired by Cohere on April 4, 2026.
|
||||
The previous default produced 768-dimensional vectors, while the new
|
||||
default produces 1024-dimensional vectors. Existing tables containing
|
||||
vectors from a v2.0 model must be re-embedded with an active model into
|
||||
a table whose vector column has the matching dimension.
|
||||
|
||||
source_input_type : str, default "search_document"
|
||||
The input type for the source column in the database
|
||||
@@ -65,7 +69,7 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
|
||||
cohere = EmbeddingFunctionRegistry
|
||||
.get_instance()
|
||||
.get("cohere")
|
||||
.create(name="embed-multilingual-v2.0")
|
||||
.create(name="embed-multilingual-v3.0")
|
||||
|
||||
class TextModel(LanceModel):
|
||||
text: str = cohere.SourceField()
|
||||
@@ -81,7 +85,7 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
|
||||
|
||||
"""
|
||||
|
||||
name: str = "embed-multilingual-v2.0"
|
||||
name: str = "embed-multilingual-v3.0"
|
||||
source_input_type: str = "search_document"
|
||||
query_input_type: str = "search_query"
|
||||
client: ClassVar = None
|
||||
|
||||
@@ -1115,3 +1115,13 @@ def test_jina_embeddings_use_their_own_api_key(monkeypatch):
|
||||
func_b.compute_source_embeddings(["hello"])
|
||||
|
||||
assert sent == ["Bearer key-a", "Bearer key-b"]
|
||||
|
||||
|
||||
def test_cohere_default_model_is_not_a_retired_v2_model():
|
||||
# Cohere retired embed-english-v2.0, embed-english-light-v2.0 and
|
||||
# embed-multilingual-v2.0 on April 4, 2026.
|
||||
cohere = get_registry().get("cohere").create()
|
||||
|
||||
assert cohere.name == "embed-multilingual-v3.0"
|
||||
assert "v2.0" not in cohere.name
|
||||
assert cohere.ndims() == 1024
|
||||
|
||||
@@ -272,7 +272,7 @@ def test_cohere_embedding_function():
|
||||
cohere = (
|
||||
get_registry()
|
||||
.get("cohere")
|
||||
.create(name="embed-multilingual-v2.0", max_retries=0)
|
||||
.create(name="embed-multilingual-v3.0", max_retries=0)
|
||||
)
|
||||
|
||||
class TextModel(LanceModel):
|
||||
|
||||
Reference in New Issue
Block a user