update

When we turned on fat LTO builds, we made the release build job **much**
more compute and memory intensive. The ARM runners have particularly low
memory per core, which makes them susceptible to OOM errors. To avoid
issues, I have enabled memory swap on ARM and bumped the side of the
runner.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.8
+current_version = 0.4.10
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.cargo/config.toml

@@ -25,6 +25,7 @@ rustflags = [
     "-Dclippy::dbg_macro",
     # not too much we can do to avoid multiple crate versions
     "-Aclippy::multiple-crate-versions",
+    "-Aclippy::wildcard_dependencies",
 ]
 
 [target.x86_64-unknown-linux-gnu]

.github/workflows/npm-publish.yml

@@ -80,10 +80,25 @@ jobs:
           - arch: x86_64
             runner: ubuntu-latest
           - arch: aarch64
-            runner: buildjet-4vcpu-ubuntu-2204-arm
+            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
+            runner: buildjet-16vcpu-ubuntu-2204-arm
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
+      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
+      - name: Configure aarch64 build
+        if: ${{ matrix.config.arch == 'aarch64' }}
+        run: |
+          free -h
+          sudo fallocate -l 16G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
+          # print info
+          swapon --show
+          free -h
       - name: Build Linux Artifacts
         run: |
           bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}

Cargo.toml

@@ -14,10 +14,10 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.9.15", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.9.15" }
-lance-linalg = { "version" = "=0.9.15" }
-lance-testing = { "version" = "=0.9.15" }
+lance = { "version" = "=0.9.16", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.9.16" }
+lance-linalg = { "version" = "=0.9.16" }
+lance-testing = { "version" = "=0.9.16" }
 # Note that this one does not include pyarrow
 arrow = { version = "50.0", optional = false }
 arrow-array = "50.0"

ci/build_linux_artifacts.sh

@@ -13,7 +13,9 @@ docker build \
     .
 popd
 
+# We turn on memory swap to avoid OOM killer
 docker run \
     -v $(pwd):/io -w /io \
+    --memory-swap=-1 \
     lancedb-node-manylinux \
     bash ci/manylinux_node/build.sh $ARCH

docs/mkdocs.yml

@@ -90,7 +90,10 @@ nav:
     - Building an ANN index: ann_indexes.md
     - Vector Search: search.md
     - Full-text search: fts.md
-    - Hybrid search: hybrid_search.md
+    - Hybrid search:
+      - Overview: hybrid_search/hybrid_search.md
+      - Comparing Rerankers: hybrid_search/eval.md
+      - Airbnb financial data example: notebooks/hybrid_search.ipynb
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md
@@ -152,7 +155,10 @@ nav:
     - Building an ANN index: ann_indexes.md
     - Vector Search: search.md
     - Full-text search: fts.md
-    - Hybrid search: hybrid_search.md
+    - Hybrid search:
+      - Overview: hybrid_search/hybrid_search.md
+      - Comparing Rerankers: hybrid_search/eval.md
+      - Airbnb financial data example: notebooks/hybrid_search.ipynb
     - Filtering: sql.md
     - Versioning & Reproducibility: notebooks/reproducibility.ipynb
     - Configuring Storage: guides/storage.md

docs/src/embeddings/api.md

@@ -17,6 +17,7 @@ Let's implement `SentenceTransformerEmbeddings` class. All you need to do is imp
 
 ```python
 from lancedb.embeddings import register
+from lancedb.util import attempt_import_or_raise
 
 @register("sentence-transformers")
 class SentenceTransformerEmbeddings(TextEmbeddingFunction):
@@ -81,7 +82,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        open_clip = self.safe_import("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
+        open_clip = attempt_import_or_raise("open_clip", "open-clip") # EmbeddingFunction util to import external libs and raise if not found
         model, _, preprocess = open_clip.create_model_and_transforms(
             self.name, pretrained=self.pretrained
         )
@@ -109,14 +110,14 @@ class OpenClipEmbeddings(EmbeddingFunction):
         if isinstance(query, str):
             return [self.generate_text_embeddings(query)]
         else:
-            PIL = self.safe_import("PIL", "pillow")
+            PIL = attempt_import_or_raise("PIL", "pillow")
             if isinstance(query, PIL.Image.Image):
                 return [self.generate_image_embedding(query)]
             else:
                 raise TypeError("OpenClip supports str or PIL Image as query")
 
     def generate_text_embeddings(self, text: str) -> np.ndarray:
-        torch = self.safe_import("torch")
+        torch = attempt_import_or_raise("torch")
         text = self.sanitize_input(text)
         text = self._tokenizer(text)
         text.to(self.device)
@@ -175,7 +176,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
             The image to embed. If the image is a str, it is treated as a uri.
             If the image is bytes, it is treated as the raw image bytes.
         """
-        torch = self.safe_import("torch")
+        torch = attempt_import_or_raise("torch")
         # TODO handle retry and errors for https
         image = self._to_pil(image)
         image = self._preprocess(image).unsqueeze(0)
@@ -183,7 +184,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
             return self._encode_and_normalize_image(image)
 
     def _to_pil(self, image: Union[str, bytes]):
-        PIL = self.safe_import("PIL", "pillow")
+        PIL = attempt_import_or_raise("PIL", "pillow")
         if isinstance(image, bytes):
             return PIL.Image.open(io.BytesIO(image))
         if isinstance(image, PIL.Image.Image):

docs/src/embeddings/default_embedding_functions.md

@@ -9,6 +9,9 @@ Contains the text embedding functions registered by default.
 ### Sentence transformers
 Allows you to set parameters when registering a `sentence-transformers` object.
 
+!!! info
+    Sentence transformer embeddings are normalized by default. It is recommended to use normalized embeddings for similarity search.
+
 | Parameter | Type | Default Value | Description |
 |---|---|---|---|
 | `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |

docs/src/hybrid_search/eval.md

@@ -0,0 +1,49 @@
+# Hybrid Search
+
+Hybrid Search is a broad (often misused) term. It can mean anything from combining multiple methods for searching, to applying ranking methods to better sort the results. In this blog, we use the definition of "hybrid search" to mean using a combination of keyword-based and vector search.
+
+## The challenge of (re)ranking search results
+Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
+There are two approaches for reranking search results from multiple sources.
+* <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
+* <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
+
+Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.
+
+### Example evaluation of hybrid search with Reranking
+
+Here's some evaluation numbers from experiment comparing these re-rankers on about 800 queries. It is modified version of an evaluation script from [llama-index](https://github.com/run-llama/finetune-embedding/blob/main/evaluate.ipynb) that measures hit-rate at top-k.
+
+<b> With OpenAI ada2 embedding </b>
+
+Vector Search baseline - `0.64`
+
+| Reranker | Top-3 | Top-5 | Top-10 |
+| --- | --- | --- | --- |
+| Linear Combination | `0.73` | `0.74` | `0.85` |
+| Cross Encoder | `0.71` | `0.70` | `0.77` |
+| Cohere | `0.81` | `0.81` | `0.85` |
+| ColBERT | `0.68` | `0.68` | `0.73` |
+
+<p>
+<img src="https://github.com/AyushExel/assets/assets/15766192/d57b1780-ef27-414c-a5c3-73bee7808a45">
+</p>
+
+<b> With OpenAI embedding-v3-small </b>
+
+Vector Search baseline - `0.59`
+
+| Reranker | Top-3 | Top-5 | Top-10 |
+| --- | --- | --- | --- |
+| Linear Combination | `0.68` | `0.70` | `0.84` |
+| Cross Encoder | `0.72` | `0.72` | `0.79` |
+| Cohere | `0.79` | `0.79` | `0.84` |
+| ColBERT | `0.70` | `0.70` | `0.76` |
+
+<p>
+<img src="https://github.com/AyushExel/assets/assets/15766192/259adfd2-6ec6-4df6-a77d-1456598970dd">
+</p>
+
+### Conclusion
+
+The results show that the reranking methods are able to improve the search results. However, the improvement is not consistent across all rerankers. The choice of reranker depends on the dataset and the application. It is also important to note that the reranking methods are not a replacement for the search methods. They are complementary and should be used together to get the best results. The speed to recall tradeoff is also an important factor to consider when choosing the reranker.

docs/src/hybrid_search/hybrid_search.md

@@ -1,6 +1,6 @@
 # Hybrid Search
 
-LanceDB supports both semantic and keyword-based search. In real world applications, it is often useful to combine these two approaches to get the best best results. For example, you may want to search for a document that is semantically similar to a query document, but also contains a specific keyword. This is an example of *hybrid search*, a search algorithm that combines multiple search techniques.
+LanceDB supports both semantic and keyword-based search (also termed full-text search, or FTS). In real world applications, it is often useful to combine these two approaches to get the best best results. For example, you may want to search for a document that is semantically similar to a query document, but also contains a specific keyword. This is an example of *hybrid search*, a search algorithm that combines multiple search techniques.
 
 ## Hybrid search in LanceDB
 You can perform hybrid search in LanceDB by combining the results of semantic and full-text search via a reranking algorithm of your choice. LanceDB provides multiple rerankers out of the box. However, you can always write a custom reranker if your use case need more sophisticated logic .
@@ -69,7 +69,7 @@ reranker = LinearCombinationReranker(weight=0.3) # Use 0.3 as the weight for vec
 results = table.search("rebel", query_type="hybrid").rerank(reranker=reranker).to_pandas()
 ```
 
-Arguments
+### Arguments
 ----------------
 * `weight`: `float`, default `0.7`:
     The weight to use for the semantic search score. The weight for the full-text search score is `1 - weights`.
@@ -91,9 +91,9 @@ reranker = CohereReranker()
 results = table.search("vampire weekend", query_type="hybrid").rerank(reranker=reranker).to_pandas()
 ```
 
-Arguments
+### Arguments
 ----------------
-* `model_name`` : str, default `"rerank-english-v2.0"``
+* `model_name` : str, default `"rerank-english-v2.0"`
         The name of the cross encoder model to use. Available cohere models are:
         - rerank-english-v2.0
         - rerank-multilingual-v2.0
@@ -117,7 +117,7 @@ results = table.search("harmony hall", query_type="hybrid").rerank(reranker=rera
 ```
 
 
-Arguments
+### Arguments
 ----------------
 * `model` : str, default `"cross-encoder/ms-marco-TinyBERT-L-6"`
         The name of the cross encoder model to use. Available cross encoder models can be found [here](https://www.sbert.net/docs/pretrained_cross-encoders.html)
@@ -143,7 +143,7 @@ reranker = ColbertReranker()
 results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
 ```
 
-Arguments
+### Arguments
 ----------------
 * `model_name` : `str`, default `"colbert-ir/colbertv2.0"`
         The name of the cross encoder model to use.
@@ -162,7 +162,8 @@ This reranker uses the OpenAI API to combine the results of semantic and full-te
     This prompts chat model to rerank results which is not a dedicated reranker model. This should be treated as experimental.
 
 !!! Tip
-    You might run out of token limit so set the search `limits` based on your token limit.
+    - You might run out of token limit so set the search `limits` based on your token limit.
+    - It is recommended to use gpt-4-turbo-preview, the default model, older models might lead to undesired behaviour
 
 ```python
 from lancedb.rerankers import OpenaiReranker
@@ -172,15 +173,15 @@ reranker = OpenaiReranker()
 results = table.search("harmony hall", query_type="hybrid").rerank(reranker=reranker).to_pandas()
 ```
 
-Arguments
+### Arguments
 ----------------
-`model_name` : `str`, default `"gpt-3.5-turbo-1106"`
+* `model_name` : `str`, default `"gpt-4-turbo-preview"`
     The name of the cross encoder model to use.
-`column` : `str`, default `"text"`
+* `column` : `str`, default `"text"`
     The name of the column to use as input to the cross encoder model.
-`return_score` : `str`, default `"relevance"`
+* `return_score` : `str`, default `"relevance"`
     options are "relevance" or "all". Only "relevance" is supported for now.
-`api_key` : `str`, default `None`
+* `api_key` : `str`, default `None`
     The API key to use. If None, will use the OPENAI_API_KEY environment variable.
 
 
@@ -212,24 +213,30 @@ class MyReranker(Reranker):
 
 ```
 
-You can also accept additional arguments like a filter along with fts and vector search results
+### Example of a Custom Reranker
+For the sake of simplicity let's build custom reranker that just enchances the Cohere Reranker by accepting a filter query, and accept other CohereReranker params as kwags.
 
 ```python
 
-from lancedb.rerankers import Reranker
-import pyarrow as pa
+from typing import List, Union
+import pandas as pd
+from lancedb.rerankers import CohereReranker
 
-class MyReranker(Reranker):
-    ...
-    
-    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table, filter: str):
-        # Use the built-in merging function
-        combined_result = self.merge_results(vector_results, fts_results)
-        
-        # Do something with the combined results & filter
-        # ...
+class MofidifiedCohereReranker(CohereReranker):
+    def __init__(self, filters: Union[str, List[str]], **kwargs):
+        super().__init__(**kwargs)
+        filters = filters if isinstance(filters, list) else [filters]
+        self.filters = filters
 
-        # Return the combined results
-        return combined_result
+    def rerank_hybrid(self, query: str, vector_results: pa.Table, fts_results: pa.Table)-> pa.Table:
+        combined_result = super().rerank_hybrid(query, vector_results, fts_results)
+        df = combined_result.to_pandas()
+        for filter in self.filters:
+            df = df.query("not text.str.contains(@filter)")
+
+        return pa.Table.from_pandas(df)
 
 ```
+
+!!! tip
+    The `vector_results` and `fts_results` are pyarrow tables. You can convert them to pandas dataframes using `to_pandas()` method and perform any operations you want. After you are done, you can convert the dataframe back to pyarrow table using `pa.Table.from_pandas()` method and return it.

docs/test/md_testing.py

@@ -14,7 +14,7 @@ excluded_globs = [
     "../src/concepts/*.md",
     "../src/ann_indexes.md",
     "../src/basic.md",
-    "../src/hybrid_search.md",
+    "../src/hybrid_search/hybrid_search.md",
 ]
 
 python_prefix = "py"

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.4.8",
+  "version": "0.4.10",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.4.8",
+      "version": "0.4.10",
       "cpu": [
         "x64",
         "arm64"
@@ -53,11 +53,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.4.8",
-        "@lancedb/vectordb-darwin-x64": "0.4.8",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.4.8",
-        "@lancedb/vectordb-linux-x64-gnu": "0.4.8",
-        "@lancedb/vectordb-win32-x64-msvc": "0.4.8"
+        "@lancedb/vectordb-darwin-arm64": "0.4.10",
+        "@lancedb/vectordb-darwin-x64": "0.4.10",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.10",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.10",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.10"
       }
     },
     "node_modules/@75lb/deep-merge": {
@@ -329,9 +329,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.4.8",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.8.tgz",
-      "integrity": "sha512-FpnJaw7KmNdD/FtOw9AcmPL5P+L04AcnfPj9ZyEjN8iCwB/qaOGYgdfBv+EbEtfHIsqA12q/1BRduu9KdB6BIA==",
+      "version": "0.4.10",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.10.tgz",
+      "integrity": "sha512-y/uHOGb0g15pvqv5tdTyZ6oN+0QVpBmZDzKFWW6pPbuSZjB2uPqcs+ti0RB+AUdmS21kavVQqaNsw/HLKEGrHA==",
       "cpu": [
         "arm64"
       ],
@@ -341,9 +341,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.4.8",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.8.tgz",
-      "integrity": "sha512-RafOEYyZIgphp8wPGuVLFaTc8aAqo0NCO1LQMx0mB0xV96vrdo0Mooivs+dYN3RFfSHtTKPw9O1Jc957Vp1TLg==",
+      "version": "0.4.10",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.10.tgz",
+      "integrity": "sha512-XbfR58OkQpAe0xMSTrwJh9ZjGSzG9EZ7zwO6HfYem8PxcLYAcC6eWRWoSG/T0uObyrPTcYYyvHsp0eNQWYBFAQ==",
       "cpu": [
         "x64"
       ],
@@ -353,9 +353,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.4.8",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.8.tgz",
-      "integrity": "sha512-WlbYNfj4+v1hBHUluF+hnlG/A0ZaQFdXBTGDfHQniL11o+n3emWm4ujP5nSAoQHXjSH9DaOTGr/N4Mc9Xe+luw==",
+      "version": "0.4.10",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.10.tgz",
+      "integrity": "sha512-x40WKH9b+KxorRmKr9G7fv8p5mMj8QJQvRMA0v6v+nbZHr2FLlAZV+9mvhHOnm4AGIkPP5335cUgv6Qz6hgwkQ==",
       "cpu": [
         "arm64"
       ],
@@ -365,9 +365,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.4.8",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.8.tgz",
-      "integrity": "sha512-z+qFJrDqnNEv4JcwYDyt51PHmWjuM/XaOlSjpBnyyuUImeY+QcwctMuyXt8+Q4zhuqQR1AhLKrMwCU+YmMfk5g==",
+      "version": "0.4.10",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.10.tgz",
+      "integrity": "sha512-CTGPpuzlqq2nVjUxI9gAJOT1oBANIovtIaFsOmBSnEAHgX7oeAxKy2b6L/kJzsgqSzvR5vfLwYcWFrr6ZmBxSA==",
       "cpu": [
         "x64"
       ],
@@ -377,9 +377,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.4.8",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.8.tgz",
-      "integrity": "sha512-VjUryVvEA04r0j4lU9pJy84cmjuQm1GhBzbPc8kwbn5voT4A6BPglrlNsU0Zc+j8Fbjyvauzw2lMEcMsF4F0rw==",
+      "version": "0.4.10",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.10.tgz",
+      "integrity": "sha512-Fd7r74coZyrKzkfXg4WthqOL+uKyJyPTia6imcrMNqKOlTGdKmHf02Qi2QxWZrFaabkRYo4Tpn5FeRJ3yYX8CA==",
       "cpu": [
         "x64"
       ],

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.4.8",
+  "version": "0.4.10",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -85,10 +85,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.8",
-    "@lancedb/vectordb-darwin-x64": "0.4.8",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.8",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.8",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.8"
+    "@lancedb/vectordb-darwin-arm64": "0.4.10",
+    "@lancedb/vectordb-darwin-x64": "0.4.10",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.10",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.10",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.10"
   }
 }

node/src/arrow.ts

@@ -14,8 +14,6 @@
 
 import {
   Field,
-  type FixedSizeListBuilder,
-  Float32,
   makeBuilder,
   RecordBatchFileWriter,
   Utf8,
@@ -26,14 +24,19 @@ import {
   Table as ArrowTable,
   RecordBatchStreamWriter,
   List,
-  Float64,
   RecordBatch,
   makeData,
   Struct,
-  type Float
+  type Float,
+  DataType,
+  Binary,
+  Float32
 } from 'apache-arrow'
 import { type EmbeddingFunction } from './index'
 
+/*
+ * Options to control how a column should be converted to a vector array
+ */
 export class VectorColumnOptions {
   /** Vector column type. */
   type: Float = new Float32()
@@ -45,14 +48,50 @@ export class VectorColumnOptions {
 
 /** Options to control the makeArrowTable call. */
 export class MakeArrowTableOptions {
-  /** Provided schema. */
+  /*
+   * Schema of the data.
+   *
+   * If this is not provided then the data type will be inferred from the
+   * JS type.  Integer numbers will become int64, floating point numbers
+   * will become float64 and arrays will become variable sized lists with
+   * the data type inferred from the first element in the array.
+   *
+   * The schema must be specified if there are no records (e.g. to make
+   * an empty table)
+   */
   schema?: Schema
 
-  /** Vector columns */
+  /*
+   * Mapping from vector column name to expected type
+   *
+   * Lance expects vector columns to be fixed size list arrays (i.e. tensors)
+   * However, `makeArrowTable` will not infer this by default (it creates
+   * variable size list arrays).  This field can be used to indicate that a column
+   * should be treated as a vector column and converted to a fixed size list.
+   *
+   * The keys should be the names of the vector columns.  The value specifies the
+   * expected data type of the vector columns.
+   *
+   * If `schema` is provided then this field is ignored.
+   *
+   * By default, the column named "vector" will be assumed to be a float32
+   * vector column.
+   */
   vectorColumns: Record<string, VectorColumnOptions> = {
     vector: new VectorColumnOptions()
   }
 
+  /**
+   * If true then string columns will be encoded with dictionary encoding
+   *
+   * Set this to true if your string columns tend to repeat the same values
+   * often.  For more precise control use the `schema` property to specify the
+   * data type for individual columns.
+   *
+   * If `schema` is provided then this property is ignored.
+   */
+  dictionaryEncodeStrings: boolean = false
+
   constructor (values?: Partial<MakeArrowTableOptions>) {
     Object.assign(this, values)
   }
@@ -62,8 +101,29 @@ export class MakeArrowTableOptions {
  * An enhanced version of the {@link makeTable} function from Apache Arrow
  * that supports nested fields and embeddings columns.
  *
+ * This function converts an array of Record<String, any> (row-major JS objects)
+ * to an Arrow Table (a columnar structure)
+ *
  * Note that it currently does not support nulls.
  *
+ * If a schema is provided then it will be used to determine the resulting array
+ * types.  Fields will also be reordered to fit the order defined by the schema.
+ *
+ * If a schema is not provided then the types will be inferred and the field order
+ * will be controlled by the order of properties in the first record.
+ *
+ * If the input is empty then a schema must be provided to create an empty table.
+ *
+ * When a schema is not specified then data types will be inferred.  The inference
+ * rules are as follows:
+ *
+ *  - boolean => Bool
+ *  - number => Float64
+ *  - String => Utf8
+ *  - Buffer => Binary
+ *  - Record<String, any> => Struct
+ *  - Array<any> => List
+ *
  * @param data input data
  * @param options options to control the makeArrowTable call.
  *
@@ -86,8 +146,10 @@ export class MakeArrowTableOptions {
  *  ], { schema });
  * ```
  *
- * It guesses the vector columns if the schema is not provided. For example,
- * by default it assumes that the column named `vector` is a vector column.
+ * By default it assumes that the column named `vector` is a vector column
+ * and it will be converted into a fixed size list array of type float32.
+ * The `vectorColumns` option can be used to support other vector column
+ * names and data types.
  *
  * ```ts
  *
@@ -134,211 +196,304 @@ export function makeArrowTable (
   data: Array<Record<string, any>>,
   options?: Partial<MakeArrowTableOptions>
 ): ArrowTable {
-  if (data.length === 0) {
-    throw new Error('At least one record needs to be provided')
+  if (data.length === 0 && (options?.schema === undefined || options?.schema === null)) {
+    throw new Error('At least one record or a schema needs to be provided')
   }
+
   const opt = new MakeArrowTableOptions(options !== undefined ? options : {})
   const columns: Record<string, Vector> = {}
   // TODO: sample dataset to find missing columns
-  const columnNames = Object.keys(data[0])
+  // Prefer the field ordering of the schema, if present
+  const columnNames = ((options?.schema) != null) ? (options?.schema?.names as string[]) : Object.keys(data[0])
   for (const colName of columnNames) {
-    const values = data.map((datum) => datum[colName])
-    let vector: Vector
+    if (data.length !== 0 && !Object.prototype.hasOwnProperty.call(data[0], colName)) {
+      // The field is present in the schema, but not in the data, skip it
+      continue
+    }
+    // Extract a single column from the records (transpose from row-major to col-major)
+    let values = data.map((datum) => datum[colName])
 
+    // By default (type === undefined) arrow will infer the type from the JS type
+    let type
     if (opt.schema !== undefined) {
-      // Explicit schema is provided, highest priority
-      vector = vectorFromArray(
-        values,
-        opt.schema?.fields.filter((f) => f.name === colName)[0]?.type
-      )
+      // If there is a schema provided, then use that for the type instead
+      type = opt.schema?.fields.filter((f) => f.name === colName)[0]?.type
+      if (DataType.isInt(type) && type.bitWidth === 64) {
+        // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
+        values = values.map((v) => {
+          if (v === null) {
+            return v
+          }
+          return BigInt(v)
+        })
+      }
     } else {
+      // Otherwise, check to see if this column is one of the vector columns
+      // defined by opt.vectorColumns and, if so, use the fixed size list type
       const vectorColumnOptions = opt.vectorColumns[colName]
       if (vectorColumnOptions !== undefined) {
-        const fslType = new FixedSizeList(
-          values[0].length,
-          new Field('item', vectorColumnOptions.type, false)
-        )
-        vector = vectorFromArray(values, fslType)
-      } else {
-        // Normal case
-        vector = vectorFromArray(values)
+        type = newVectorType(values[0].length, vectorColumnOptions.type)
       }
     }
-    columns[colName] = vector
+
+    try {
+      // Convert an Array of JS values to an arrow vector
+      columns[colName] = makeVector(values, type, opt.dictionaryEncodeStrings)
+    } catch (error: unknown) {
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+      throw Error(`Could not convert column "${colName}" to Arrow: ${error}`)
+    }
   }
 
-  return new ArrowTable(columns)
+  if (opt.schema != null) {
+    // `new ArrowTable(columns)` infers a schema which may sometimes have
+    // incorrect nullability (it assumes nullable=true if there are 0 rows)
+    //
+    // `new ArrowTable(schema, columns)` will also fail because it will create a
+    // batch with an inferred schema and then complain that the batch schema
+    // does not match the provided schema.
+    //
+    // To work around this we first create a table with the wrong schema and
+    // then patch the schema of the batches so we can use
+    // `new ArrowTable(schema, batches)` which does not do any schema inference
+    const firstTable = new ArrowTable(columns)
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const batchesFixed = firstTable.batches.map(batch => new RecordBatch(opt.schema!, batch.data))
+    return new ArrowTable(opt.schema, batchesFixed)
+  } else {
+    return new ArrowTable(columns)
+  }
 }
 
-// Converts an Array of records into an Arrow Table, optionally applying an embeddings function to it.
+/**
+ * Create an empty Arrow table with the provided schema
+ */
+export function makeEmptyTable (schema: Schema): ArrowTable {
+  return makeArrowTable([], { schema })
+}
+
+// Helper function to convert Array<Array<any>> to a variable sized list array
+function makeListVector (lists: any[][]): Vector<any> {
+  if (lists.length === 0 || lists[0].length === 0) {
+    throw Error('Cannot infer list vector from empty array or empty list')
+  }
+  const sampleList = lists[0]
+  let inferredType
+  try {
+    const sampleVector = makeVector(sampleList)
+    inferredType = sampleVector.type
+  } catch (error: unknown) {
+    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+    throw Error(`Cannot infer list vector.  Cannot infer inner type: ${error}`)
+  }
+
+  const listBuilder = makeBuilder({
+    type: new List(new Field('item', inferredType, true))
+  })
+  for (const list of lists) {
+    listBuilder.append(list)
+  }
+  return listBuilder.finish().toVector()
+}
+
+// Helper function to convert an Array of JS values to an Arrow Vector
+function makeVector (values: any[], type?: DataType, stringAsDictionary?: boolean): Vector<any> {
+  if (type !== undefined) {
+    // No need for inference, let Arrow create it
+    return vectorFromArray(values, type)
+  }
+  if (values.length === 0) {
+    throw Error('makeVector requires at least one value or the type must be specfied')
+  }
+  const sampleValue = values.find(val => val !== null && val !== undefined)
+  if (sampleValue === undefined) {
+    throw Error('makeVector cannot infer the type if all values are null or undefined')
+  }
+  if (Array.isArray(sampleValue)) {
+    // Default Arrow inference doesn't handle list types
+    return makeListVector(values)
+  } else if (Buffer.isBuffer(sampleValue)) {
+    // Default Arrow inference doesn't handle Buffer
+    return vectorFromArray(values, new Binary())
+  } else if (!(stringAsDictionary ?? false) && (typeof sampleValue === 'string' || sampleValue instanceof String)) {
+    // If the type is string then don't use Arrow's default inference unless dictionaries are requested
+    // because it will always use dictionary encoding for strings
+    return vectorFromArray(values, new Utf8())
+  } else {
+    // Convert a JS array of values to an arrow vector
+    return vectorFromArray(values)
+  }
+}
+
+async function applyEmbeddings<T> (table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<ArrowTable> {
+  if (embeddings == null) {
+    return table
+  }
+
+  // Convert from ArrowTable to Record<String, Vector>
+  const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
+    const name = table.schema.fields[idx].name
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const vec = table.getChildAt(idx)!
+    return [name, vec]
+  })
+  const newColumns = Object.fromEntries(colEntries)
+
+  const sourceColumn = newColumns[embeddings.sourceColumn]
+  const destColumn = embeddings.destColumn ?? 'vector'
+  const innerDestType = embeddings.embeddingDataType ?? new Float32()
+  if (sourceColumn === undefined) {
+    throw new Error(`Cannot apply embedding function because the source column '${embeddings.sourceColumn}' was not present in the data`)
+  }
+
+  if (table.numRows === 0) {
+    if (Object.prototype.hasOwnProperty.call(newColumns, destColumn)) {
+      // We have an empty table and it already has the embedding column so no work needs to be done
+      // Note: we don't return an error like we did below because this is a common occurrence.  For example,
+      // if we call convertToTable with 0 records and a schema that includes the embedding
+      return table
+    }
+    if (embeddings.embeddingDimension !== undefined) {
+      const destType = newVectorType(embeddings.embeddingDimension, innerDestType)
+      newColumns[destColumn] = makeVector([], destType)
+    } else if (schema != null) {
+      const destField = schema.fields.find(f => f.name === destColumn)
+      if (destField != null) {
+        newColumns[destColumn] = makeVector([], destField.type)
+      } else {
+        throw new Error(`Attempt to apply embeddings to an empty table failed because schema was missing embedding column '${destColumn}'`)
+      }
+    } else {
+      throw new Error('Attempt to apply embeddings to an empty table when the embeddings function does not specify `embeddingDimension`')
+    }
+  } else {
+    if (Object.prototype.hasOwnProperty.call(newColumns, destColumn)) {
+      throw new Error(`Attempt to apply embeddings to table failed because column ${destColumn} already existed`)
+    }
+    if (table.batches.length > 1) {
+      throw new Error('Internal error: `makeArrowTable` unexpectedly created a table with more than one batch')
+    }
+    const values = sourceColumn.toArray()
+    const vectors = await embeddings.embed(values as T[])
+    if (vectors.length !== values.length) {
+      throw new Error('Embedding function did not return an embedding for each input element')
+    }
+    const destType = newVectorType(vectors[0].length, innerDestType)
+    newColumns[destColumn] = makeVector(vectors, destType)
+  }
+
+  const newTable = new ArrowTable(newColumns)
+  if (schema != null) {
+    if (schema.fields.find(f => f.name === destColumn) === undefined) {
+      throw new Error(`When using embedding functions and specifying a schema the schema should include the embedding column but the column ${destColumn} was missing`)
+    }
+    return alignTable(newTable, schema)
+  }
+  return newTable
+}
+
+/*
+ * Convert an Array of records into an Arrow Table, optionally applying an
+ * embeddings function to it.
+ *
+ * This function calls `makeArrowTable` first to create the Arrow Table.
+ * Any provided `makeTableOptions` (e.g. a schema) will be passed on to
+ * that call.
+ *
+ * The embedding function will be passed a column of values (based on the
+ * `sourceColumn` of the embedding function) and expects to receive back
+ * number[][] which will be converted into a fixed size list column.  By
+ * default this will be a fixed size list of Float32 but that can be
+ * customized by the `embeddingDataType` property of the embedding function.
+ *
+ * If a schema is provided in `makeTableOptions` then it should include the
+ * embedding columns.  If no schema is provded then embedding columns will
+ * be placed at the end of the table, after all of the input columns.
+ */
 export async function convertToTable<T> (
   data: Array<Record<string, unknown>>,
-  embeddings?: EmbeddingFunction<T>
+  embeddings?: EmbeddingFunction<T>,
+  makeTableOptions?: Partial<MakeArrowTableOptions>
 ): Promise<ArrowTable> {
-  if (data.length === 0) {
-    throw new Error('At least one record needs to be provided')
-  }
-
-  const columns = Object.keys(data[0])
-  const records: Record<string, Vector> = {}
-
-  for (const columnsKey of columns) {
-    if (columnsKey === 'vector') {
-      const vectorSize = (data[0].vector as any[]).length
-      const listBuilder = newVectorBuilder(vectorSize)
-      for (const datum of data) {
-        if ((datum[columnsKey] as any[]).length !== vectorSize) {
-          throw new Error(`Invalid vector size, expected ${vectorSize}`)
-        }
-
-        listBuilder.append(datum[columnsKey])
-      }
-      records[columnsKey] = listBuilder.finish().toVector()
-    } else {
-      const values = []
-      for (const datum of data) {
-        values.push(datum[columnsKey])
-      }
-
-      if (columnsKey === embeddings?.sourceColumn) {
-        const vectors = await embeddings.embed(values as T[])
-        records.vector = vectorFromArray(
-          vectors,
-          newVectorType(vectors[0].length)
-        )
-      }
-
-      if (typeof values[0] === 'string') {
-        // `vectorFromArray` converts strings into dictionary vectors, forcing it back to a string column
-        records[columnsKey] = vectorFromArray(values, new Utf8())
-      } else if (Array.isArray(values[0])) {
-        const elementType = getElementType(values[0])
-        let innerType
-        if (elementType === 'string') {
-          innerType = new Utf8()
-        } else if (elementType === 'number') {
-          innerType = new Float64()
-        } else {
-          // TODO: pass in schema if it exists, else keep going to the next element
-          throw new Error(`Unsupported array element type ${elementType}`)
-        }
-        const listBuilder = makeBuilder({
-          type: new List(new Field('item', innerType, true))
-        })
-        for (const value of values) {
-          listBuilder.append(value)
-        }
-        records[columnsKey] = listBuilder.finish().toVector()
-      } else {
-        // TODO if this is a struct field then recursively align the subfields
-        records[columnsKey] = vectorFromArray(values)
-      }
-    }
-  }
-
-  return new ArrowTable(records)
-}
-
-function getElementType (arr: any[]): string {
-  if (arr.length === 0) {
-    return 'undefined'
-  }
-
-  return typeof arr[0]
-}
-
-// Creates a new Arrow ListBuilder that stores a Vector column
-function newVectorBuilder (dim: number): FixedSizeListBuilder<Float32> {
-  return makeBuilder({
-    type: newVectorType(dim)
-  })
+  const table = makeArrowTable(data, makeTableOptions)
+  return await applyEmbeddings(table, embeddings, makeTableOptions?.schema)
 }
 
 // Creates the Arrow Type for a Vector column with dimension `dim`
-function newVectorType (dim: number): FixedSizeList<Float32> {
+function newVectorType <T extends Float> (dim: number, innerType: T): FixedSizeList<T> {
   // Somewhere we always default to have the elements nullable, so we need to set it to true
   // otherwise we often get schema mismatches because the stored data always has schema with nullable elements
-  const children = new Field<Float32>('item', new Float32(), true)
+  const children = new Field<T>('item', innerType, true)
   return new FixedSizeList(dim, children)
 }
 
-// Converts an Array of records into Arrow IPC format
+/**
+ * Serialize an Array of records into a buffer using the Arrow IPC File serialization
+ *
+ * This function will call `convertToTable` and pass on `embeddings` and `schema`
+ *
+ * `schema` is required if data is empty
+ */
 export async function fromRecordsToBuffer<T> (
   data: Array<Record<string, unknown>>,
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
-  let table = await convertToTable(data, embeddings)
-  if (schema !== undefined) {
-    table = alignTable(table, schema)
-  }
+  const table = await convertToTable(data, embeddings, { schema })
   const writer = RecordBatchFileWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }
 
-// Converts an Array of records into Arrow IPC stream format
+/**
+ * Serialize an Array of records into a buffer using the Arrow IPC Stream serialization
+ *
+ * This function will call `convertToTable` and pass on `embeddings` and `schema`
+ *
+ * `schema` is required if data is empty
+ */
 export async function fromRecordsToStreamBuffer<T> (
   data: Array<Record<string, unknown>>,
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
-  let table = await convertToTable(data, embeddings)
-  if (schema !== undefined) {
-    table = alignTable(table, schema)
-  }
+  const table = await convertToTable(data, embeddings, { schema })
   const writer = RecordBatchStreamWriter.writeAll(table)
   return Buffer.from(await writer.toUint8Array())
 }
 
-// Converts an Arrow Table into Arrow IPC format
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
 export async function fromTableToBuffer<T> (
   table: ArrowTable,
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
-  if (embeddings !== undefined) {
-    const source = table.getChild(embeddings.sourceColumn)
-
-    if (source === null) {
-      throw new Error(
-        `The embedding source column ${embeddings.sourceColumn} was not found in the Arrow Table`
-      )
-    }
-
-    const vectors = await embeddings.embed(source.toArray() as T[])
-    const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
-    table = table.assign(new ArrowTable({ vector: column }))
-  }
-  if (schema !== undefined) {
-    table = alignTable(table, schema)
-  }
-  const writer = RecordBatchFileWriter.writeAll(table)
+  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema)
+  const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings)
   return Buffer.from(await writer.toUint8Array())
 }
 
-// Converts an Arrow Table into Arrow IPC stream format
+/**
+ * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
+ *
+ * This function will apply `embeddings` to the table in a manner similar to
+ * `convertToTable`.
+ *
+ * `schema` is required if the table is empty
+ */
 export async function fromTableToStreamBuffer<T> (
   table: ArrowTable,
   embeddings?: EmbeddingFunction<T>,
   schema?: Schema
 ): Promise<Buffer> {
-  if (embeddings !== undefined) {
-    const source = table.getChild(embeddings.sourceColumn)
-
-    if (source === null) {
-      throw new Error(
-        `The embedding source column ${embeddings.sourceColumn} was not found in the Arrow Table`
-      )
-    }
-
-    const vectors = await embeddings.embed(source.toArray() as T[])
-    const column = vectorFromArray(vectors, newVectorType(vectors[0].length))
-    table = table.assign(new ArrowTable({ vector: column }))
-  }
-  if (schema !== undefined) {
-    table = alignTable(table, schema)
-  }
-  const writer = RecordBatchStreamWriter.writeAll(table)
+  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema)
+  const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings)
   return Buffer.from(await writer.toUint8Array())
 }
 

node/src/embedding/embedding_function.ts

@@ -12,18 +12,53 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import { type Float } from 'apache-arrow'
+
 /**
  * An embedding function that automatically creates vector representation for a given column.
  */
 export interface EmbeddingFunction<T> {
   /**
-     * The name of the column that will be used as input for the Embedding Function.
-     */
+   * The name of the column that will be used as input for the Embedding Function.
+   */
   sourceColumn: string
 
   /**
-     * Creates a vector representation for the given values.
-     */
+   * The data type of the embedding
+   *
+   * The embedding function should return `number`.  This will be converted into
+   * an Arrow float array.  By default this will be Float32 but this property can
+   * be used to control the conversion.
+   */
+  embeddingDataType?: Float
+
+  /**
+   * The dimension of the embedding
+   *
+   * This is optional, normally this can be determined by looking at the results of
+   * `embed`.  If this is not specified, and there is an attempt to apply the embedding
+   * to an empty table, then that process will fail.
+   */
+  embeddingDimension?: number
+
+  /**
+   * The name of the column that will contain the embedding
+   *
+   * By default this is "vector"
+   */
+  destColumn?: string
+
+  /**
+   * Should the source column be excluded from the resulting table
+   *
+   * By default the source column is included.  Set this to true and
+   * only the embedding will be stored.
+   */
+  excludeSource?: boolean
+
+  /**
+   * Creates a vector representation for the given values.
+   */
   embed: (data: T[]) => Promise<number[][]>
 }
 

node/src/index.ts

@@ -49,7 +49,7 @@ const {
 export { Query }
 export type { EmbeddingFunction }
 export { OpenAIEmbeddingFunction } from './embedding/openai'
-export { makeArrowTable, type MakeArrowTableOptions } from './arrow'
+export { convertToTable, makeArrowTable, type MakeArrowTableOptions } from './arrow'
 
 const defaultAwsRegion = 'us-west-2'
 

node/src/test/arrow.test.ts

@@ -13,9 +13,10 @@
 // limitations under the License.
 
 import { describe } from 'mocha'
-import { assert } from 'chai'
+import { assert, expect, use as chaiUse } from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
 
-import { fromTableToBuffer, makeArrowTable } from '../arrow'
+import { convertToTable, fromTableToBuffer, makeArrowTable, makeEmptyTable } from '../arrow'
 import {
   Field,
   FixedSizeList,
@@ -24,21 +25,79 @@ import {
   Int32,
   tableFromIPC,
   Schema,
-  Float64
+  Float64,
+  type Table,
+  Binary,
+  Bool,
+  Utf8,
+  Struct,
+  List,
+  DataType,
+  Dictionary,
+  Int64
 } from 'apache-arrow'
+import { type EmbeddingFunction } from '../embedding/embedding_function'
 
-describe('Apache Arrow tables', function () {
-  it('customized schema', async function () {
+chaiUse(chaiAsPromised)
+
+function sampleRecords (): Array<Record<string, any>> {
+  return [
+    {
+      binary: Buffer.alloc(5),
+      boolean: false,
+      number: 7,
+      string: 'hello',
+      struct: { x: 0, y: 0 },
+      list: ['anime', 'action', 'comedy']
+    }
+  ]
+}
+
+// Helper method to verify various ways to create a table
+async function checkTableCreation (tableCreationMethod: (records: any, recordsReversed: any, schema: Schema) => Promise<Table>): Promise<void> {
+  const records = sampleRecords()
+  const recordsReversed = [{
+    list: ['anime', 'action', 'comedy'],
+    struct: { x: 0, y: 0 },
+    string: 'hello',
+    number: 7,
+    boolean: false,
+    binary: Buffer.alloc(5)
+  }]
+  const schema = new Schema([
+    new Field('binary', new Binary(), false),
+    new Field('boolean', new Bool(), false),
+    new Field('number', new Float64(), false),
+    new Field('string', new Utf8(), false),
+    new Field('struct', new Struct([
+      new Field('x', new Float64(), false),
+      new Field('y', new Float64(), false)
+    ])),
+    new Field('list', new List(new Field('item', new Utf8(), false)), false)
+  ])
+
+  const table = await tableCreationMethod(records, recordsReversed, schema)
+  schema.fields.forEach((field, idx) => {
+    const actualField = table.schema.fields[idx]
+    assert.isFalse(actualField.nullable)
+    assert.equal(table.getChild(field.name)?.type.toString(), field.type.toString())
+    assert.equal(table.getChildAt(idx)?.type.toString(), field.type.toString())
+  })
+}
+
+describe('The function makeArrowTable', function () {
+  it('will use data types from a provided schema instead of inference', async function () {
     const schema = new Schema([
       new Field('a', new Int32()),
       new Field('b', new Float32()),
-      new Field('c', new FixedSizeList(3, new Field('item', new Float16())))
+      new Field('c', new FixedSizeList(3, new Field('item', new Float16()))),
+      new Field('d', new Int64())
     ])
     const table = makeArrowTable(
       [
-        { a: 1, b: 2, c: [1, 2, 3] },
-        { a: 4, b: 5, c: [4, 5, 6] },
-        { a: 7, b: 8, c: [7, 8, 9] }
+        { a: 1, b: 2, c: [1, 2, 3], d: 9 },
+        { a: 4, b: 5, c: [4, 5, 6], d: 10 },
+        { a: 7, b: 8, c: [7, 8, 9], d: null }
       ],
       { schema }
     )
@@ -52,13 +111,13 @@ describe('Apache Arrow tables', function () {
     assert.deepEqual(actualSchema, schema)
   })
 
-  it('default vector column', async function () {
+  it('will assume the column `vector` is FixedSizeList<Float32> by default', async function () {
     const schema = new Schema([
       new Field('a', new Float64()),
       new Field('b', new Float64()),
       new Field(
         'vector',
-        new FixedSizeList(3, new Field('item', new Float32()))
+        new FixedSizeList(3, new Field('item', new Float32(), true))
       )
     ])
     const table = makeArrowTable([
@@ -76,12 +135,12 @@ describe('Apache Arrow tables', function () {
     assert.deepEqual(actualSchema, schema)
   })
 
-  it('2 vector columns', async function () {
+  it('can support multiple vector columns', async function () {
     const schema = new Schema([
       new Field('a', new Float64()),
       new Field('b', new Float64()),
-      new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
-      new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+      new Field('vec1', new FixedSizeList(3, new Field('item', new Float16(), true))),
+      new Field('vec2', new FixedSizeList(3, new Field('item', new Float16(), true)))
     ])
     const table = makeArrowTable(
       [
@@ -105,4 +164,157 @@ describe('Apache Arrow tables', function () {
     const actualSchema = actual.schema
     assert.deepEqual(actualSchema, schema)
   })
+
+  it('will allow different vector column types', async function () {
+    const table = makeArrowTable(
+      [
+        { fp16: [1], fp32: [1], fp64: [1] }
+      ],
+      {
+        vectorColumns: {
+          fp16: { type: new Float16() },
+          fp32: { type: new Float32() },
+          fp64: { type: new Float64() }
+        }
+      }
+    )
+
+    assert.equal(table.getChild('fp16')?.type.children[0].type.toString(), new Float16().toString())
+    assert.equal(table.getChild('fp32')?.type.children[0].type.toString(), new Float32().toString())
+    assert.equal(table.getChild('fp64')?.type.children[0].type.toString(), new Float64().toString())
+  })
+
+  it('will use dictionary encoded strings if asked', async function () {
+    const table = makeArrowTable([{ str: 'hello' }])
+    assert.isTrue(DataType.isUtf8(table.getChild('str')?.type))
+
+    const tableWithDict = makeArrowTable([{ str: 'hello' }], { dictionaryEncodeStrings: true })
+    assert.isTrue(DataType.isDictionary(tableWithDict.getChild('str')?.type))
+
+    const schema = new Schema([
+      new Field('str', new Dictionary(new Utf8(), new Int32()))
+    ])
+
+    const tableWithDict2 = makeArrowTable([{ str: 'hello' }], { schema })
+    assert.isTrue(DataType.isDictionary(tableWithDict2.getChild('str')?.type))
+  })
+
+  it('will infer data types correctly', async function () {
+    await checkTableCreation(async (records) => makeArrowTable(records))
+  })
+
+  it('will allow a schema to be provided', async function () {
+    await checkTableCreation(async (records, _, schema) => makeArrowTable(records, { schema }))
+  })
+
+  it('will use the field order of any provided schema', async function () {
+    await checkTableCreation(async (_, recordsReversed, schema) => makeArrowTable(recordsReversed, { schema }))
+  })
+
+  it('will make an empty table', async function () {
+    await checkTableCreation(async (_, __, schema) => makeArrowTable([], { schema }))
+  })
+})
+
+class DummyEmbedding implements EmbeddingFunction<string> {
+  public readonly sourceColumn = 'string'
+  public readonly embeddingDimension = 2
+  public readonly embeddingDataType = new Float16()
+
+  async embed (data: string[]): Promise<number[][]> {
+    return data.map(
+      () => [0.0, 0.0]
+    )
+  }
+}
+
+class DummyEmbeddingWithNoDimension implements EmbeddingFunction<string> {
+  public readonly sourceColumn = 'string'
+
+  async embed (data: string[]): Promise<number[][]> {
+    return data.map(
+      () => [0.0, 0.0]
+    )
+  }
+}
+
+describe('convertToTable', function () {
+  it('will infer data types correctly', async function () {
+    await checkTableCreation(async (records) => await convertToTable(records))
+  })
+
+  it('will allow a schema to be provided', async function () {
+    await checkTableCreation(async (records, _, schema) => await convertToTable(records, undefined, { schema }))
+  })
+
+  it('will use the field order of any provided schema', async function () {
+    await checkTableCreation(async (_, recordsReversed, schema) => await convertToTable(recordsReversed, undefined, { schema }))
+  })
+
+  it('will make an empty table', async function () {
+    await checkTableCreation(async (_, __, schema) => await convertToTable([], undefined, { schema }))
+  })
+
+  it('will apply embeddings', async function () {
+    const records = sampleRecords()
+    const table = await convertToTable(records, new DummyEmbedding())
+    assert.isTrue(DataType.isFixedSizeList(table.getChild('vector')?.type))
+    assert.equal(table.getChild('vector')?.type.children[0].type.toString(), new Float16().toString())
+  })
+
+  it('will fail if missing the embedding source column', async function () {
+    return await expect(convertToTable([{ id: 1 }], new DummyEmbedding())).to.be.rejectedWith("'string' was not present")
+  })
+
+  it('use embeddingDimension if embedding missing from table', async function () {
+    const schema = new Schema([
+      new Field('string', new Utf8(), false)
+    ])
+    // Simulate getting an empty Arrow table (minus embedding) from some other source
+    // In other words, we aren't starting with records
+    const table = makeEmptyTable(schema)
+
+    // If the embedding specifies the dimension we are fine
+    await fromTableToBuffer(table, new DummyEmbedding())
+
+    // We can also supply a schema and should be ok
+    const schemaWithEmbedding = new Schema([
+      new Field('string', new Utf8(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float16(), false)), false)
+    ])
+    await fromTableToBuffer(table, new DummyEmbeddingWithNoDimension(), schemaWithEmbedding)
+
+    // Otherwise we will get an error
+    return await expect(fromTableToBuffer(table, new DummyEmbeddingWithNoDimension())).to.be.rejectedWith('does not specify `embeddingDimension`')
+  })
+
+  it('will apply embeddings to an empty table', async function () {
+    const schema = new Schema([
+      new Field('string', new Utf8(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float16(), false)), false)
+    ])
+    const table = await convertToTable([], new DummyEmbedding(), { schema })
+    assert.isTrue(DataType.isFixedSizeList(table.getChild('vector')?.type))
+    assert.equal(table.getChild('vector')?.type.children[0].type.toString(), new Float16().toString())
+  })
+
+  it('will complain if embeddings present but schema missing embedding column', async function () {
+    const schema = new Schema([
+      new Field('string', new Utf8(), false)
+    ])
+    return await expect(convertToTable([], new DummyEmbedding(), { schema })).to.be.rejectedWith('column vector was missing')
+  })
+
+  it('will provide a nice error if run twice', async function () {
+    const records = sampleRecords()
+    const table = await convertToTable(records, new DummyEmbedding())
+    // fromTableToBuffer will try and apply the embeddings again
+    return await expect(fromTableToBuffer(table, new DummyEmbedding())).to.be.rejectedWith('already existed')
+  })
+})
+
+describe('makeEmptyTable', function () {
+  it('will make an empty table', async function () {
+    await checkTableCreation(async (_, __, schema) => makeEmptyTable(schema))
+  })
 })

node/tsconfig.json

@@ -9,6 +9,6 @@
     "declaration": true,
     "outDir": "./dist",
     "strict": true,
-    // "esModuleInterop": true,
+    "sourceMap": true,
   }
 }

nodejs/Cargo.toml

@@ -23,5 +23,8 @@ napi = { version = "2.15", default-features = false, features = [
 ] }
 napi-derive = "2"
 
+# Prevent dynamic linking of lzma, which comes from datafusion
+lzma-sys = { version = "*", features = ["static"] }
+
 [build-dependencies]
 napi-build = "2.1"

nodejs/__test__/arrow.test.ts

@@ -14,6 +14,7 @@
 
 import { makeArrowTable, toBuffer } from "../vectordb/arrow";
 import {
+  Int64,
   Field,
   FixedSizeList,
   Float16,
@@ -104,3 +105,16 @@ test("2 vector columns", function () {
   const actualSchema = actual.schema;
   expect(actualSchema.toString()).toEqual(schema.toString());
 });
+
+test("handles int64", function() {
+  // https://github.com/lancedb/lancedb/issues/960
+  const schema = new Schema([
+    new Field("x", new Int64(), true)
+  ]);
+  const table = makeArrowTable([
+    { x: 1 },
+    { x: 2 },
+    { x: 3 }
+  ], { schema });
+  expect(table.schema).toEqual(schema);
+})

nodejs/vectordb/arrow.ts

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 import {
+  Int64,
   Field,
   FixedSizeList,
   Float,
@@ -23,6 +24,7 @@ import {
   Vector,
   vectorFromArray,
   tableToIPC,
+  DataType,
 } from "apache-arrow";
 
 /** Data type accepted by NodeJS SDK */
@@ -137,15 +139,18 @@ export function makeArrowTable(
   const columnNames = Object.keys(data[0]);
   for (const colName of columnNames) {
     // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-    const values = data.map((datum) => datum[colName]);
+    let values = data.map((datum) => datum[colName]);
     let vector: Vector;
 
     if (opt.schema !== undefined) {
       // Explicit schema is provided, highest priority
-      vector = vectorFromArray(
-        values,
-        opt.schema?.fields.filter((f) => f.name === colName)[0]?.type
-      );
+      const fieldType: DataType | undefined = opt.schema.fields.filter((f) => f.name === colName)[0]?.type as DataType;
+      if (fieldType instanceof Int64) {
+        // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+        values = values.map((v) => BigInt(v));
+      }
+      vector = vectorFromArray(values, fieldType);
     } else {
       const vectorColumnOptions = opt.vectorColumns[colName];
       if (vectorColumnOptions !== undefined) {

nodejs/vectordb/native.d.ts

@@ -73,7 +73,7 @@ export class Table {
   /** Return Schema as empty Arrow IPC file. */
   schema(): Buffer
   add(buf: Buffer): Promise<void>
-  countRows(filter?: string): Promise<bigint>
+  countRows(filter?: string | undefined | null): Promise<bigint>
   delete(predicate: string): Promise<void>
   createIndex(): IndexBuilder
   query(): Query

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.5.4
+current_version = 0.5.5
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/lancedb/embeddings/base.py

@@ -10,7 +10,6 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-import importlib
 from abc import ABC, abstractmethod
 from typing import List, Union
 
@@ -91,25 +90,6 @@ class EmbeddingFunction(BaseModel, ABC):
             texts = texts.combine_chunks().to_pylist()
         return texts
 
-    @classmethod
-    def safe_import(cls, module: str, mitigation=None):
-        """
-        Import the specified module. If the module is not installed,
-        raise an ImportError with a helpful message.
-
-        Parameters
-        ----------
-        module : str
-            The name of the module to import
-        mitigation : Optional[str]
-            The package(s) to install to mitigate the error.
-            If not provided then the module name will be used.
-        """
-        try:
-            return importlib.import_module(module)
-        except ImportError:
-            raise ImportError(f"Please install {mitigation or module}")
-
     def safe_model_dump(self):
         from ..pydantic import PYDANTIC_VERSION
 

python/lancedb/embeddings/bedrock.py

@@ -19,6 +19,7 @@ import numpy as np
 
 from lancedb.pydantic import PYDANTIC_VERSION
 
+from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
 from .registry import register
 from .utils import TEXT
@@ -183,8 +184,8 @@ class BedRockText(TextEmbeddingFunction):
         boto3.client
             The boto3 client for Amazon Bedrock service
         """
-        botocore = self.safe_import("botocore")
-        boto3 = self.safe_import("boto3")
+        botocore = attempt_import_or_raise("botocore")
+        boto3 = attempt_import_or_raise("boto3")
 
         session_kwargs = {"region_name": self.region}
         client_kwargs = {**session_kwargs}

python/lancedb/embeddings/cohere.py

@@ -16,6 +16,7 @@ from typing import ClassVar, List, Union
 
 import numpy as np
 
+from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
 from .registry import register
 from .utils import api_key_not_found_help
@@ -84,7 +85,7 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
         return [emb for emb in rs.embeddings]
 
     def _init_client(self):
-        cohere = self.safe_import("cohere")
+        cohere = attempt_import_or_raise("cohere")
         if CohereEmbeddingFunction.client is None:
             if os.environ.get("COHERE_API_KEY") is None:
                 api_key_not_found_help("cohere")

python/lancedb/embeddings/gemini_text.py

@@ -19,6 +19,7 @@ import numpy as np
 
 from lancedb.pydantic import PYDANTIC_VERSION
 
+from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
 from .registry import register
 from .utils import TEXT, api_key_not_found_help
@@ -134,7 +135,7 @@ class GeminiText(TextEmbeddingFunction):
 
     @cached_property
     def client(self):
-        genai = self.safe_import("google.generativeai", "google.generativeai")
+        genai = attempt_import_or_raise("google.generativeai", "google.generativeai")
 
         if not os.environ.get("GOOGLE_API_KEY"):
             api_key_not_found_help("google")

python/lancedb/embeddings/gte.py

@@ -14,6 +14,7 @@ from typing import List, Union
 
 import numpy as np
 
+from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
 from .registry import register
 from .utils import weak_lru
@@ -122,7 +123,7 @@ class GteEmbeddings(TextEmbeddingFunction):
 
             return Model()
         else:
-            sentence_transformers = self.safe_import(
+            sentence_transformers = attempt_import_or_raise(
                 "sentence_transformers", "sentence-transformers"
             )
             return sentence_transformers.SentenceTransformer(

python/lancedb/embeddings/instructor.py

@@ -14,6 +14,7 @@ from typing import List
 
 import numpy as np
 
+from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
 from .registry import register
 from .utils import TEXT, weak_lru
@@ -102,9 +103,9 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
     # convert_to_numpy: bool = True # Hardcoding this as numpy can be ingested directly
 
     source_instruction: str = "represent the document for retrieval"
-    query_instruction: str = (
-        "represent the document for retrieving the most similar documents"
-    )
+    query_instruction: (
+        str
+    ) = "represent the document for retrieving the most similar documents"
 
     @weak_lru(maxsize=1)
     def ndims(self):
@@ -131,10 +132,10 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
 
     @weak_lru(maxsize=1)
     def get_model(self):
-        instructor_embedding = self.safe_import(
+        instructor_embedding = attempt_import_or_raise(
             "InstructorEmbedding", "InstructorEmbedding"
         )
-        torch = self.safe_import("torch", "torch")
+        torch = attempt_import_or_raise("torch", "torch")
 
         model = instructor_embedding.INSTRUCTOR(self.name)
         if self.quantize:

python/lancedb/embeddings/open_clip.py

@@ -21,6 +21,7 @@ import pyarrow as pa
 from pydantic import PrivateAttr
 from tqdm import tqdm
 
+from ..util import attempt_import_or_raise
 from .base import EmbeddingFunction
 from .registry import register
 from .utils import IMAGES, url_retrieve
@@ -50,7 +51,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        open_clip = self.safe_import("open_clip", "open-clip")
+        open_clip = attempt_import_or_raise("open_clip", "open-clip")
         model, _, preprocess = open_clip.create_model_and_transforms(
             self.name, pretrained=self.pretrained
         )
@@ -78,14 +79,14 @@ class OpenClipEmbeddings(EmbeddingFunction):
         if isinstance(query, str):
             return [self.generate_text_embeddings(query)]
         else:
-            PIL = self.safe_import("PIL", "pillow")
+            PIL = attempt_import_or_raise("PIL", "pillow")
             if isinstance(query, PIL.Image.Image):
                 return [self.generate_image_embedding(query)]
             else:
                 raise TypeError("OpenClip supports str or PIL Image as query")
 
     def generate_text_embeddings(self, text: str) -> np.ndarray:
-        torch = self.safe_import("torch")
+        torch = attempt_import_or_raise("torch")
         text = self.sanitize_input(text)
         text = self._tokenizer(text)
         text.to(self.device)
@@ -144,7 +145,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
             The image to embed. If the image is a str, it is treated as a uri.
             If the image is bytes, it is treated as the raw image bytes.
         """
-        torch = self.safe_import("torch")
+        torch = attempt_import_or_raise("torch")
         # TODO handle retry and errors for https
         image = self._to_pil(image)
         image = self._preprocess(image).unsqueeze(0)
@@ -152,7 +153,7 @@ class OpenClipEmbeddings(EmbeddingFunction):
             return self._encode_and_normalize_image(image)
 
     def _to_pil(self, image: Union[str, bytes]):
-        PIL = self.safe_import("PIL", "pillow")
+        PIL = attempt_import_or_raise("PIL", "pillow")
         if isinstance(image, bytes):
             return PIL.Image.open(io.BytesIO(image))
         if isinstance(image, PIL.Image.Image):

python/lancedb/embeddings/openai.py

@@ -16,6 +16,7 @@ from typing import List, Optional, Union
 
 import numpy as np
 
+from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
 from .registry import register
 from .utils import api_key_not_found_help
@@ -68,7 +69,7 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
 
     @cached_property
     def _openai_client(self):
-        openai = self.safe_import("openai")
+        openai = attempt_import_or_raise("openai")
 
         if not os.environ.get("OPENAI_API_KEY"):
             api_key_not_found_help("openai")

python/lancedb/embeddings/sentence_transformers.py

@@ -14,6 +14,7 @@ from typing import List, Union
 
 import numpy as np
 
+from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
 from .registry import register
 from .utils import weak_lru
@@ -75,7 +76,7 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction):
 
         TODO: use lru_cache instead with a reasonable/configurable maxsize
         """
-        sentence_transformers = self.safe_import(
+        sentence_transformers = attempt_import_or_raise(
             "sentence_transformers", "sentence-transformers"
         )
         return sentence_transformers.SentenceTransformer(self.name, device=self.device)

python/lancedb/query.py

@@ -24,7 +24,7 @@ import pyarrow as pa
 import pydantic
 
 from . import __version__
-from .common import VEC, VECTOR_COLUMN_NAME
+from .common import VEC
 from .rerankers.base import Reranker
 from .rerankers.linear_combination import LinearCombinationReranker
 from .util import safe_import_pandas
@@ -75,7 +75,7 @@ class Query(pydantic.BaseModel):
           tuning advice.
     """
 
-    vector_column: str = VECTOR_COLUMN_NAME
+    vector_column: Optional[str] = None
 
     # vector to search for
     vector: Union[List[float], List[List[float]]]
@@ -403,7 +403,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         self,
         table: "Table",
         query: Union[np.ndarray, list, "PIL.Image.Image"],
-        vector_column: str = VECTOR_COLUMN_NAME,
+        vector_column: str,
     ):
         super().__init__(table)
         self._query = query

python/lancedb/remote/table.py

@@ -24,7 +24,7 @@ from lancedb.merge import LanceMergeInsertBuilder
 
 from ..query import LanceVectorQueryBuilder
 from ..table import Query, Table, _sanitize_data
-from ..util import value_to_sql
+from ..util import inf_vector_column_query, value_to_sql
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE
 from .db import RemoteDBConnection
@@ -198,7 +198,9 @@ class RemoteTable(Table):
         )
 
     def search(
-        self, query: Union[VEC, str], vector_column_name: str = VECTOR_COLUMN_NAME
+        self,
+        query: Union[VEC, str],
+        vector_column_name: Optional[str] = None,
     ) -> LanceVectorQueryBuilder:
         """Create a search query to find the nearest neighbors
         of the given query vector. We currently support [vector search][search]
@@ -217,7 +219,7 @@ class RemoteTable(Table):
         ... ]
         >>> table = db.create_table("my_table", data) # doctest: +SKIP
         >>> query = [0.4, 1.4, 2.4]
-        >>> (table.search(query, vector_column_name="vector") # doctest: +SKIP
+        >>> (table.search(query) # doctest: +SKIP
         ...     .where("original_width > 1000", prefilter=True) # doctest: +SKIP
         ...     .select(["caption", "original_width"]) # doctest: +SKIP
         ...     .limit(2) # doctest: +SKIP
@@ -236,9 +238,14 @@ class RemoteTable(Table):
 
             - If None then the select/where/limit clauses are applied to filter
             the table
-        vector_column_name: str
+        vector_column_name: str, optional
             The name of the vector column to search.
-            *default "vector"*
+
+            - If not specified then the vector column is inferred from
+            the table schema
+
+            - If the table has multiple vector columns then the *vector_column_name*
+            needs to be specified. Otherwise, an error is raised.
 
         Returns
         -------
@@ -253,6 +260,8 @@ class RemoteTable(Table):
             - and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
+        if vector_column_name is None:
+            vector_column_name = inf_vector_column_query(self.schema)
         return LanceVectorQueryBuilder(self, query, vector_column_name)
 
     def _execute_query(self, query: Query) -> pa.Table:

python/lancedb/rerankers/cohere.py

@@ -4,7 +4,7 @@ from typing import Union
 
 import pyarrow as pa
 
-from ..util import safe_import
+from ..util import attempt_import_or_raise
 from .base import Reranker
 
 
@@ -41,7 +41,7 @@ class CohereReranker(Reranker):
 
     @cached_property
     def _client(self):
-        cohere = safe_import("cohere")
+        cohere = attempt_import_or_raise("cohere")
         if os.environ.get("COHERE_API_KEY") is None and self.api_key is None:
             raise ValueError(
                 "COHERE_API_KEY not set. Either set it in your environment or \

python/lancedb/rerankers/colbert.py

@@ -2,7 +2,7 @@ from functools import cached_property
 
 import pyarrow as pa
 
-from ..util import safe_import
+from ..util import attempt_import_or_raise
 from .base import Reranker
 
 
@@ -29,7 +29,9 @@ class ColbertReranker(Reranker):
         super().__init__(return_score)
         self.model_name = model_name
         self.column = column
-        self.torch = safe_import("torch")  # import here for faster ops later
+        self.torch = attempt_import_or_raise(
+            "torch"
+        )  # import here for faster ops later
 
     def rerank_hybrid(
         self,
@@ -80,7 +82,7 @@ class ColbertReranker(Reranker):
 
     @cached_property
     def _model(self):
-        transformers = safe_import("transformers")
+        transformers = attempt_import_or_raise("transformers")
         tokenizer = transformers.AutoTokenizer.from_pretrained(self.model_name)
         model = transformers.AutoModel.from_pretrained(self.model_name)
 

python/lancedb/rerankers/cross_encoder.py

@@ -3,7 +3,7 @@ from typing import Union
 
 import pyarrow as pa
 
-from ..util import safe_import
+from ..util import attempt_import_or_raise
 from .base import Reranker
 
 
@@ -32,7 +32,7 @@ class CrossEncoderReranker(Reranker):
         return_score="relevance",
     ):
         super().__init__(return_score)
-        torch = safe_import("torch")
+        torch = attempt_import_or_raise("torch")
         self.model_name = model_name
         self.column = column
         self.device = device
@@ -41,7 +41,7 @@ class CrossEncoderReranker(Reranker):
 
     @cached_property
     def model(self):
-        sbert = safe_import("sentence_transformers")
+        sbert = attempt_import_or_raise("sentence_transformers")
         cross_encoder = sbert.CrossEncoder(self.model_name)
 
         return cross_encoder

python/lancedb/rerankers/openai.py

@@ -5,7 +5,7 @@ from typing import Optional
 
 import pyarrow as pa
 
-from ..util import safe_import
+from ..util import attempt_import_or_raise
 from .base import Reranker
 
 
@@ -17,7 +17,7 @@ class OpenaiReranker(Reranker):
 
     Parameters
     ----------
-    model_name : str, default "gpt-3.5-turbo-1106	"
+    model_name : str, default "gpt-4-turbo-preview"
         The name of the cross encoder model to use.
     column : str, default "text"
         The name of the column to use as input to the cross encoder model.
@@ -29,7 +29,7 @@ class OpenaiReranker(Reranker):
 
     def __init__(
         self,
-        model_name: str = "gpt-3.5-turbo-1106",
+        model_name: str = "gpt-4-turbo-preview",
         column: str = "text",
         return_score="relevance",
         api_key: Optional[str] = None,
@@ -93,7 +93,9 @@ class OpenaiReranker(Reranker):
 
     @cached_property
     def _client(self):
-        openai = safe_import("openai")  # TODO: force version or handle versions < 1.0
+        openai = attempt_import_or_raise(
+            "openai"
+        )  # TODO: force version or handle versions < 1.0
         if os.environ.get("OPENAI_API_KEY") is None and self.api_key is None:
             raise ValueError(
                 "OPENAI_API_KEY not set. Either set it in your environment or \

python/lancedb/table.py

@@ -36,6 +36,7 @@ from .pydantic import LanceModel, model_to_dict
 from .query import LanceQueryBuilder, Query
 from .util import (
     fs_from_uri,
+    inf_vector_column_query,
     join_uri,
     safe_import_pandas,
     safe_import_polars,
@@ -413,7 +414,7 @@ class Table(ABC):
     def search(
         self,
         query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
-        vector_column_name: str = VECTOR_COLUMN_NAME,
+        vector_column_name: Optional[str] = None,
         query_type: str = "auto",
     ) -> LanceQueryBuilder:
         """Create a search query to find the nearest neighbors
@@ -433,7 +434,7 @@ class Table(ABC):
         ... ]
         >>> table = db.create_table("my_table", data)
         >>> query = [0.4, 1.4, 2.4]
-        >>> (table.search(query, vector_column_name="vector")
+        >>> (table.search(query)
         ...     .where("original_width > 1000", prefilter=True)
         ...     .select(["caption", "original_width"])
         ...     .limit(2)
@@ -452,11 +453,16 @@ class Table(ABC):
 
             - If None then the select/where/limit clauses are applied to filter
             the table
-        vector_column_name: str
+        vector_column_name: str, optional
             The name of the vector column to search.
 
             The vector column needs to be a pyarrow fixed size list type
-            *default "vector"*
+
+            - If not specified then the vector column is inferred from
+            the table schema
+
+            - If the table has multiple vector columns then the *vector_column_name*
+            needs to be specified. Otherwise, an error is raised.
         query_type: str
             *default "auto"*.
             Acceptable types are: "vector", "fts", "hybrid", or "auto"
@@ -1193,7 +1199,7 @@ class LanceTable(Table):
     def search(
         self,
         query: Optional[Union[VEC, str, "PIL.Image.Image", Tuple]] = None,
-        vector_column_name: str = VECTOR_COLUMN_NAME,
+        vector_column_name: Optional[str] = None,
         query_type: str = "auto",
     ) -> LanceQueryBuilder:
         """Create a search query to find the nearest neighbors
@@ -1211,7 +1217,7 @@ class LanceTable(Table):
         ... ]
         >>> table = db.create_table("my_table", data)
         >>> query = [0.4, 1.4, 2.4]
-        >>> (table.search(query, vector_column_name="vector")
+        >>> (table.search(query)
         ...     .where("original_width > 1000", prefilter=True)
         ...     .select(["caption", "original_width"])
         ...     .limit(2)
@@ -1230,8 +1236,17 @@ class LanceTable(Table):
 
             - If None then the select/[where][sql]/limit clauses are applied
             to filter the table
-        vector_column_name: str, default "vector"
+        vector_column_name: str, optional
             The name of the vector column to search.
+
+            The vector column needs to be a pyarrow fixed size list type
+            *default "vector"*
+
+            - If not specified then the vector column is inferred from
+            the table schema
+
+            - If the table has multiple vector columns then the *vector_column_name*
+            needs to be specified. Otherwise, an error is raised.
         query_type: str, default "auto"
             "vector", "fts", or "auto"
             If "auto" then the query type is inferred from the query;
@@ -1249,6 +1264,8 @@ class LanceTable(Table):
             and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
+        if vector_column_name is None and query is not None:
+            vector_column_name = inf_vector_column_query(self.schema)
         register_event("search_table")
         return LanceQueryBuilder.create(
             self, query, query_type, vector_column_name=vector_column_name
@@ -1435,6 +1452,7 @@ class LanceTable(Table):
 
     def _execute_query(self, query: Query) -> pa.Table:
         ds = self.to_lance()
+
         return ds.to_table(
             columns=query.columns,
             filter=query.filter,

python/lancedb/util.py

@@ -20,6 +20,7 @@ from typing import Tuple, Union
 from urllib.parse import urlparse
 
 import numpy as np
+import pyarrow as pa
 import pyarrow.fs as pa_fs
 
 
@@ -115,7 +116,7 @@ def join_uri(base: Union[str, pathlib.Path], *parts: str) -> str:
     return "/".join([p.rstrip("/") for p in [base, *parts]])
 
 
-def safe_import(module: str, mitigation=None):
+def attempt_import_or_raise(module: str, mitigation=None):
     """
     Import the specified module. If the module is not installed,
     raise an ImportError with a helpful message.
@@ -152,6 +153,44 @@ def safe_import_polars():
         return None
 
 
+def inf_vector_column_query(schema: pa.Schema) -> str:
+    """
+    Get the vector column name
+
+    Parameters
+    ----------
+    schema : pa.Schema
+        The schema of the vector column.
+
+    Returns
+    -------
+    str: the vector column name.
+    """
+    vector_col_name = ""
+    vector_col_count = 0
+    for field_name in schema.names:
+        field = schema.field(field_name)
+        if pa.types.is_fixed_size_list(field.type) and pa.types.is_floating(
+            field.type.value_type
+        ):
+            vector_col_count += 1
+            if vector_col_count > 1:
+                raise ValueError(
+                    "Schema has more than one vector column. "
+                    "Please specify the vector column name "
+                    "for vector search"
+                )
+                break
+            elif vector_col_count == 1:
+                vector_col_name = field_name
+    if vector_col_count == 0:
+        raise ValueError(
+            "There is no vector column in the data. "
+            "Please specify the vector column name for vector search"
+        )
+    return vector_col_name
+
+
 @singledispatch
 def value_to_sql(value):
     raise NotImplementedError("SQL conversion is not implemented for this type")

python/pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "lancedb"
-version = "0.5.4"
+version = "0.5.5"
 dependencies = [
     "deprecation",
-    "pylance==0.9.15",
+    "pylance==0.9.16",
     "ratelimiter~=1.0",
     "retry>=0.9.2",
     "tqdm>=4.27.0",

python/tests/test_embeddings_slow.py

@@ -69,10 +69,14 @@ def test_basic_text_embeddings(alias, tmp_path):
     )
 
     query = "greetings"
-    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    actual = (
+        table.search(query, vector_column_name="vector").limit(1).to_pydantic(Words)[0]
+    )
 
     vec = func.compute_query_embeddings(query)[0]
-    expected = table.search(vec).limit(1).to_pydantic(Words)[0]
+    expected = (
+        table.search(vec, vector_column_name="vector").limit(1).to_pydantic(Words)[0]
+    )
     assert actual.text == expected.text
     assert actual.text == "hello world"
     assert not np.allclose(actual.vector, actual.vector2)
@@ -116,7 +120,11 @@ def test_openclip(tmp_path):
     )
 
     # text search
-    actual = table.search("man's best friend").limit(1).to_pydantic(Images)[0]
+    actual = (
+        table.search("man's best friend", vector_column_name="vector")
+        .limit(1)
+        .to_pydantic(Images)[0]
+    )
     assert actual.label == "dog"
     frombytes = (
         table.search("man's best friend", vector_column_name="vec_from_bytes")
@@ -130,7 +138,11 @@ def test_openclip(tmp_path):
     query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
     image_bytes = requests.get(query_image_uri).content
     query_image = Image.open(io.BytesIO(image_bytes))
-    actual = table.search(query_image).limit(1).to_pydantic(Images)[0]
+    actual = (
+        table.search(query_image, vector_column_name="vector")
+        .limit(1)
+        .to_pydantic(Images)[0]
+    )
     assert actual.label == "dog"
     other = (
         table.search(query_image, vector_column_name="vec_from_bytes")

python/tests/test_remote_db.py

@@ -38,4 +38,5 @@ def test_remote_db():
     setattr(conn, "_client", FakeLanceDBClient())
 
     table = conn["test"]
+    table.schema = pa.schema([pa.field("vector", pa.list_(pa.float32(), 2))])
     table.search([1.0, 2.0]).to_pandas()

python/tests/test_table.py

@@ -710,6 +710,59 @@ def test_empty_query(db):
     assert len(df) == 100
 
 
+def test_search_with_schema_inf_single_vector(db):
+    class MyTable(LanceModel):
+        text: str
+        vector_col: Vector(10)
+
+    table = LanceTable.create(
+        db,
+        "my_table",
+        schema=MyTable,
+    )
+
+    v1 = np.random.randn(10)
+    v2 = np.random.randn(10)
+    data = [
+        {"vector_col": v1, "text": "foo"},
+        {"vector_col": v2, "text": "bar"},
+    ]
+    df = pd.DataFrame(data)
+    table.add(df)
+
+    q = np.random.randn(10)
+    result1 = table.search(q, vector_column_name="vector_col").limit(1).to_pandas()
+    result2 = table.search(q).limit(1).to_pandas()
+
+    assert result1["text"].iloc[0] == result2["text"].iloc[0]
+
+
+def test_search_with_schema_inf_multiple_vector(db):
+    class MyTable(LanceModel):
+        text: str
+        vector1: Vector(10)
+        vector2: Vector(10)
+
+    table = LanceTable.create(
+        db,
+        "my_table",
+        schema=MyTable,
+    )
+
+    v1 = np.random.randn(10)
+    v2 = np.random.randn(10)
+    data = [
+        {"vector1": v1, "vector2": v2, "text": "foo"},
+        {"vector1": v2, "vector2": v1, "text": "bar"},
+    ]
+    df = pd.DataFrame(data)
+    table.add(df)
+
+    q = np.random.randn(10)
+    with pytest.raises(ValueError):
+        table.search(q).limit(1).to_pandas()
+
+
 def test_compact_cleanup(db):
     table = LanceTable.create(
         db,

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.4.8"
+version = "0.4.10"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true
@@ -31,3 +31,6 @@ object_store = { workspace = true, features = ["aws"] }
 snafu = { workspace = true }
 async-trait = "0"
 env_logger = "0"
+
+# Prevent dynamic linking of lzma, which comes from datafusion
+lzma-sys = { version = "*", features = ["static"] }

rust/ffi/node/src/convert.rs

@@ -17,10 +17,7 @@ use neon::types::buffer::TypedArray;
 
 use crate::error::ResultExt;
 
-pub fn vec_str_to_array<'a, C: Context<'a>>(
-    vec: &Vec<String>,
-    cx: &mut C,
-) -> JsResult<'a, JsArray> {
+pub fn vec_str_to_array<'a, C: Context<'a>>(vec: &[String], cx: &mut C) -> JsResult<'a, JsArray> {
     let a = JsArray::new(cx, vec.len() as u32);
     for (i, s) in vec.iter().enumerate() {
         let v = cx.string(s);

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.4.8"
+version = "0.4.10"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true

rust/vectordb/src/connection.rs

@@ -422,13 +422,11 @@ mod tests {
         let tmp_dir = tempdir().unwrap();
         let uri = std::fs::canonicalize(tmp_dir.path().to_str().unwrap()).unwrap();
 
-        let mut relative_anacestors = vec![];
         let current_dir = std::env::current_dir().unwrap();
-        let mut ancestors = current_dir.ancestors();
-        while let Some(_) = ancestors.next() {
-            relative_anacestors.push("..");
-        }
-        let relative_root = std::path::PathBuf::from(relative_anacestors.join("/"));
+        let ancestors = current_dir.ancestors();
+        let relative_ancestors = vec![".."; ancestors.count()];
+
+        let relative_root = std::path::PathBuf::from(relative_ancestors.join("/"));
         let relative_uri = relative_root.join(&uri);
 
         let db = Database::connect(relative_uri.to_str().unwrap())

rust/vectordb/src/io/object_store.rs

@@ -357,12 +357,14 @@ mod test {
         let db = Database::connect(dir1.to_str().unwrap()).await.unwrap();
 
         let mut param = WriteParams::default();
-        let mut store_params = ObjectStoreParams::default();
-        store_params.object_store_wrapper = Some(object_store_wrapper);
+        let store_params = ObjectStoreParams {
+            object_store_wrapper: Some(object_store_wrapper),
+            ..Default::default()
+        };
         param.store_params = Some(store_params);
 
         let mut datagen = BatchGenerator::new();
-        datagen = datagen.col(Box::new(IncrementingInt32::default()));
+        datagen = datagen.col(Box::<IncrementingInt32>::default());
         datagen = datagen.col(Box::new(RandomVector::default().named("vector".into())));
 
         let res = db

rust/vectordb/src/query.rs

@@ -257,7 +257,7 @@ mod tests {
         assert_eq!(query.query_vector.unwrap(), new_vector);
         assert_eq!(query.limit.unwrap(), 100);
         assert_eq!(query.nprobes, 1000);
-        assert_eq!(query.use_index, true);
+        assert!(query.use_index);
         assert_eq!(query.metric_type, Some(MetricType::Cosine));
         assert_eq!(query.refine_factor, Some(999));
     }

rust/vectordb/src/table.rs

@@ -888,12 +888,12 @@ mod tests {
 
         let batches = make_test_batches();
         let _ = batches.schema().clone();
-        NativeTable::create(&uri, "test", batches, None, None)
+        NativeTable::create(uri, "test", batches, None, None)
             .await
             .unwrap();
 
         let batches = make_test_batches();
-        let result = NativeTable::create(&uri, "test", batches, None, None).await;
+        let result = NativeTable::create(uri, "test", batches, None, None).await;
         assert!(matches!(
             result.unwrap_err(),
             Error::TableAlreadyExists { .. }
@@ -906,7 +906,7 @@ mod tests {
         let uri = tmp_dir.path().to_str().unwrap();
 
         let batches = make_test_batches();
-        let table = NativeTable::create(&uri, "test", batches, None, None)
+        let table = NativeTable::create(uri, "test", batches, None, None)
             .await
             .unwrap();
 
@@ -924,7 +924,7 @@ mod tests {
 
         let batches = make_test_batches();
         let schema = batches.schema().clone();
-        let table = NativeTable::create(&uri, "test", batches, None, None)
+        let table = NativeTable::create(uri, "test", batches, None, None)
             .await
             .unwrap();
         assert_eq!(table.count_rows(None).await.unwrap(), 10);
@@ -952,7 +952,7 @@ mod tests {
 
         // Create a dataset with i=0..10
         let batches = merge_insert_test_batches(0, 0);
-        let table = NativeTable::create(&uri, "test", batches, None, None)
+        let table = NativeTable::create(uri, "test", batches, None, None)
             .await
             .unwrap();
         assert_eq!(table.count_rows(None).await.unwrap(), 10);
@@ -1149,12 +1149,8 @@ mod tests {
                     Arc::new(LargeStringArray::from_iter_values(vec![
                         "a", "b", "c", "d", "e", "f", "g", "h", "i", "j",
                     ])),
-                    Arc::new(Float32Array::from_iter_values(
-                        (0..10).into_iter().map(|i| i as f32),
-                    )),
-                    Arc::new(Float64Array::from_iter_values(
-                        (0..10).into_iter().map(|i| i as f64),
-                    )),
+                    Arc::new(Float32Array::from_iter_values((0..10).map(|i| i as f32))),
+                    Arc::new(Float64Array::from_iter_values((0..10).map(|i| i as f64))),
                     Arc::new(Into::<BooleanArray>::into(vec![
                         true, false, true, false, true, false, true, false, true, false,
                     ])),
@@ -1163,14 +1159,14 @@ mod tests {
                     Arc::new(TimestampMillisecondArray::from_iter_values(0..10)),
                     Arc::new(
                         create_fixed_size_list(
-                            Float32Array::from_iter_values((0..20).into_iter().map(|i| i as f32)),
+                            Float32Array::from_iter_values((0..20).map(|i| i as f32)),
                             2,
                         )
                         .unwrap(),
                     ),
                     Arc::new(
                         create_fixed_size_list(
-                            Float64Array::from_iter_values((0..20).into_iter().map(|i| i as f64)),
+                            Float64Array::from_iter_values((0..20).map(|i| i as f64)),
                             2,
                         )
                         .unwrap(),
@@ -1307,7 +1303,7 @@ mod tests {
             original: Arc<dyn object_store::ObjectStore>,
         ) -> Arc<dyn object_store::ObjectStore> {
             self.called.store(true, Ordering::Relaxed);
-            return original;
+            original
         }
     }
 
@@ -1324,8 +1320,10 @@ mod tests {
 
         let wrapper = Arc::new(NoOpCacheWrapper::default());
 
-        let mut object_store_params = ObjectStoreParams::default();
-        object_store_params.object_store_wrapper = Some(wrapper.clone());
+        let object_store_params = ObjectStoreParams {
+            object_store_wrapper: Some(wrapper.clone()),
+            ..Default::default()
+        };
         let param = ReadParams {
             store_options: Some(object_store_params),
             ..Default::default()