Bump version: 0.8.2-beta.0 → 0.8.2

This also exposed the max_batch_length configuration option in
python/node (it was needed to verify if we are actually in v2 mode or
not)

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.5.0"
+current_version = "0.5.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/java.yml

@@ -0,0 +1,85 @@
+name: Build and Run Java JNI Tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - java/**
+      - rust/**
+      - .github/workflows/java.yml
+env:
+  # This env var is used by Swatinem/rust-cache@v2 for the cache
+  # key, so we set it to make sure it is always consistent.
+  CARGO_TERM_COLOR: always
+  # Disable full debug symbol generation to speed up CI build and keep memory down
+  # "1" means line tables only, which is useful for panic tracebacks.
+  RUSTFLAGS: "-C debuginfo=1"
+  RUST_BACKTRACE: "1"
+  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
+  # CI builds are faster with incremental disabled.
+  CARGO_INCREMENTAL: "0"
+  CARGO_BUILD_JOBS: "1"
+jobs:
+  linux-build:
+    runs-on: ubuntu-22.04
+    name: ubuntu-22.04 + Java 11 & 17
+    defaults:
+      run:
+        working-directory: ./java
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: java/core/lancedb-jni
+      - name: Run cargo fmt
+        run: cargo fmt --check
+        working-directory: ./java/core/lancedb-jni
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install Java 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 17
+          cache: "maven"
+      - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV
+      - name: Install Java 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 11
+          cache: "maven"
+      - name: Java Style Check
+        run: mvn checkstyle:check
+      # Disable because of issues in lancedb rust core code
+      # - name: Rust Clippy
+      #   working-directory: java/core/lancedb-jni
+      #   run: cargo clippy --all-targets -- -D warnings
+      - name: Running tests with Java 11
+        run: mvn clean test
+      - name: Running tests with Java 17
+        run: |
+          export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \
+          -XX:+IgnoreUnrecognizedVMOptions \
+          --add-opens=java.base/java.lang=ALL-UNNAMED \
+          --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \
+          --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \
+          --add-opens=java.base/java.io=ALL-UNNAMED \
+          --add-opens=java.base/java.net=ALL-UNNAMED \
+          --add-opens=java.base/java.nio=ALL-UNNAMED \
+          --add-opens=java.base/java.util=ALL-UNNAMED \
+          --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \
+          --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \
+          --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \
+          --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
+          --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \
+          --add-opens=java.base/sun.security.action=ALL-UNNAMED \
+          --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \
+          --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
+          -Djdk.reflect.useDirectMethodHandle=false \
+          -Dio.netty.tryReflectionSetAccessible=true"
+          JAVA_HOME=$JAVA_17 mvn clean test

.github/workflows/make-release-commit.yml

@@ -94,6 +94,6 @@ jobs:
           branch: ${{ github.ref }}
           tags: true
       - uses: ./.github/workflows/update_package_lock
-        if: ${{ inputs.dry_run }} == "false"
+        if: ${{ !inputs.dry_run && inputs.other }}
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ repos:
   hooks:
     - id: local-biome-check
       name: biome check
-      entry: npx biome check
+      entry: npx @biomejs/biome check --config-path nodejs/biome.json nodejs/
       language: system
       types: [text]
       files: "nodejs/.*"

Cargo.toml

@@ -1,5 +1,11 @@
 [workspace]
-members = ["rust/ffi/node", "rust/lancedb", "nodejs", "python"]
+members = [
+    "rust/ffi/node",
+    "rust/lancedb",
+    "nodejs",
+    "python",
+    "java/core/lancedb-jni",
+]
 # Python package needs to be built by maturin.
 exclude = ["python"]
 resolver = "2"
@@ -14,10 +20,11 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.11.0", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.11.0" }
-lance-linalg = { "version" = "=0.11.0" }
-lance-testing = { "version" = "=0.11.0" }
+lance = { "version" = "=0.12.1", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.12.1" }
+lance-linalg = { "version" = "=0.12.1" }
+lance-testing = { "version" = "=0.12.1" }
+lance-datafusion = { "version" = "=0.12.1" }
 # Note that this one does not include pyarrow
 arrow = { version = "51.0", optional = false }
 arrow-array = "51.0"
@@ -29,6 +36,7 @@ arrow-arith = "51.0"
 arrow-cast = "51.0"
 async-trait = "0"
 chrono = "0.4.35"
+datafusion-physical-plan = "37.1"
 half = { "version" = "=2.4.1", default-features = false, features = [
     "num-traits",
 ] }

README.md

@@ -83,5 +83,5 @@ result = table.search([100, 100]).limit(2).to_pandas()
 ```
 
 ## Blogs, Tutorials & Videos
-* 📈 <a href="https://blog.eto.ai/benchmarking-random-access-in-lance-ed690757a826">2000x better performance with Lance over Parquet</a>
+* 📈 <a href="https://blog.lancedb.com/benchmarking-random-access-in-lance/">2000x better performance with Lance over Parquet</a>
 * 🤖 <a href="https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb">Build a question and answer bot with LanceDB</a>

java/core/lancedb-jni/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "lancedb-jni"
+description = "JNI bindings for LanceDB"
+# TODO modify lancedb/Cargo.toml for version and dependencies
+version = "0.4.18"
+edition.workspace = true
+repository.workspace = true
+readme.workspace = true
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+lancedb = { path = "../../../rust/lancedb" }
+lance = { workspace = true }
+arrow = { workspace = true, features = ["ffi"] }
+arrow-schema.workspace = true
+tokio = "1.23"
+jni = "0.21.1"
+snafu.workspace = true
+lazy_static.workspace = true
+serde = { version = "^1" }
+serde_json = { version = "1" }

java/core/lancedb-jni/src/connection.rs

@@ -0,0 +1,130 @@
+use crate::ffi::JNIEnvExt;
+use crate::traits::IntoJava;
+use crate::{Error, RT};
+use jni::objects::{JObject, JString, JValue};
+use jni::JNIEnv;
+pub const NATIVE_CONNECTION: &str = "nativeConnectionHandle";
+use crate::Result;
+use lancedb::connection::{connect, Connection};
+
+#[derive(Clone)]
+pub struct BlockingConnection {
+    pub(crate) inner: Connection,
+}
+
+impl BlockingConnection {
+    pub fn create(dataset_uri: &str) -> Result<Self> {
+        let inner = RT.block_on(connect(dataset_uri).execute())?;
+        Ok(Self { inner })
+    }
+
+    pub fn table_names(
+        &self,
+        start_after: Option<String>,
+        limit: Option<i32>,
+    ) -> Result<Vec<String>> {
+        let mut op = self.inner.table_names();
+        if let Some(start_after) = start_after {
+            op = op.start_after(start_after);
+        }
+        if let Some(limit) = limit {
+            op = op.limit(limit as u32);
+        }
+        Ok(RT.block_on(op.execute())?)
+    }
+}
+
+impl IntoJava for BlockingConnection {
+    fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> JObject<'a> {
+        attach_native_connection(env, self)
+    }
+}
+
+fn attach_native_connection<'local>(
+    env: &mut JNIEnv<'local>,
+    connection: BlockingConnection,
+) -> JObject<'local> {
+    let j_connection = create_java_connection_object(env);
+    // This block sets a native Rust object (Connection) as a field in the Java object (j_Connection).
+    // Caution: This creates a potential for memory leaks. The Rust object (Connection) is not
+    // automatically garbage-collected by Java, and its memory will not be freed unless
+    // explicitly handled.
+    //
+    // To prevent memory leaks, ensure the following:
+    // 1. The Java object (`j_Connection`) should implement the `java.io.Closeable` interface.
+    // 2. Users of this Java object should be instructed to always use it within a try-with-resources
+    //    statement (or manually call the `close()` method) to ensure that `self.close()` is invoked.
+    match unsafe { env.set_rust_field(&j_connection, NATIVE_CONNECTION, connection) } {
+        Ok(_) => j_connection,
+        Err(err) => {
+            env.throw_new(
+                "java/lang/RuntimeException",
+                format!("Failed to set native handle for Connection: {}", err),
+            )
+            .expect("Error throwing exception");
+            JObject::null()
+        }
+    }
+}
+
+fn create_java_connection_object<'a>(env: &mut JNIEnv<'a>) -> JObject<'a> {
+    env.new_object("com/lancedb/lancedb/Connection", "()V", &[])
+        .expect("Failed to create Java Lance Connection instance")
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_lancedb_lancedb_Connection_releaseNativeConnection(
+    mut env: JNIEnv,
+    j_connection: JObject,
+) {
+    let _: BlockingConnection = unsafe {
+        env.take_rust_field(j_connection, NATIVE_CONNECTION)
+            .expect("Failed to take native Connection handle")
+    };
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_lancedb_lancedb_Connection_connect<'local>(
+    mut env: JNIEnv<'local>,
+    _obj: JObject,
+    dataset_uri_object: JString,
+) -> JObject<'local> {
+    let dataset_uri: String = ok_or_throw!(env, env.get_string(&dataset_uri_object)).into();
+    let blocking_connection = ok_or_throw!(env, BlockingConnection::create(&dataset_uri));
+    blocking_connection.into_java(&mut env)
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_lancedb_lancedb_Connection_tableNames<'local>(
+    mut env: JNIEnv<'local>,
+    j_connection: JObject,
+    start_after_obj: JObject, // Optional<String>
+    limit_obj: JObject,       // Optional<Integer>
+) -> JObject<'local> {
+    ok_or_throw!(
+        env,
+        inner_table_names(&mut env, j_connection, start_after_obj, limit_obj)
+    )
+}
+
+fn inner_table_names<'local>(
+    env: &mut JNIEnv<'local>,
+    j_connection: JObject,
+    start_after_obj: JObject, // Optional<String>
+    limit_obj: JObject,       // Optional<Integer>
+) -> Result<JObject<'local>> {
+    let start_after = env.get_string_opt(&start_after_obj)?;
+    let limit = env.get_int_opt(&limit_obj)?;
+    let conn =
+        unsafe { env.get_rust_field::<_, _, BlockingConnection>(j_connection, NATIVE_CONNECTION) }?;
+    let table_names = conn.table_names(start_after, limit)?;
+    drop(conn);
+    let j_names = env.new_object("java/util/ArrayList", "()V", &[])?;
+    for item in table_names {
+        let jstr_item = env.new_string(item)?;
+        let item_jobj = JObject::from(jstr_item);
+        let item_gen = JValue::Object(&item_jobj);
+        env.call_method(&j_names, "add", "(Ljava/lang/Object;)Z", &[item_gen])?;
+    }
+    Ok(j_names)
+}

java/core/lancedb-jni/src/error.rs

@@ -0,0 +1,225 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::str::Utf8Error;
+
+use arrow_schema::ArrowError;
+use jni::errors::Error as JniError;
+use serde_json::Error as JsonError;
+use snafu::{Location, Snafu};
+
+type BoxedError = Box<dyn std::error::Error + Send + Sync + 'static>;
+
+/// Java Exception types
+pub enum JavaException {
+    IllegalArgumentException,
+    IOException,
+    RuntimeException,
+}
+
+impl JavaException {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::IllegalArgumentException => "java/lang/IllegalArgumentException",
+            Self::IOException => "java/io/IOException",
+            Self::RuntimeException => "java/lang/RuntimeException",
+        }
+    }
+}
+/// TODO(lu) change to lancedb-jni
+#[derive(Debug, Snafu)]
+#[snafu(visibility(pub))]
+pub enum Error {
+    #[snafu(display("JNI error: {message}, {location}"))]
+    Jni { message: String, location: Location },
+    #[snafu(display("Invalid argument: {message}, {location}"))]
+    InvalidArgument { message: String, location: Location },
+    #[snafu(display("IO error: {source}, {location}"))]
+    IO {
+        source: BoxedError,
+        location: Location,
+    },
+    #[snafu(display("Arrow error: {message}, {location}"))]
+    Arrow { message: String, location: Location },
+    #[snafu(display("Index error: {message}, {location}"))]
+    Index { message: String, location: Location },
+    #[snafu(display("JSON error: {message}, {location}"))]
+    JSON { message: String, location: Location },
+    #[snafu(display("Dataset at path {path} was not found, {location}"))]
+    DatasetNotFound { path: String, location: Location },
+    #[snafu(display("Dataset already exists: {uri}, {location}"))]
+    DatasetAlreadyExists { uri: String, location: Location },
+    #[snafu(display("Table '{name}' already exists"))]
+    TableAlreadyExists { name: String },
+    #[snafu(display("Table '{name}' was not found"))]
+    TableNotFound { name: String },
+    #[snafu(display("Invalid table name '{name}': {reason}"))]
+    InvalidTableName { name: String, reason: String },
+    #[snafu(display("Embedding function '{name}' was not found: {reason}, {location}"))]
+    EmbeddingFunctionNotFound {
+        name: String,
+        reason: String,
+        location: Location,
+    },
+    #[snafu(display("Other Lance error: {message}, {location}"))]
+    OtherLance { message: String, location: Location },
+    #[snafu(display("Other LanceDB error: {message}, {location}"))]
+    OtherLanceDB { message: String, location: Location },
+}
+
+impl Error {
+    /// Throw as Java Exception
+    pub fn throw(&self, env: &mut jni::JNIEnv) {
+        match self {
+            Self::InvalidArgument { .. }
+            | Self::DatasetNotFound { .. }
+            | Self::DatasetAlreadyExists { .. }
+            | Self::TableAlreadyExists { .. }
+            | Self::TableNotFound { .. }
+            | Self::InvalidTableName { .. }
+            | Self::EmbeddingFunctionNotFound { .. } => {
+                self.throw_as(env, JavaException::IllegalArgumentException)
+            }
+            Self::IO { .. } | Self::Index { .. } => self.throw_as(env, JavaException::IOException),
+            Self::Arrow { .. }
+            | Self::JSON { .. }
+            | Self::OtherLance { .. }
+            | Self::OtherLanceDB { .. }
+            | Self::Jni { .. } => self.throw_as(env, JavaException::RuntimeException),
+        }
+    }
+
+    /// Throw as an concrete Java Exception
+    pub fn throw_as(&self, env: &mut jni::JNIEnv, exception: JavaException) {
+        let message = &format!(
+            "Error when throwing Java exception: {}:{}",
+            exception.as_str(),
+            self
+        );
+        env.throw_new(exception.as_str(), self.to_string())
+            .expect(message);
+    }
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+trait ToSnafuLocation {
+    fn to_snafu_location(&'static self) -> snafu::Location;
+}
+
+impl ToSnafuLocation for std::panic::Location<'static> {
+    fn to_snafu_location(&'static self) -> snafu::Location {
+        snafu::Location::new(self.file(), self.line(), self.column())
+    }
+}
+
+impl From<JniError> for Error {
+    #[track_caller]
+    fn from(source: JniError) -> Self {
+        Self::Jni {
+            message: source.to_string(),
+            location: std::panic::Location::caller().to_snafu_location(),
+        }
+    }
+}
+
+impl From<Utf8Error> for Error {
+    #[track_caller]
+    fn from(source: Utf8Error) -> Self {
+        Self::InvalidArgument {
+            message: source.to_string(),
+            location: std::panic::Location::caller().to_snafu_location(),
+        }
+    }
+}
+
+impl From<ArrowError> for Error {
+    #[track_caller]
+    fn from(source: ArrowError) -> Self {
+        Self::Arrow {
+            message: source.to_string(),
+            location: std::panic::Location::caller().to_snafu_location(),
+        }
+    }
+}
+
+impl From<JsonError> for Error {
+    #[track_caller]
+    fn from(source: JsonError) -> Self {
+        Self::JSON {
+            message: source.to_string(),
+            location: std::panic::Location::caller().to_snafu_location(),
+        }
+    }
+}
+
+impl From<lance::Error> for Error {
+    #[track_caller]
+    fn from(source: lance::Error) -> Self {
+        match source {
+            lance::Error::DatasetNotFound {
+                path,
+                source: _,
+                location,
+            } => Self::DatasetNotFound { path, location },
+            lance::Error::DatasetAlreadyExists { uri, location } => {
+                Self::DatasetAlreadyExists { uri, location }
+            }
+            lance::Error::IO { source, location } => Self::IO { source, location },
+            lance::Error::Arrow { message, location } => Self::Arrow { message, location },
+            lance::Error::Index { message, location } => Self::Index { message, location },
+            lance::Error::InvalidInput { source, location } => Self::InvalidArgument {
+                message: source.to_string(),
+                location,
+            },
+            _ => Self::OtherLance {
+                message: source.to_string(),
+                location: std::panic::Location::caller().to_snafu_location(),
+            },
+        }
+    }
+}
+
+impl From<lancedb::Error> for Error {
+    #[track_caller]
+    fn from(source: lancedb::Error) -> Self {
+        match source {
+            lancedb::Error::InvalidTableName { name, reason } => {
+                Self::InvalidTableName { name, reason }
+            }
+            lancedb::Error::InvalidInput { message } => Self::InvalidArgument {
+                message,
+                location: std::panic::Location::caller().to_snafu_location(),
+            },
+            lancedb::Error::TableNotFound { name } => Self::TableNotFound { name },
+            lancedb::Error::TableAlreadyExists { name } => Self::TableAlreadyExists { name },
+            lancedb::Error::EmbeddingFunctionNotFound { name, reason } => {
+                Self::EmbeddingFunctionNotFound {
+                    name,
+                    reason,
+                    location: std::panic::Location::caller().to_snafu_location(),
+                }
+            }
+            lancedb::Error::Arrow { source } => Self::Arrow {
+                message: source.to_string(),
+                location: std::panic::Location::caller().to_snafu_location(),
+            },
+            lancedb::Error::Lance { source } => Self::from(source),
+            _ => Self::OtherLanceDB {
+                message: source.to_string(),
+                location: std::panic::Location::caller().to_snafu_location(),
+            },
+        }
+    }
+}

java/core/lancedb-jni/src/ffi.rs

@@ -0,0 +1,204 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use core::slice;
+
+use jni::objects::{JByteBuffer, JObjectArray, JString};
+use jni::sys::jobjectArray;
+use jni::{objects::JObject, JNIEnv};
+
+use crate::error::{Error, Result};
+
+/// TODO(lu) import from lance-jni without duplicate
+/// Extend JNIEnv with helper functions.
+pub trait JNIEnvExt {
+    /// Get integers from Java List<Integer> object.
+    fn get_integers(&mut self, obj: &JObject) -> Result<Vec<i32>>;
+
+    /// Get strings from Java List<String> object.
+    fn get_strings(&mut self, obj: &JObject) -> Result<Vec<String>>;
+
+    /// Get strings from Java String[] object.
+    /// Note that get Option<Vec<String>> from Java Optional<String[]> just doesn't work.
+    #[allow(unused)]
+    fn get_strings_array(&mut self, obj: jobjectArray) -> Result<Vec<String>>;
+
+    /// Get Option<String> from Java Optional<String>.
+    fn get_string_opt(&mut self, obj: &JObject) -> Result<Option<String>>;
+
+    /// Get Option<Vec<String>> from Java Optional<List<String>>.
+    #[allow(unused)]
+    fn get_strings_opt(&mut self, obj: &JObject) -> Result<Option<Vec<String>>>;
+
+    /// Get Option<i32> from Java Optional<Integer>.
+    fn get_int_opt(&mut self, obj: &JObject) -> Result<Option<i32>>;
+
+    /// Get Option<Vec<i32>> from Java Optional<List<Integer>>.
+    fn get_ints_opt(&mut self, obj: &JObject) -> Result<Option<Vec<i32>>>;
+
+    /// Get Option<i64> from Java Optional<Long>.
+    #[allow(unused)]
+    fn get_long_opt(&mut self, obj: &JObject) -> Result<Option<i64>>;
+
+    /// Get Option<u64> from Java Optional<Long>.
+    #[allow(unused)]
+    fn get_u64_opt(&mut self, obj: &JObject) -> Result<Option<u64>>;
+
+    /// Get Option<&[u8]> from Java Optional<ByteBuffer>.
+    #[allow(unused)]
+    fn get_bytes_opt(&mut self, obj: &JObject) -> Result<Option<&[u8]>>;
+
+    fn get_optional<T, F>(&mut self, obj: &JObject, f: F) -> Result<Option<T>>
+    where
+        F: FnOnce(&mut JNIEnv, &JObject) -> Result<T>;
+}
+
+impl JNIEnvExt for JNIEnv<'_> {
+    fn get_integers(&mut self, obj: &JObject) -> Result<Vec<i32>> {
+        let list = self.get_list(obj)?;
+        let mut iter = list.iter(self)?;
+        let mut results = Vec::with_capacity(list.size(self)? as usize);
+        while let Some(elem) = iter.next(self)? {
+            let int_obj = self.call_method(elem, "intValue", "()I", &[])?;
+            let int_value = int_obj.i()?;
+            results.push(int_value);
+        }
+        Ok(results)
+    }
+
+    fn get_strings(&mut self, obj: &JObject) -> Result<Vec<String>> {
+        let list = self.get_list(obj)?;
+        let mut iter = list.iter(self)?;
+        let mut results = Vec::with_capacity(list.size(self)? as usize);
+        while let Some(elem) = iter.next(self)? {
+            let jstr = JString::from(elem);
+            let val = self.get_string(&jstr)?;
+            results.push(val.to_str()?.to_string())
+        }
+        Ok(results)
+    }
+
+    fn get_strings_array(&mut self, obj: jobjectArray) -> Result<Vec<String>> {
+        let jobject_array = unsafe { JObjectArray::from_raw(obj) };
+        let array_len = self.get_array_length(&jobject_array)?;
+        let mut res: Vec<String> = Vec::new();
+        for i in 0..array_len {
+            let item: JString = self.get_object_array_element(&jobject_array, i)?.into();
+            res.push(self.get_string(&item)?.into());
+        }
+        Ok(res)
+    }
+
+    fn get_string_opt(&mut self, obj: &JObject) -> Result<Option<String>> {
+        self.get_optional(obj, |env, inner_obj| {
+            let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?;
+            let java_string_obj = java_obj_gen.l()?;
+            let jstr = JString::from(java_string_obj);
+            let val = env.get_string(&jstr)?;
+            Ok(val.to_str()?.to_string())
+        })
+    }
+
+    fn get_strings_opt(&mut self, obj: &JObject) -> Result<Option<Vec<String>>> {
+        self.get_optional(obj, |env, inner_obj| {
+            let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?;
+            let java_list_obj = java_obj_gen.l()?;
+            env.get_strings(&java_list_obj)
+        })
+    }
+
+    fn get_int_opt(&mut self, obj: &JObject) -> Result<Option<i32>> {
+        self.get_optional(obj, |env, inner_obj| {
+            let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?;
+            let java_int_obj = java_obj_gen.l()?;
+            let int_obj = env.call_method(java_int_obj, "intValue", "()I", &[])?;
+            let int_value = int_obj.i()?;
+            Ok(int_value)
+        })
+    }
+
+    fn get_ints_opt(&mut self, obj: &JObject) -> Result<Option<Vec<i32>>> {
+        self.get_optional(obj, |env, inner_obj| {
+            let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?;
+            let java_list_obj = java_obj_gen.l()?;
+            env.get_integers(&java_list_obj)
+        })
+    }
+
+    fn get_long_opt(&mut self, obj: &JObject) -> Result<Option<i64>> {
+        self.get_optional(obj, |env, inner_obj| {
+            let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?;
+            let java_long_obj = java_obj_gen.l()?;
+            let long_obj = env.call_method(java_long_obj, "longValue", "()J", &[])?;
+            let long_value = long_obj.j()?;
+            Ok(long_value)
+        })
+    }
+
+    fn get_u64_opt(&mut self, obj: &JObject) -> Result<Option<u64>> {
+        self.get_optional(obj, |env, inner_obj| {
+            let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?;
+            let java_long_obj = java_obj_gen.l()?;
+            let long_obj = env.call_method(java_long_obj, "longValue", "()J", &[])?;
+            let long_value = long_obj.j()?;
+            Ok(long_value as u64)
+        })
+    }
+
+    fn get_bytes_opt(&mut self, obj: &JObject) -> Result<Option<&[u8]>> {
+        self.get_optional(obj, |env, inner_obj| {
+            let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?;
+            let java_byte_buffer_obj = java_obj_gen.l()?;
+            let j_byte_buffer = JByteBuffer::from(java_byte_buffer_obj);
+            let raw_data = env.get_direct_buffer_address(&j_byte_buffer)?;
+            let capacity = env.get_direct_buffer_capacity(&j_byte_buffer)?;
+            let data = unsafe { slice::from_raw_parts(raw_data, capacity) };
+            Ok(data)
+        })
+    }
+
+    fn get_optional<T, F>(&mut self, obj: &JObject, f: F) -> Result<Option<T>>
+    where
+        F: FnOnce(&mut JNIEnv, &JObject) -> Result<T>,
+    {
+        if obj.is_null() {
+            return Ok(None);
+        }
+        let is_empty = self.call_method(obj, "isEmpty", "()Z", &[])?;
+        if is_empty.z()? {
+            // TODO(lu): put get java object into here cuz can only get java Object
+            Ok(None)
+        } else {
+            f(self, obj).map(Some)
+        }
+    }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_lancedb_lance_test_JniTestHelper_parseInts(
+    mut env: JNIEnv,
+    _obj: JObject,
+    list_obj: JObject, // List<Integer>
+) {
+    ok_or_throw_without_return!(env, env.get_integers(&list_obj));
+}
+
+#[no_mangle]
+pub extern "system" fn Java_com_lancedb_lance_test_JniTestHelper_parseIntsOpt(
+    mut env: JNIEnv,
+    _obj: JObject,
+    list_obj: JObject, // Optional<List<Integer>>
+) {
+    ok_or_throw_without_return!(env, env.get_ints_opt(&list_obj));
+}

java/core/lancedb-jni/src/lib.rs

@@ -0,0 +1,68 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use lazy_static::lazy_static;
+
+// TODO import from lance-jni without duplicate
+#[macro_export]
+macro_rules! ok_or_throw {
+    ($env:expr, $result:expr) => {
+        match $result {
+            Ok(value) => value,
+            Err(err) => {
+                Error::from(err).throw(&mut $env);
+                return JObject::null();
+            }
+        }
+    };
+}
+
+macro_rules! ok_or_throw_without_return {
+    ($env:expr, $result:expr) => {
+        match $result {
+            Ok(value) => value,
+            Err(err) => {
+                Error::from(err).throw(&mut $env);
+                return;
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! ok_or_throw_with_return {
+    ($env:expr, $result:expr, $ret:expr) => {
+        match $result {
+            Ok(value) => value,
+            Err(err) => {
+                Error::from(err).throw(&mut $env);
+                return $ret;
+            }
+        }
+    };
+}
+
+mod connection;
+pub mod error;
+mod ffi;
+mod traits;
+
+pub use error::{Error, Result};
+
+lazy_static! {
+    static ref RT: tokio::runtime::Runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .expect("Failed to create tokio runtime");
+}

java/core/lancedb-jni/src/traits.rs

@@ -0,0 +1,122 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use jni::objects::{JMap, JObject, JString, JValue};
+use jni::JNIEnv;
+
+use crate::Result;
+
+pub trait FromJObject<T> {
+    fn extract(&self) -> Result<T>;
+}
+
+/// Convert a Rust type into a Java Object.
+pub trait IntoJava {
+    fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> JObject<'a>;
+}
+
+impl FromJObject<i32> for JObject<'_> {
+    fn extract(&self) -> Result<i32> {
+        Ok(JValue::from(self).i()?)
+    }
+}
+
+impl FromJObject<i64> for JObject<'_> {
+    fn extract(&self) -> Result<i64> {
+        Ok(JValue::from(self).j()?)
+    }
+}
+
+impl FromJObject<f32> for JObject<'_> {
+    fn extract(&self) -> Result<f32> {
+        Ok(JValue::from(self).f()?)
+    }
+}
+
+impl FromJObject<f64> for JObject<'_> {
+    fn extract(&self) -> Result<f64> {
+        Ok(JValue::from(self).d()?)
+    }
+}
+
+pub trait FromJString {
+    fn extract(&self, env: &mut JNIEnv) -> Result<String>;
+}
+
+impl FromJString for JString<'_> {
+    fn extract(&self, env: &mut JNIEnv) -> Result<String> {
+        Ok(env.get_string(self)?.into())
+    }
+}
+
+pub trait JMapExt {
+    #[allow(dead_code)]
+    fn get_string(&self, env: &mut JNIEnv, key: &str) -> Result<Option<String>>;
+
+    #[allow(dead_code)]
+    fn get_i32(&self, env: &mut JNIEnv, key: &str) -> Result<Option<i32>>;
+
+    #[allow(dead_code)]
+    fn get_i64(&self, env: &mut JNIEnv, key: &str) -> Result<Option<i64>>;
+
+    #[allow(dead_code)]
+    fn get_f32(&self, env: &mut JNIEnv, key: &str) -> Result<Option<f32>>;
+
+    #[allow(dead_code)]
+    fn get_f64(&self, env: &mut JNIEnv, key: &str) -> Result<Option<f64>>;
+}
+
+fn get_map_value<T>(env: &mut JNIEnv, map: &JMap, key: &str) -> Result<Option<T>>
+where
+    for<'a> JObject<'a>: FromJObject<T>,
+{
+    let key_obj: JObject = env.new_string(key)?.into();
+    if let Some(value) = map.get(env, &key_obj)? {
+        if value.is_null() {
+            Ok(None)
+        } else {
+            Ok(Some(value.extract()?))
+        }
+    } else {
+        Ok(None)
+    }
+}
+
+impl JMapExt for JMap<'_, '_, '_> {
+    fn get_string(&self, env: &mut JNIEnv, key: &str) -> Result<Option<String>> {
+        let key_obj: JObject = env.new_string(key)?.into();
+        if let Some(value) = self.get(env, &key_obj)? {
+            let value_str: JString = value.into();
+            Ok(Some(value_str.extract(env)?))
+        } else {
+            Ok(None)
+        }
+    }
+
+    fn get_i32(&self, env: &mut JNIEnv, key: &str) -> Result<Option<i32>> {
+        get_map_value(env, self, key)
+    }
+
+    fn get_i64(&self, env: &mut JNIEnv, key: &str) -> Result<Option<i64>> {
+        get_map_value(env, self, key)
+    }
+
+    fn get_f32(&self, env: &mut JNIEnv, key: &str) -> Result<Option<f32>> {
+        get_map_value(env, self, key)
+    }
+
+    fn get_f64(&self, env: &mut JNIEnv, key: &str) -> Result<Option<f64>> {
+        get_map_value(env, self, key)
+    }
+}

java/core/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.lancedb</groupId>
+        <artifactId>lancedb-parent</artifactId>
+        <version>0.1-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>lancedb-core</artifactId>
+    <name>LanceDB Core</name>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-netty</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-c-data</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-dataset</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.json</groupId>
+            <artifactId>json</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.questdb</groupId>
+            <artifactId>jar-jni</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+             <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>build-jni</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.questdb</groupId>
+                        <artifactId>rust-maven-plugin</artifactId>
+                        <version>1.1.1</version>
+                        <executions>
+                            <execution>
+                                <id>lancedb-jni</id>
+                                <goals>
+                                    <goal>build</goal>
+                                </goals>
+                                <configuration>
+                                    <path>lancedb-jni</path>
+                                    <!--<release>true</release>-->
+                                    <!-- Copy native libraries to target/classes for runtime access -->
+                                    <copyTo>${project.build.directory}/classes/nativelib</copyTo>
+                                    <copyWithPlatformDir>true</copyWithPlatformDir>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>lancedb-jni-test</id>
+                                <goals>
+                                    <goal>test</goal>
+                                </goals>
+                                <configuration>
+                                    <path>lancedb-jni</path>
+                                    <release>false</release>
+                                    <verbosity>-v</verbosity>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>

java/core/src/main/java/com/lancedb/lancedb/Connection.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.lancedb.lancedb;
+
+import io.questdb.jar.jni.JarJniLoader;
+import java.io.Closeable;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Represents LanceDB database.
+ */
+public class Connection implements Closeable {
+  static {
+    JarJniLoader.loadLib(Connection.class, "/nativelib", "lancedb_jni");
+  }
+
+  private long nativeConnectionHandle;
+
+  /**
+   * Connect to a LanceDB instance.
+   */
+  public static native Connection connect(String uri);
+
+  /**
+   * Get the names of all tables in the database. The names are sorted in
+   * ascending order.
+   *
+   * @return the table names
+   */
+  public List<String> tableNames() {
+    return tableNames(Optional.empty(), Optional.empty());
+  }
+
+  /**
+   * Get the names of filtered tables in the database. The names are sorted in
+   * ascending order.
+   *
+   * @param limit The number of results to return.
+   * @return the table names
+   */
+  public List<String> tableNames(int limit) {
+    return tableNames(Optional.empty(), Optional.of(limit));
+  }
+
+  /**
+   * Get the names of filtered tables in the database. The names are sorted in
+   * ascending order.
+   *
+   * @param startAfter If present, only return names that come lexicographically after the supplied
+   *                   value. This can be combined with limit to implement pagination
+   *                   by setting this to the last table name from the previous page.
+   * @return the table names
+   */
+  public List<String> tableNames(String startAfter) {
+    return tableNames(Optional.of(startAfter), Optional.empty());
+  }
+
+  /**
+   * Get the names of filtered tables in the database. The names are sorted in
+   * ascending order.
+   *
+   * @param startAfter If present, only return names that come lexicographically after the supplied
+   *                   value. This can be combined with limit to implement pagination
+   *                   by setting this to the last table name from the previous page.
+   * @param limit The number of results to return.
+   * @return the table names
+   */
+  public List<String> tableNames(String startAfter, int limit) {
+    return tableNames(Optional.of(startAfter), Optional.of(limit));
+  }
+
+  /**
+   * Get the names of filtered tables in the database. The names are sorted in
+   * ascending order.
+   *
+   * @param startAfter If present, only return names that come lexicographically after the supplied
+   *                   value. This can be combined with limit to implement pagination
+   *                   by setting this to the last table name from the previous page.
+   * @param limit The number of results to return.
+   * @return the table names
+   */
+  public native List<String> tableNames(
+      Optional<String> startAfter, Optional<Integer> limit);
+
+  /**
+   * Closes this connection and releases any system resources associated with it. If
+   * the connection is
+   * already closed, then invoking this method has no effect.
+   */
+  @Override
+  public void close() {
+    if (nativeConnectionHandle != 0) {
+      releaseNativeConnection(nativeConnectionHandle);
+      nativeConnectionHandle = 0;
+    }
+  }
+
+  /**
+   * Native method to release the Lance connection resources associated with the
+   * given handle.
+   *
+   * @param handle The native handle to the connection resource.
+   */
+  private native void releaseNativeConnection(long handle);
+
+  private Connection() {}
+}

java/core/src/test/java/com/lancedb/lancedb/ConnectionTest.java

@@ -0,0 +1,135 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.lancedb.lancedb;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Path;
+import java.util.List;
+import java.net.URL;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class ConnectionTest {
+  private static final String[] TABLE_NAMES = {
+      "dataset_version",
+      "new_empty_dataset",
+      "test",
+      "write_stream"
+  };
+
+  @TempDir
+  static Path tempDir; // Temporary directory for the tests
+  private static URL lanceDbURL;
+
+  @BeforeAll
+  static void setUp() {
+    ClassLoader classLoader = ConnectionTest.class.getClassLoader();
+    lanceDbURL = classLoader.getResource("example_db");
+  }
+
+  @Test
+  void emptyDB() {
+    String databaseUri = tempDir.resolve("emptyDB").toString();
+    try (Connection conn = Connection.connect(databaseUri)) {
+      List<String> tableNames = conn.tableNames();
+      assertTrue(tableNames.isEmpty());
+    }
+  }
+
+  @Test
+  void tableNames() {
+    try (Connection conn = Connection.connect(lanceDbURL.toString())) {
+      List<String> tableNames = conn.tableNames();
+      assertEquals(4, tableNames.size());
+      for (int i = 0; i < TABLE_NAMES.length; i++) {
+        assertEquals(TABLE_NAMES[i], tableNames.get(i));
+      }
+    }
+  }
+
+  @Test
+  void tableNamesStartAfter() {
+    try (Connection conn = Connection.connect(lanceDbURL.toString())) {
+      assertTableNamesStartAfter(conn, TABLE_NAMES[0], 3, TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
+      assertTableNamesStartAfter(conn, TABLE_NAMES[1], 2, TABLE_NAMES[2], TABLE_NAMES[3]);
+      assertTableNamesStartAfter(conn, TABLE_NAMES[2], 1, TABLE_NAMES[3]);
+      assertTableNamesStartAfter(conn, TABLE_NAMES[3], 0);
+      assertTableNamesStartAfter(conn, "a_dataset", 4, TABLE_NAMES[0], TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]);
+      assertTableNamesStartAfter(conn, "o_dataset", 2, TABLE_NAMES[2], TABLE_NAMES[3]);
+      assertTableNamesStartAfter(conn, "v_dataset", 1, TABLE_NAMES[3]);
+      assertTableNamesStartAfter(conn, "z_dataset", 0);
+    }
+  }
+
+  private void assertTableNamesStartAfter(Connection conn, String startAfter, int expectedSize, String... expectedNames) {
+    List<String> tableNames = conn.tableNames(startAfter);
+    assertEquals(expectedSize, tableNames.size());
+    for (int i = 0; i < expectedNames.length; i++) {
+      assertEquals(expectedNames[i], tableNames.get(i));
+    }
+  }
+
+  @Test
+  void tableNamesLimit() {
+      try (Connection conn = Connection.connect(lanceDbURL.toString())) {
+      for (int i = 0; i <= TABLE_NAMES.length; i++) {
+        List<String> tableNames = conn.tableNames(i);
+        assertEquals(i, tableNames.size());
+        for (int j = 0; j < i; j++) {
+          assertEquals(TABLE_NAMES[j], tableNames.get(j));
+        }
+      }
+    }
+  }
+
+  @Test
+  void tableNamesStartAfterLimit() {
+    try (Connection conn = Connection.connect(lanceDbURL.toString())) {
+      List<String> tableNames = conn.tableNames(TABLE_NAMES[0], 2);
+      assertEquals(2, tableNames.size());
+      assertEquals(TABLE_NAMES[1], tableNames.get(0));
+      assertEquals(TABLE_NAMES[2], tableNames.get(1));
+      tableNames = conn.tableNames(TABLE_NAMES[1], 1);
+      assertEquals(1, tableNames.size());
+      assertEquals(TABLE_NAMES[2], tableNames.get(0));
+      tableNames = conn.tableNames(TABLE_NAMES[2], 2);
+      assertEquals(1, tableNames.size());
+      assertEquals(TABLE_NAMES[3], tableNames.get(0));
+      tableNames = conn.tableNames(TABLE_NAMES[3], 2);
+      assertEquals(0, tableNames.size());
+      tableNames = conn.tableNames(TABLE_NAMES[0], 0);
+      assertEquals(0, tableNames.size());
+
+      // Limit larger than the number of remaining tables
+      tableNames = conn.tableNames(TABLE_NAMES[0], 10);
+      assertEquals(3, tableNames.size());
+      assertEquals(TABLE_NAMES[1], tableNames.get(0));
+      assertEquals(TABLE_NAMES[2], tableNames.get(1));
+      assertEquals(TABLE_NAMES[3], tableNames.get(2));
+
+      // Start after a value not in the list
+      tableNames = conn.tableNames("non_existent_table", 2);
+      assertEquals(2, tableNames.size());
+      assertEquals(TABLE_NAMES[2], tableNames.get(0));
+      assertEquals(TABLE_NAMES[3], tableNames.get(1));
+
+      // Start after the last table with a limit
+      tableNames = conn.tableNames(TABLE_NAMES[3], 1);
+      assertEquals(0, tableNames.size());
+    }
+  }
+}

java/core/src/test/resources/example_db/dataset_version.lance/_transactions/0-d51afd07-e3cd-4c76-9b9b-787e13fd55b0.txn

@@ -0,0 +1 @@
+$d51afd07-e3cd-4c76-9b9b-787e13fd55b0<62>=id <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
*int320
8
name
 <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
*string0
8

java/core/src/test/resources/example_db/new_empty_dataset.lance/_transactions/0-15648e72-076f-4ef1-8b90-10d305b95b3b.txn

@@ -0,0 +1 @@
+$15648e72-076f-4ef1-8b90-10d305b95b3b<33>=id <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
*int320
8
name
 <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
*string0
8

java/core/src/test/resources/example_db/test.lance/_transactions/0-a3689caf-4f6b-4afc-a3c7-97af75661843.txn

@@ -0,0 +1 @@
+$a3689caf-4f6b-4afc-a3c7-97af75661843<34>oitem <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
*string8price
 <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
*double8
0vector <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
*fixed_size_list:float:28

java/pom.xml

@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.lancedb</groupId>
+    <artifactId>lancedb-parent</artifactId>
+    <version>0.1-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <name>Lance Parent</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <arrow.version>15.0.0</arrow.version>
+    </properties>
+
+    <modules>
+        <module>core</module>
+    </modules>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.apache.arrow</groupId>
+                <artifactId>arrow-vector</artifactId>
+                <version>${arrow.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.arrow</groupId>
+                <artifactId>arrow-memory-netty</artifactId>
+                <version>${arrow.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.arrow</groupId>
+                <artifactId>arrow-c-data</artifactId>
+                <version>${arrow.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.arrow</groupId>
+                <artifactId>arrow-dataset</artifactId>
+                <version>${arrow.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.questdb</groupId>
+                <artifactId>jar-jni</artifactId>
+                <version>1.1.1</version>
+            </dependency>
+            <dependency>
+                <groupId>org.junit.jupiter</groupId>
+                <artifactId>junit-jupiter</artifactId>
+                <version>5.10.1</version>
+            </dependency>
+            <dependency>
+                <groupId>org.json</groupId>
+                <artifactId>json</artifactId>
+                <version>20210307</version>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+     <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-checkstyle-plugin</artifactId>
+                <version>3.3.1</version>
+                <configuration>
+                    <configLocation>google_checks.xml</configLocation>
+                    <consoleOutput>true</consoleOutput>
+                    <failsOnError>true</failsOnError>
+                    <violationSeverity>warning</violationSeverity>
+                    <linkXRef>false</linkXRef>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>validate</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>check</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <artifactId>maven-clean-plugin</artifactId>
+                    <version>3.1.0</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-resources-plugin</artifactId>
+                    <version>3.0.2</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-compiler-plugin</artifactId>
+                    <version>3.8.1</version>
+                    <configuration>
+                        <compilerArgs>
+                            <arg>-h</arg>
+                            <arg>target/headers</arg>
+                        </compilerArgs>
+                    </configuration>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <version>3.2.5</version>
+                    <configuration>
+                        <argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
+                        <forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
+                        <useSystemClassLoader>false</useSystemClassLoader>
+                    </configuration>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-jar-plugin</artifactId>
+                    <version>3.0.2</version>
+                </plugin>
+                <plugin>
+                    <artifactId>maven-install-plugin</artifactId>
+                    <version>2.5.2</version>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+</project>

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.5.0",
+      "version": "0.5.1",
       "cpu": [
         "x64",
         "arm64"

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

node/src/index.ts

@@ -704,6 +704,9 @@ export interface VectorIndex {
 export interface IndexStats {
   numIndexedRows: number | null
   numUnindexedRows: number | null
+  index_type: string | null
+  distance_type: string | null
+  completed_at: string | null
 }
 
 /**

node/src/remote/index.ts

@@ -509,7 +509,8 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return (await results.body()).indexes?.map((index: any) => ({
       columns: index.columns,
       name: index.index_name,
-      uuid: index.index_uuid
+      uuid: index.index_uuid,
+      status: index.status
     }))
   }
 
@@ -520,7 +521,10 @@ export class RemoteTable<T = number[]> implements Table<T> {
     const body = await results.body()
     return {
       numIndexedRows: body?.num_indexed_rows,
-      numUnindexedRows: body?.num_unindexed_rows
+      numUnindexedRows: body?.num_unindexed_rows,
+      index_type: body?.index_type,
+      distance_type: body?.distance_type,
+      completed_at: body?.completed_at
     }
   }
 

nodejs/__test__/arrow.test.ts

@@ -31,6 +31,7 @@ import {
   Schema,
   Struct,
   type Table,
+  Type,
   Utf8,
   tableFromIPC,
 } from "apache-arrow";
@@ -51,7 +52,12 @@ import {
   makeArrowTable,
   makeEmptyTable,
 } from "../lancedb/arrow";
-import { type EmbeddingFunction } from "../lancedb/embedding/embedding_function";
+import {
+  EmbeddingFunction,
+  FieldOptions,
+  FunctionOptions,
+} from "../lancedb/embedding/embedding_function";
+import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry";
 
 // biome-ignore lint/suspicious/noExplicitAny: skip
 function sampleRecords(): Array<Record<string, any>> {
@@ -280,23 +286,46 @@ describe("The function makeArrowTable", function () {
   });
 });
 
-class DummyEmbedding implements EmbeddingFunction<string> {
-  public readonly sourceColumn = "string";
-  public readonly embeddingDimension = 2;
-  public readonly embeddingDataType = new Float16();
+class DummyEmbedding extends EmbeddingFunction<string> {
+  toJSON(): Partial<FunctionOptions> {
+    return {};
+  }
 
-  async embed(data: string[]): Promise<number[][]> {
+  async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
     return data.map(() => [0.0, 0.0]);
   }
+
+  ndims(): number {
+    return 2;
+  }
+
+  embeddingDataType() {
+    return new Float16();
+  }
 }
 
-class DummyEmbeddingWithNoDimension implements EmbeddingFunction<string> {
-  public readonly sourceColumn = "string";
+class DummyEmbeddingWithNoDimension extends EmbeddingFunction<string> {
+  toJSON(): Partial<FunctionOptions> {
+    return {};
+  }
 
-  async embed(data: string[]): Promise<number[][]> {
+  embeddingDataType(): Float {
+    return new Float16();
+  }
+
+  async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
     return data.map(() => [0.0, 0.0]);
   }
 }
+const dummyEmbeddingConfig: EmbeddingFunctionConfig = {
+  sourceColumn: "string",
+  function: new DummyEmbedding(),
+};
+
+const dummyEmbeddingConfigWithNoDimension: EmbeddingFunctionConfig = {
+  sourceColumn: "string",
+  function: new DummyEmbeddingWithNoDimension(),
+};
 
 describe("convertToTable", function () {
   it("will infer data types correctly", async function () {
@@ -331,7 +360,7 @@ describe("convertToTable", function () {
 
   it("will apply embeddings", async function () {
     const records = sampleRecords();
-    const table = await convertToTable(records, new DummyEmbedding());
+    const table = await convertToTable(records, dummyEmbeddingConfig);
     expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
     expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
       new Float16().toString(),
@@ -340,7 +369,7 @@ describe("convertToTable", function () {
 
   it("will fail if missing the embedding source column", async function () {
     await expect(
-      convertToTable([{ id: 1 }], new DummyEmbedding()),
+      convertToTable([{ id: 1 }], dummyEmbeddingConfig),
     ).rejects.toThrow("'string' was not present");
   });
 
@@ -351,7 +380,7 @@ describe("convertToTable", function () {
     const table = makeEmptyTable(schema);
 
     // If the embedding specifies the dimension we are fine
-    await fromTableToBuffer(table, new DummyEmbedding());
+    await fromTableToBuffer(table, dummyEmbeddingConfig);
 
     // We can also supply a schema and should be ok
     const schemaWithEmbedding = new Schema([
@@ -364,13 +393,13 @@ describe("convertToTable", function () {
     ]);
     await fromTableToBuffer(
       table,
-      new DummyEmbeddingWithNoDimension(),
+      dummyEmbeddingConfigWithNoDimension,
       schemaWithEmbedding,
     );
 
     // Otherwise we will get an error
     await expect(
-      fromTableToBuffer(table, new DummyEmbeddingWithNoDimension()),
+      fromTableToBuffer(table, dummyEmbeddingConfigWithNoDimension),
     ).rejects.toThrow("does not specify `embeddingDimension`");
   });
 
@@ -383,7 +412,7 @@ describe("convertToTable", function () {
         false,
       ),
     ]);
-    const table = await convertToTable([], new DummyEmbedding(), { schema });
+    const table = await convertToTable([], dummyEmbeddingConfig, { schema });
     expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
     expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
       new Float16().toString(),
@@ -393,16 +422,17 @@ describe("convertToTable", function () {
   it("will complain if embeddings present but schema missing embedding column", async function () {
     const schema = new Schema([new Field("string", new Utf8(), false)]);
     await expect(
-      convertToTable([], new DummyEmbedding(), { schema }),
+      convertToTable([], dummyEmbeddingConfig, { schema }),
     ).rejects.toThrow("column vector was missing");
   });
 
   it("will provide a nice error if run twice", async function () {
     const records = sampleRecords();
-    const table = await convertToTable(records, new DummyEmbedding());
+    const table = await convertToTable(records, dummyEmbeddingConfig);
+
     // fromTableToBuffer will try and apply the embeddings again
     await expect(
-      fromTableToBuffer(table, new DummyEmbedding()),
+      fromTableToBuffer(table, dummyEmbeddingConfig),
     ).rejects.toThrow("already existed");
   });
 });

nodejs/__test__/connection.test.ts

@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import { Field, Float64, Schema } from "apache-arrow";
 import * as tmp from "tmp";
-
-import { Connection, connect } from "../lancedb";
+import { Connection, Table, connect } from "../lancedb";
 
 describe("when connecting", () => {
   let tmpDir: tmp.DirResult;
@@ -87,4 +87,39 @@ describe("given a connection", () => {
     tables = await db.tableNames({ startAfter: "a" });
     expect(tables).toEqual(["b", "c"]);
   });
+
+  it("should create tables in v2 mode", async () => {
+    const db = await connect(tmpDir.name);
+    const data = [...Array(10000).keys()].map((i) => ({ id: i }));
+
+    // Create in v1 mode
+    let table = await db.createTable("test", data);
+
+    const isV2 = async (table: Table) => {
+      const data = await table.query().toArrow({ maxBatchLength: 100000 });
+      console.log(data.batches.length);
+      return data.batches.length < 5;
+    };
+
+    await expect(isV2(table)).resolves.toBe(false);
+
+    // Create in v2 mode
+    table = await db.createTable("test_v2", data, { useLegacyFormat: false });
+
+    await expect(isV2(table)).resolves.toBe(true);
+
+    await table.add(data);
+
+    await expect(isV2(table)).resolves.toBe(true);
+
+    // Create empty in v2 mode
+    const schema = new Schema([new Field("id", new Float64(), true)]);
+
+    table = await db.createEmptyTable("test_v2_empty", schema, {
+      useLegacyFormat: false,
+    });
+
+    await table.add(data);
+    await expect(isV2(table)).resolves.toBe(true);
+  });
 });

nodejs/__test__/embedding.test.ts

@@ -0,0 +1,314 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as tmp from "tmp";
+
+import { connect } from "../lancedb";
+import {
+  Field,
+  FixedSizeList,
+  Float,
+  Float16,
+  Float32,
+  Float64,
+  Schema,
+  Utf8,
+} from "../lancedb/arrow";
+import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
+import { getRegistry, register } from "../lancedb/embedding/registry";
+
+describe("embedding functions", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => {
+    tmpDir.removeCallback();
+    getRegistry().reset();
+  });
+
+  it("should be able to create a table with an embedding function", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {};
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+    const func = new MockEmbeddingFunction();
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable(
+      "test",
+      [
+        { id: 1, text: "hello" },
+        { id: 2, text: "world" },
+      ],
+      {
+        embeddingFunction: {
+          function: func,
+          sourceColumn: "text",
+        },
+      },
+    );
+    // biome-ignore lint/suspicious/noExplicitAny: test
+    const arr = (await table.query().toArray()) as any;
+    expect(arr[0].vector).toBeDefined();
+
+    // we round trip through JSON to make sure the vector properly gets converted to an array
+    // otherwise it'll be a TypedArray or Vector
+    const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
+    expect(vector0).toEqual([1, 2, 3]);
+  });
+
+  it("should be able to create an empty table with an embedding function", async () => {
+    @register()
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {};
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+    const schema = new Schema([
+      new Field("text", new Utf8(), true),
+      new Field(
+        "vector",
+        new FixedSizeList(3, new Field("item", new Float32(), true)),
+        true,
+      ),
+    ]);
+
+    const func = new MockEmbeddingFunction();
+    const db = await connect(tmpDir.name);
+    const table = await db.createEmptyTable("test", schema, {
+      embeddingFunction: {
+        function: func,
+        sourceColumn: "text",
+      },
+    });
+    const outSchema = await table.schema();
+    expect(outSchema.metadata.get("embedding_functions")).toBeDefined();
+    await table.add([{ text: "hello world" }]);
+
+    // biome-ignore lint/suspicious/noExplicitAny: test
+    const arr = (await table.query().toArray()) as any;
+    expect(arr[0].vector).toBeDefined();
+
+    // we round trip through JSON to make sure the vector properly gets converted to an array
+    // otherwise it'll be a TypedArray or Vector
+    const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
+    expect(vector0).toEqual([1, 2, 3]);
+  });
+  it("should error when appending to a table with an unregistered embedding function", async () => {
+    @register("mock")
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {};
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+    const func = getRegistry().get<MockEmbeddingFunction>("mock")!.create();
+
+    const schema = LanceSchema({
+      id: new Float64(),
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+
+    const db = await connect(tmpDir.name);
+    await db.createTable(
+      "test",
+      [
+        { id: 1, text: "hello" },
+        { id: 2, text: "world" },
+      ],
+      {
+        schema,
+      },
+    );
+
+    getRegistry().reset();
+    const db2 = await connect(tmpDir.name);
+
+    const tbl = await db2.openTable("test");
+
+    expect(tbl.add([{ id: 3, text: "hello" }])).rejects.toThrow(
+      `Function "mock" not found in registry`,
+    );
+  });
+  test.each([new Float16(), new Float32(), new Float64()])(
+    "should be able to provide manual embeddings with multiple float datatype",
+    async (floatType) => {
+      class MockEmbeddingFunction extends EmbeddingFunction<string> {
+        toJSON(): object {
+          return {};
+        }
+        ndims() {
+          return 3;
+        }
+        embeddingDataType(): Float {
+          return floatType;
+        }
+        async computeQueryEmbeddings(_data: string) {
+          return [1, 2, 3];
+        }
+        async computeSourceEmbeddings(data: string[]) {
+          return Array.from({ length: data.length }).fill([
+            1, 2, 3,
+          ]) as number[][];
+        }
+      }
+      const data = [{ text: "hello" }, { text: "hello world" }];
+
+      const schema = new Schema([
+        new Field("vector", new FixedSizeList(3, new Field("item", floatType))),
+        new Field("text", new Utf8()),
+      ]);
+      const func = new MockEmbeddingFunction();
+
+      const name = "test";
+      const db = await connect(tmpDir.name);
+
+      const table = await db.createTable(name, data, {
+        schema,
+        embeddingFunction: {
+          sourceColumn: "text",
+          function: func,
+        },
+      });
+      const res = await table.query().toArray();
+
+      expect([...res[0].vector]).toEqual([1, 2, 3]);
+    },
+  );
+
+  test.only.each([new Float16(), new Float32(), new Float64()])(
+    "should be able to provide auto embeddings with multiple float datatypes",
+    async (floatType) => {
+      @register("test1")
+      class MockEmbeddingFunctionWithoutNDims extends EmbeddingFunction<string> {
+        toJSON(): object {
+          return {};
+        }
+
+        embeddingDataType(): Float {
+          return floatType;
+        }
+        async computeQueryEmbeddings(_data: string) {
+          return [1, 2, 3];
+        }
+        async computeSourceEmbeddings(data: string[]) {
+          return Array.from({ length: data.length }).fill([
+            1, 2, 3,
+          ]) as number[][];
+        }
+      }
+      @register("test")
+      class MockEmbeddingFunction extends EmbeddingFunction<string> {
+        toJSON(): object {
+          return {};
+        }
+        ndims() {
+          return 3;
+        }
+        embeddingDataType(): Float {
+          return floatType;
+        }
+        async computeQueryEmbeddings(_data: string) {
+          return [1, 2, 3];
+        }
+        async computeSourceEmbeddings(data: string[]) {
+          return Array.from({ length: data.length }).fill([
+            1, 2, 3,
+          ]) as number[][];
+        }
+      }
+      const func = getRegistry().get<MockEmbeddingFunction>("test")!.create();
+      const func2 = getRegistry()
+        .get<MockEmbeddingFunctionWithoutNDims>("test1")!
+        .create();
+
+      const schema = LanceSchema({
+        text: func.sourceField(new Utf8()),
+        vector: func.vectorField(floatType),
+      });
+
+      const schema2 = LanceSchema({
+        text: func2.sourceField(new Utf8()),
+        vector: func2.vectorField({ datatype: floatType, dims: 3 }),
+      });
+      const schema3 = LanceSchema({
+        text: func2.sourceField(new Utf8()),
+        vector: func.vectorField({
+          datatype: new FixedSizeList(3, new Field("item", floatType, true)),
+          dims: 3,
+        }),
+      });
+
+      const expectedSchema = new Schema([
+        new Field("text", new Utf8(), true),
+        new Field(
+          "vector",
+          new FixedSizeList(3, new Field("item", floatType, true)),
+          true,
+        ),
+      ]);
+      const stringSchema = JSON.stringify(schema, null, 2);
+      const stringSchema2 = JSON.stringify(schema2, null, 2);
+      const stringSchema3 = JSON.stringify(schema3, null, 2);
+      const stringExpectedSchema = JSON.stringify(expectedSchema, null, 2);
+
+      expect(stringSchema).toEqual(stringExpectedSchema);
+      expect(stringSchema2).toEqual(stringExpectedSchema);
+      expect(stringSchema3).toEqual(stringExpectedSchema);
+    },
+  );
+});

nodejs/__test__/registry.test.ts

@@ -0,0 +1,169 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import * as arrow from "apache-arrow";
+import * as arrowOld from "apache-arrow-old";
+
+import * as tmp from "tmp";
+
+import { connect } from "../lancedb";
+import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
+import { getRegistry, register } from "../lancedb/embedding/registry";
+
+describe.each([arrow, arrowOld])("LanceSchema", (arrow) => {
+  test("should preserve input order", async () => {
+    const schema = LanceSchema({
+      id: new arrow.Int32(),
+      text: new arrow.Utf8(),
+      vector: new arrow.Float32(),
+    });
+    expect(schema.fields.map((x) => x.name)).toEqual(["id", "text", "vector"]);
+  });
+});
+
+describe("Registry", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+
+  afterEach(() => {
+    tmpDir.removeCallback();
+    getRegistry().reset();
+  });
+
+  it("should register a new item to the registry", async () => {
+    @register("mock-embedding")
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {
+          someText: "hello",
+        };
+      }
+      constructor() {
+        super();
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): arrow.Float {
+        return new arrow.Float32();
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+    const func = getRegistry()
+      .get<MockEmbeddingFunction>("mock-embedding")!
+      .create();
+
+    const schema = LanceSchema({
+      id: new arrow.Int32(),
+      text: func.sourceField(new arrow.Utf8()),
+      vector: func.vectorField(),
+    });
+
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable(
+      "test",
+      [
+        { id: 1, text: "hello" },
+        { id: 2, text: "world" },
+      ],
+      { schema },
+    );
+    const expected = [
+      [1, 2, 3],
+      [1, 2, 3],
+    ];
+    const actual = await table.query().toArrow();
+    const vectors = actual
+      .getChild("vector")
+      ?.toArray()
+      .map((x: unknown) => {
+        if (x instanceof arrow.Vector) {
+          return [...x];
+        } else {
+          return x;
+        }
+      });
+    expect(vectors).toEqual(expected);
+  });
+  test("should error if registering with the same name", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {
+          someText: "hello",
+        };
+      }
+      constructor() {
+        super();
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): arrow.Float {
+        return new arrow.Float32();
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+    register("mock-embedding")(MockEmbeddingFunction);
+    expect(() => register("mock-embedding")(MockEmbeddingFunction)).toThrow(
+      'Embedding function with alias "mock-embedding" already exists',
+    );
+  });
+  test("schema should contain correct metadata", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {
+          someText: "hello",
+        };
+      }
+      constructor() {
+        super();
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): arrow.Float {
+        return new arrow.Float32();
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+    const func = new MockEmbeddingFunction();
+
+    const schema = LanceSchema({
+      id: new arrow.Int32(),
+      text: func.sourceField(new arrow.Utf8()),
+      vector: func.vectorField(),
+    });
+    const expectedMetadata = new Map<string, string>([
+      [
+        "embedding_functions",
+        JSON.stringify([
+          {
+            sourceColumn: "text",
+            vectorColumn: "vector",
+            name: "MockEmbeddingFunction",
+            model: { someText: "hello" },
+          },
+        ]),
+      ],
+    ]);
+    expect(schema.metadata).toEqual(expectedMetadata);
+  });
+});

nodejs/__test__/table.test.ts

@@ -16,7 +16,12 @@ import * as fs from "fs";
 import * as path from "path";
 import * as tmp from "tmp";
 
+import * as arrow from "apache-arrow";
+import * as arrowOld from "apache-arrow-old";
+
+import { Table, connect } from "../lancedb";
 import {
+  Table as ArrowTable,
   Field,
   FixedSizeList,
   Float32,
@@ -24,15 +29,20 @@ import {
   Int32,
   Int64,
   Schema,
-} from "apache-arrow";
-import { Table, connect } from "../lancedb";
-import { makeArrowTable } from "../lancedb/arrow";
+  makeArrowTable,
+} from "../lancedb/arrow";
+import { EmbeddingFunction, LanceSchema, register } from "../lancedb/embedding";
 import { Index } from "../lancedb/indices";
 
-describe("Given a table", () => {
+// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+describe.each([arrow, arrowOld])("Given a table", (arrow: any) => {
   let tmpDir: tmp.DirResult;
   let table: Table;
-  const schema = new Schema([new Field("id", new Float64(), true)]);
+
+  const schema = new arrow.Schema([
+    new arrow.Field("id", new arrow.Float64(), true),
+  ]);
+
   beforeEach(async () => {
     tmpDir = tmp.dirSync({ unsafeCleanup: true });
     const conn = await connect(tmpDir.name);
@@ -83,6 +93,43 @@ describe("Given a table", () => {
     expect(await table.countRows("id == 7")).toBe(1);
     expect(await table.countRows("id == 10")).toBe(1);
   });
+
+  // https://github.com/lancedb/lancedb/issues/1293
+  test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
+    "can create empty table with non default float type: %s",
+    async (floatType) => {
+      const db = await connect(tmpDir.name);
+
+      const data = [
+        { text: "hello", vector: Array(512).fill(1.0) },
+        { text: "hello world", vector: Array(512).fill(1.0) },
+      ];
+      const f64Schema = new arrow.Schema([
+        new arrow.Field("text", new arrow.Utf8(), true),
+        new arrow.Field(
+          "vector",
+          new arrow.FixedSizeList(512, new arrow.Field("item", floatType)),
+          true,
+        ),
+      ]);
+
+      const f64Table = await db.createEmptyTable("f64", f64Schema, {
+        mode: "overwrite",
+      });
+      try {
+        await f64Table.add(data);
+        const res = await f64Table.query().toArray();
+        expect(res.length).toBe(2);
+      } catch (e) {
+        expect(e).toBeUndefined();
+      }
+    },
+  );
+
+  it("should return the table as an instance of an arrow table", async () => {
+    const arrowTbl = await table.toArrow();
+    expect(arrowTbl).toBeInstanceOf(ArrowTable);
+  });
 });
 
 describe("When creating an index", () => {
@@ -447,3 +494,99 @@ describe("when optimizing a dataset", () => {
     expect(stats.prune.oldVersionsRemoved).toBe(3);
   });
 });
+
+describe("table.search", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  test("can search using a string", async () => {
+    @register()
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {};
+      }
+      ndims() {
+        return 1;
+      }
+      embeddingDataType(): arrow.Float {
+        return new Float32();
+      }
+
+      // Hardcoded embeddings for the sake of testing
+      async computeQueryEmbeddings(_data: string) {
+        switch (_data) {
+          case "greetings":
+            return [0.1];
+          case "farewell":
+            return [0.2];
+          default:
+            return null as never;
+        }
+      }
+
+      // Hardcoded embeddings for the sake of testing
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map((s) => {
+          switch (s) {
+            case "hello world":
+              return [0.1];
+            case "goodbye world":
+              return [0.2];
+            default:
+              return null as never;
+          }
+        });
+      }
+    }
+
+    const func = new MockEmbeddingFunction();
+    const schema = LanceSchema({
+      text: func.sourceField(new arrow.Utf8()),
+      vector: func.vectorField(),
+    });
+    const db = await connect(tmpDir.name);
+    const data = [{ text: "hello world" }, { text: "goodbye world" }];
+    const table = await db.createTable("test", data, { schema });
+
+    const results = await table.search("greetings").then((r) => r.toArray());
+    expect(results[0].text).toBe(data[0].text);
+
+    const results2 = await table.search("farewell").then((r) => r.toArray());
+    expect(results2[0].text).toBe(data[1].text);
+  });
+
+  test("rejects if no embedding function provided", async () => {
+    const db = await connect(tmpDir.name);
+    const data = [
+      { text: "hello world", vector: [0.1, 0.2, 0.3] },
+      { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
+    ];
+    const table = await db.createTable("test", data);
+
+    expect(table.search("hello")).rejects.toThrow(
+      "No embedding functions are defined in the table",
+    );
+  });
+
+  test.each([
+    [0.4, 0.5, 0.599], // number[]
+    Float32Array.of(0.4, 0.5, 0.599), // Float32Array
+    Float64Array.of(0.4, 0.5, 0.599), // Float64Array
+  ])("can search using vectorlike datatypes", async (vectorlike) => {
+    const db = await connect(tmpDir.name);
+    const data = [
+      { text: "hello world", vector: [0.1, 0.2, 0.3] },
+      { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
+    ];
+    const table = await db.createTable("test", data);
+
+    // biome-ignore lint/suspicious/noExplicitAny: test
+    const results: any[] = await table.search(vectorlike).toArray();
+
+    expect(results.length).toBe(2);
+    expect(results[0].text).toBe(data[1].text);
+  });
+});

nodejs/biome.json

@@ -48,7 +48,7 @@
         "noUnsafeFinally": "error",
         "noUnsafeOptionalChaining": "error",
         "noUnusedLabels": "error",
-        "noUnusedVariables": "error",
+        "noUnusedVariables": "warn",
         "useIsNan": "error",
         "useValidForDirection": "error",
         "useYield": "error"
@@ -101,7 +101,13 @@
   },
   "overrides": [
     {
-      "include": ["**/*.ts", "**/*.tsx", "**/*.mts", "**/*.cts"],
+      "include": [
+        "**/*.ts",
+        "**/*.tsx",
+        "**/*.mts",
+        "**/*.cts",
+        "__test__/*.test.ts"
+      ],
       "linter": {
         "rules": {
           "correctness": {

nodejs/lancedb/arrow.ts

@@ -17,24 +17,122 @@ import {
   Binary,
   DataType,
   Field,
+  FixedSizeBinary,
   FixedSizeList,
-  type Float,
+  Float,
   Float32,
+  Int,
+  LargeBinary,
   List,
+  Null,
   RecordBatch,
   RecordBatchFileWriter,
   RecordBatchStreamWriter,
   Schema,
   Struct,
   Utf8,
-  type Vector,
+  Vector,
   makeBuilder,
   makeData,
   type makeTable,
   vectorFromArray,
 } from "apache-arrow";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
-import { sanitizeSchema } from "./sanitize";
+import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
+import { sanitizeField, sanitizeSchema, sanitizeType } from "./sanitize";
+export * from "apache-arrow";
+
+export type IntoVector = Float32Array | Float64Array | number[];
+
+export function isArrowTable(value: object): value is ArrowTable {
+  if (value instanceof ArrowTable) return true;
+  return "schema" in value && "batches" in value;
+}
+
+export function isDataType(value: unknown): value is DataType {
+  return (
+    value instanceof DataType ||
+    DataType.isNull(value) ||
+    DataType.isInt(value) ||
+    DataType.isFloat(value) ||
+    DataType.isBinary(value) ||
+    DataType.isLargeBinary(value) ||
+    DataType.isUtf8(value) ||
+    DataType.isLargeUtf8(value) ||
+    DataType.isBool(value) ||
+    DataType.isDecimal(value) ||
+    DataType.isDate(value) ||
+    DataType.isTime(value) ||
+    DataType.isTimestamp(value) ||
+    DataType.isInterval(value) ||
+    DataType.isDuration(value) ||
+    DataType.isList(value) ||
+    DataType.isStruct(value) ||
+    DataType.isUnion(value) ||
+    DataType.isFixedSizeBinary(value) ||
+    DataType.isFixedSizeList(value) ||
+    DataType.isMap(value) ||
+    DataType.isDictionary(value)
+  );
+}
+export function isNull(value: unknown): value is Null {
+  return value instanceof Null || DataType.isNull(value);
+}
+export function isInt(value: unknown): value is Int {
+  return value instanceof Int || DataType.isInt(value);
+}
+export function isFloat(value: unknown): value is Float {
+  return value instanceof Float || DataType.isFloat(value);
+}
+export function isBinary(value: unknown): value is Binary {
+  return value instanceof Binary || DataType.isBinary(value);
+}
+export function isLargeBinary(value: unknown): value is LargeBinary {
+  return value instanceof LargeBinary || DataType.isLargeBinary(value);
+}
+export function isUtf8(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isUtf8(value);
+}
+export function isLargeUtf8(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isLargeUtf8(value);
+}
+export function isBool(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isBool(value);
+}
+export function isDecimal(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isDecimal(value);
+}
+export function isDate(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isDate(value);
+}
+export function isTime(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isTime(value);
+}
+export function isTimestamp(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isTimestamp(value);
+}
+export function isInterval(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isInterval(value);
+}
+export function isDuration(value: unknown): value is Utf8 {
+  return value instanceof Utf8 || DataType.isDuration(value);
+}
+export function isList(value: unknown): value is List {
+  return value instanceof List || DataType.isList(value);
+}
+export function isStruct(value: unknown): value is Struct {
+  return value instanceof Struct || DataType.isStruct(value);
+}
+export function isUnion(value: unknown): value is Struct {
+  return value instanceof Struct || DataType.isUnion(value);
+}
+export function isFixedSizeBinary(value: unknown): value is FixedSizeBinary {
+  return value instanceof FixedSizeBinary || DataType.isFixedSizeBinary(value);
+}
+
+export function isFixedSizeList(value: unknown): value is FixedSizeList {
+  return value instanceof FixedSizeList || DataType.isFixedSizeList(value);
+}
 
 /** Data type accepted by NodeJS SDK */
 export type Data = Record<string, unknown>[] | ArrowTable;
@@ -86,6 +184,7 @@ export class MakeArrowTableOptions {
     vector: new VectorColumnOptions(),
   };
   embeddings?: EmbeddingFunction<unknown>;
+  embeddingFunction?: EmbeddingFunctionConfig;
 
   /**
    * If true then string columns will be encoded with dictionary encoding
@@ -198,6 +297,7 @@ export class MakeArrowTableOptions {
 export function makeArrowTable(
   data: Array<Record<string, unknown>>,
   options?: Partial<MakeArrowTableOptions>,
+  metadata?: Map<string, string>,
 ): ArrowTable {
   if (
     data.length === 0 &&
@@ -209,7 +309,11 @@ export function makeArrowTable(
   const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
   if (opt.schema !== undefined && opt.schema !== null) {
     opt.schema = sanitizeSchema(opt.schema);
-    opt.schema = validateSchemaEmbeddings(opt.schema, data, opt.embeddings);
+    opt.schema = validateSchemaEmbeddings(
+      opt.schema,
+      data,
+      options?.embeddingFunction,
+    );
   }
   const columns: Record<string, Vector> = {};
   // TODO: sample dataset to find missing columns
@@ -290,20 +394,41 @@ export function makeArrowTable(
     // `new ArrowTable(schema, batches)` which does not do any schema inference
     const firstTable = new ArrowTable(columns);
     const batchesFixed = firstTable.batches.map(
-      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
       (batch) => new RecordBatch(opt.schema!, batch.data),
     );
-    return new ArrowTable(opt.schema, batchesFixed);
-  } else {
-    return new ArrowTable(columns);
+    let schema: Schema;
+    if (metadata !== undefined) {
+      let schemaMetadata = opt.schema.metadata;
+      if (schemaMetadata.size === 0) {
+        schemaMetadata = metadata;
+      } else {
+        for (const [key, entry] of schemaMetadata.entries()) {
+          schemaMetadata.set(key, entry);
+        }
+      }
+
+      schema = new Schema(opt.schema.fields, schemaMetadata);
+    } else {
+      schema = opt.schema;
+    }
+    return new ArrowTable(schema, batchesFixed);
   }
+  const tbl = new ArrowTable(columns);
+  if (metadata !== undefined) {
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    (<any>tbl.schema).metadata = metadata;
+  }
+  return tbl;
 }
 
 /**
  * Create an empty Arrow table with the provided schema
  */
-export function makeEmptyTable(schema: Schema): ArrowTable {
-  return makeArrowTable([], { schema });
+export function makeEmptyTable(
+  schema: Schema,
+  metadata?: Map<string, string>,
+): ArrowTable {
+  return makeArrowTable([], { schema }, metadata);
 }
 
 /**
@@ -375,13 +500,74 @@ function makeVector(
   }
 }
 
+/** Helper function to apply embeddings from metadata to an input table */
+async function applyEmbeddingsFromMetadata(
+  table: ArrowTable,
+  schema: Schema,
+): Promise<ArrowTable> {
+  const registry = getRegistry();
+  const functions = registry.parseFunctions(schema.metadata);
+
+  const columns = Object.fromEntries(
+    table.schema.fields.map((field) => [
+      field.name,
+      table.getChild(field.name)!,
+    ]),
+  );
+
+  for (const functionEntry of functions.values()) {
+    const sourceColumn = columns[functionEntry.sourceColumn];
+    const destColumn = functionEntry.vectorColumn ?? "vector";
+    if (sourceColumn === undefined) {
+      throw new Error(
+        `Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`,
+      );
+    }
+    if (columns[destColumn] !== undefined) {
+      throw new Error(
+        `Attempt to apply embeddings to table failed because column ${destColumn} already existed`,
+      );
+    }
+    if (table.batches.length > 1) {
+      throw new Error(
+        "Internal error: `makeArrowTable` unexpectedly created a table with more than one batch",
+      );
+    }
+    const values = sourceColumn.toArray();
+
+    const vectors =
+      await functionEntry.function.computeSourceEmbeddings(values);
+    if (vectors.length !== values.length) {
+      throw new Error(
+        "Embedding function did not return an embedding for each input element",
+      );
+    }
+    let destType: DataType;
+    const dtype = schema.fields.find((f) => f.name === destColumn)!.type;
+    if (isFixedSizeList(dtype)) {
+      destType = sanitizeType(dtype);
+    } else {
+      throw new Error(
+        "Expected FixedSizeList as datatype for vector field, instead got: " +
+          dtype,
+      );
+    }
+    const vector = makeVector(vectors, destType);
+    columns[destColumn] = vector;
+  }
+  const newTable = new ArrowTable(columns);
+  return alignTable(newTable, schema);
+}
+
 /** Helper function to apply embeddings to an input table */
 async function applyEmbeddings<T>(
   table: ArrowTable,
-  embeddings?: EmbeddingFunction<T>,
+  embeddings?: EmbeddingFunctionConfig,
   schema?: Schema,
 ): Promise<ArrowTable> {
-  if (embeddings == null) {
+  if (schema?.metadata.has("embedding_functions")) {
+    return applyEmbeddingsFromMetadata(table, schema!);
+  } else if (embeddings == null || embeddings === undefined) {
     return table;
   }
 
@@ -399,8 +585,9 @@ async function applyEmbeddings<T>(
   const newColumns = Object.fromEntries(colEntries);
 
   const sourceColumn = newColumns[embeddings.sourceColumn];
-  const destColumn = embeddings.destColumn ?? "vector";
-  const innerDestType = embeddings.embeddingDataType ?? new Float32();
+  const destColumn = embeddings.vectorColumn ?? "vector";
+  const innerDestType =
+    embeddings.function.embeddingDataType() ?? new Float32();
   if (sourceColumn === undefined) {
     throw new Error(
       `Cannot apply embedding function because the source column '${embeddings.sourceColumn}' was not present in the data`,
@@ -414,11 +601,9 @@ async function applyEmbeddings<T>(
       // if we call convertToTable with 0 records and a schema that includes the embedding
       return table;
     }
-    if (embeddings.embeddingDimension !== undefined) {
-      const destType = newVectorType(
-        embeddings.embeddingDimension,
-        innerDestType,
-      );
+    const dimensions = embeddings.function.ndims();
+    if (dimensions !== undefined) {
+      const destType = newVectorType(dimensions, innerDestType);
       newColumns[destColumn] = makeVector([], destType);
     } else if (schema != null) {
       const destField = schema.fields.find((f) => f.name === destColumn);
@@ -446,7 +631,9 @@ async function applyEmbeddings<T>(
       );
     }
     const values = sourceColumn.toArray();
-    const vectors = await embeddings.embed(values as T[]);
+    const vectors = await embeddings.function.computeSourceEmbeddings(
+      values as T[],
+    );
     if (vectors.length !== values.length) {
       throw new Error(
         "Embedding function did not return an embedding for each input element",
@@ -486,9 +673,9 @@ async function applyEmbeddings<T>(
  * embedding columns.  If no schema is provded then embedding columns will
  * be placed at the end of the table, after all of the input columns.
  */
-export async function convertToTable<T>(
+export async function convertToTable(
   data: Array<Record<string, unknown>>,
-  embeddings?: EmbeddingFunction<T>,
+  embeddings?: EmbeddingFunctionConfig,
   makeTableOptions?: Partial<MakeArrowTableOptions>,
 ): Promise<ArrowTable> {
   const table = makeArrowTable(data, makeTableOptions);
@@ -496,13 +683,13 @@ export async function convertToTable<T>(
 }
 
 /** Creates the Arrow Type for a Vector column with dimension `dim` */
-function newVectorType<T extends Float>(
+export function newVectorType<T extends Float>(
   dim: number,
   innerType: T,
 ): FixedSizeList<T> {
   // in Lance we always default to have the elements nullable, so we need to set it to true
   // otherwise we often get schema mismatches because the stored data always has schema with nullable elements
-  const children = new Field<T>("item", innerType, true);
+  const children = new Field("item", <T>sanitizeType(innerType), true);
   return new FixedSizeList(dim, children);
 }
 
@@ -513,9 +700,9 @@ function newVectorType<T extends Float>(
  *
  * `schema` is required if data is empty
  */
-export async function fromRecordsToBuffer<T>(
+export async function fromRecordsToBuffer(
   data: Array<Record<string, unknown>>,
-  embeddings?: EmbeddingFunction<T>,
+  embeddings?: EmbeddingFunctionConfig,
   schema?: Schema,
 ): Promise<Buffer> {
   if (schema !== undefined && schema !== null) {
@@ -533,9 +720,9 @@ export async function fromRecordsToBuffer<T>(
  *
  * `schema` is required if data is empty
  */
-export async function fromRecordsToStreamBuffer<T>(
+export async function fromRecordsToStreamBuffer(
   data: Array<Record<string, unknown>>,
-  embeddings?: EmbeddingFunction<T>,
+  embeddings?: EmbeddingFunctionConfig,
   schema?: Schema,
 ): Promise<Buffer> {
   if (schema !== undefined && schema !== null) {
@@ -554,9 +741,9 @@ export async function fromRecordsToStreamBuffer<T>(
  *
  * `schema` is required if the table is empty
  */
-export async function fromTableToBuffer<T>(
+export async function fromTableToBuffer(
   table: ArrowTable,
-  embeddings?: EmbeddingFunction<T>,
+  embeddings?: EmbeddingFunctionConfig,
   schema?: Schema,
 ): Promise<Buffer> {
   if (schema !== undefined && schema !== null) {
@@ -575,19 +762,19 @@ export async function fromTableToBuffer<T>(
  *
  * `schema` is required if the table is empty
  */
-export async function fromDataToBuffer<T>(
+export async function fromDataToBuffer(
   data: Data,
-  embeddings?: EmbeddingFunction<T>,
+  embeddings?: EmbeddingFunctionConfig,
   schema?: Schema,
 ): Promise<Buffer> {
   if (schema !== undefined && schema !== null) {
     schema = sanitizeSchema(schema);
   }
-  if (data instanceof ArrowTable) {
+  if (isArrowTable(data)) {
     return fromTableToBuffer(data, embeddings, schema);
   } else {
-    const table = await convertToTable(data);
-    return fromTableToBuffer(table, embeddings, schema);
+    const table = await convertToTable(data, embeddings, { schema });
+    return fromTableToBuffer(table);
   }
 }
 
@@ -599,9 +786,9 @@ export async function fromDataToBuffer<T>(
  *
  * `schema` is required if the table is empty
  */
-export async function fromTableToStreamBuffer<T>(
+export async function fromTableToStreamBuffer(
   table: ArrowTable,
-  embeddings?: EmbeddingFunction<T>,
+  embeddings?: EmbeddingFunctionConfig,
   schema?: Schema,
 ): Promise<Buffer> {
   const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
@@ -654,7 +841,7 @@ export function createEmptyTable(schema: Schema): ArrowTable {
 function validateSchemaEmbeddings(
   schema: Schema,
   data: Array<Record<string, unknown>>,
-  embeddings: EmbeddingFunction<unknown> | undefined,
+  embeddings: EmbeddingFunctionConfig | undefined,
 ) {
   const fields = [];
   const missingEmbeddingFields = [];
@@ -664,10 +851,25 @@ function validateSchemaEmbeddings(
   // if it does not, we add it to the list of missing embedding fields
   // Finally, we check if those missing embedding fields are `this._embeddings`
   // if they are not, we throw an error
-  for (const field of schema.fields) {
-    if (field.type instanceof FixedSizeList) {
+  for (let field of schema.fields) {
+    if (isFixedSizeList(field.type)) {
+      field = sanitizeField(field);
+
       if (data.length !== 0 && data?.[0]?.[field.name] === undefined) {
-        missingEmbeddingFields.push(field);
+        if (schema.metadata.has("embedding_functions")) {
+          const embeddings = JSON.parse(
+            schema.metadata.get("embedding_functions")!,
+          );
+          if (
+            // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
+            embeddings.find((f: any) => f["vectorColumn"] === field.name) ===
+            undefined
+          ) {
+            missingEmbeddingFields.push(field);
+          }
+        } else {
+          missingEmbeddingFields.push(field);
+        }
       } else {
         fields.push(field);
       }

nodejs/lancedb/connection.ts

@@ -12,8 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Table as ArrowTable, Schema } from "apache-arrow";
-import { fromTableToBuffer, makeArrowTable, makeEmptyTable } from "./arrow";
+import { Table as ArrowTable, Schema } from "./arrow";
+import {
+  fromTableToBuffer,
+  isArrowTable,
+  makeArrowTable,
+  makeEmptyTable,
+} from "./arrow";
+import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { ConnectionOptions, Connection as LanceDbConnection } from "./native";
 import { Table } from "./table";
 
@@ -65,6 +71,14 @@ export interface CreateTableOptions {
    * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
    */
   storageOptions?: Record<string, string>;
+  /**
+   * If true then data files will be written with the legacy format
+   *
+   * The default is true while the new format is in beta
+   */
+  useLegacyFormat?: boolean;
+  schema?: Schema;
+  embeddingFunction?: EmbeddingFunctionConfig;
 }
 
 export interface OpenTableOptions {
@@ -174,6 +188,7 @@ export class Connection {
       cleanseStorageOptions(options?.storageOptions),
       options?.indexCacheSize,
     );
+
     return new Table(innerTable);
   }
 
@@ -196,18 +211,25 @@ export class Connection {
     }
 
     let table: ArrowTable;
-    if (data instanceof ArrowTable) {
+    if (isArrowTable(data)) {
       table = data;
     } else {
-      table = makeArrowTable(data);
+      table = makeArrowTable(data, options);
     }
-    const buf = await fromTableToBuffer(table);
+
+    const buf = await fromTableToBuffer(
+      table,
+      options?.embeddingFunction,
+      options?.schema,
+    );
     const innerTable = await this.inner.createTable(
       name,
       buf,
       mode,
       cleanseStorageOptions(options?.storageOptions),
+      options?.useLegacyFormat,
     );
+
     return new Table(innerTable);
   }
 
@@ -227,14 +249,21 @@ export class Connection {
     if (mode === "create" && existOk) {
       mode = "exist_ok";
     }
+    let metadata: Map<string, string> | undefined = undefined;
+    if (options?.embeddingFunction !== undefined) {
+      const embeddingFunction = options.embeddingFunction;
+      const registry = getRegistry();
+      metadata = registry.getTableMetadata([embeddingFunction]);
+    }
 
-    const table = makeEmptyTable(schema);
+    const table = makeEmptyTable(schema, metadata);
     const buf = await fromTableToBuffer(table);
     const innerTable = await this.inner.createEmptyTable(
       name,
       buf,
       mode,
       cleanseStorageOptions(options?.storageOptions),
+      options?.useLegacyFormat,
     );
     return new Table(innerTable);
   }

nodejs/lancedb/embedding/embedding_function.ts

@@ -1,4 +1,4 @@
-// Copyright 2023 Lance Developers.
+// Copyright 2024 Lance Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,67 +12,172 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { type Float } from "apache-arrow";
+import "reflect-metadata";
+import {
+  DataType,
+  Field,
+  FixedSizeList,
+  Float,
+  Float32,
+  type IntoVector,
+  isDataType,
+  isFixedSizeList,
+  isFloat,
+  newVectorType,
+} from "../arrow";
+import { sanitizeType } from "../sanitize";
+
+/**
+ * Options for a given embedding function
+ */
+export interface FunctionOptions {
+  // biome-ignore lint/suspicious/noExplicitAny: options can be anything
+  [key: string]: any;
+}
 
 /**
  * An embedding function that automatically creates vector representation for a given column.
  */
-export interface EmbeddingFunction<T> {
+export abstract class EmbeddingFunction<
+  // biome-ignore lint/suspicious/noExplicitAny: we don't know what the implementor will do
+  T = any,
+  M extends FunctionOptions = FunctionOptions,
+> {
   /**
-   * The name of the column that will be used as input for the Embedding Function.
+   * Convert the embedding function to a JSON object
+   * It is used to serialize the embedding function to the schema
+   * It's important that any object returned by this method contains all the necessary
+   * information to recreate the embedding function
+   *
+   * It should return the same object that was passed to the constructor
+   * If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly
+   *
+   * @example
+   * ```ts
+   * class MyEmbeddingFunction extends EmbeddingFunction {
+   *   constructor(options: {model: string, timeout: number}) {
+   *     super();
+   *     this.model = options.model;
+   *     this.timeout = options.timeout;
+   *   }
+   *   toJSON() {
+   *     return {
+   *       model: this.model,
+   *       timeout: this.timeout,
+   *     };
+   * }
+   * ```
    */
-  sourceColumn: string;
+  abstract toJSON(): Partial<M>;
 
   /**
-   * The data type of the embedding
+   * sourceField is used in combination with `LanceSchema` to provide a declarative data model
    *
-   * The embedding function should return `number`.  This will be converted into
-   * an Arrow float array.  By default this will be Float32 but this property can
-   * be used to control the conversion.
+   * @param optionsOrDatatype - The options for the field or the datatype
+   *
+   * @see {@link lancedb.LanceSchema}
    */
-  embeddingDataType?: Float;
+  sourceField(
+    optionsOrDatatype: Partial<FieldOptions> | DataType,
+  ): [DataType, Map<string, EmbeddingFunction>] {
+    let datatype = isDataType(optionsOrDatatype)
+      ? optionsOrDatatype
+      : optionsOrDatatype?.datatype;
+    if (!datatype) {
+      throw new Error("Datatype is required");
+    }
+    datatype = sanitizeType(datatype);
+    const metadata = new Map<string, EmbeddingFunction>();
+    metadata.set("source_column_for", this);
+
+    return [datatype, metadata];
+  }
 
   /**
-   * The dimension of the embedding
+   * vectorField is used in combination with `LanceSchema` to provide a declarative data model
    *
-   * This is optional, normally this can be determined by looking at the results of
-   * `embed`.  If this is not specified, and there is an attempt to apply the embedding
-   * to an empty table, then that process will fail.
+   * @param options - The options for the field
+   *
+   * @see {@link lancedb.LanceSchema}
    */
-  embeddingDimension?: number;
+  vectorField(
+    optionsOrDatatype?: Partial<FieldOptions> | DataType,
+  ): [DataType, Map<string, EmbeddingFunction>] {
+    let dtype: DataType | undefined;
+    let vectorType: DataType;
+    let dims: number | undefined = this.ndims();
 
-  /**
-   * The name of the column that will contain the embedding
-   *
-   * By default this is "vector"
-   */
-  destColumn?: string;
+    // `func.vectorField(new Float32())`
+    if (isDataType(optionsOrDatatype)) {
+      dtype = optionsOrDatatype;
+    } else {
+      // `func.vectorField({
+      //  datatype: new Float32(),
+      //  dims: 10
+      // })`
+      dims = dims ?? optionsOrDatatype?.dims;
+      dtype = optionsOrDatatype?.datatype;
+    }
 
-  /**
-   * Should the source column be excluded from the resulting table
-   *
-   * By default the source column is included.  Set this to true and
-   * only the embedding will be stored.
-   */
-  excludeSource?: boolean;
+    if (dtype !== undefined) {
+      // `func.vectorField(new FixedSizeList(dims, new Field("item", new Float32(), true)))`
+      // or `func.vectorField({datatype: new FixedSizeList(dims, new Field("item", new Float32(), true))})`
+      if (isFixedSizeList(dtype)) {
+        vectorType = dtype;
+        // `func.vectorField(new Float32())`
+        // or `func.vectorField({datatype: new Float32()})`
+      } else if (isFloat(dtype)) {
+        // No `ndims` impl and no `{dims: n}` provided;
+        if (dims === undefined) {
+          throw new Error("ndims is required for vector field");
+        }
+        vectorType = newVectorType(dims, dtype);
+      } else {
+        throw new Error(
+          "Expected FixedSizeList or Float as datatype for vector field",
+        );
+      }
+    } else {
+      if (dims === undefined) {
+        throw new Error("ndims is required for vector field");
+      }
+      vectorType = new FixedSizeList(
+        dims,
+        new Field("item", new Float32(), true),
+      );
+    }
+    const metadata = new Map<string, EmbeddingFunction>();
+    metadata.set("vector_column_for", this);
+
+    return [vectorType, metadata];
+  }
+
+  /** The number of dimensions of the embeddings */
+  ndims(): number | undefined {
+    return undefined;
+  }
+
+  /** The datatype of the embeddings */
+  abstract embeddingDataType(): Float;
 
   /**
    * Creates a vector representation for the given values.
    */
-  embed: (data: T[]) => Promise<number[][]>;
+  abstract computeSourceEmbeddings(
+    data: T[],
+  ): Promise<number[][] | Float32Array[] | Float64Array[]>;
+
+  /**
+  Compute the embeddings for a single query
+ */
+  async computeQueryEmbeddings(data: T): Promise<IntoVector> {
+    return this.computeSourceEmbeddings([data]).then(
+      (embeddings) => embeddings[0],
+    );
+  }
 }
 
-/** Test if the input seems to be an embedding function */
-export function isEmbeddingFunction<T>(
-  value: unknown,
-): value is EmbeddingFunction<T> {
-  if (typeof value !== "object" || value === null) {
-    return false;
-  }
-  if (!("sourceColumn" in value) || !("embed" in value)) {
-    return false;
-  }
-  return (
-    typeof value.sourceColumn === "string" && typeof value.embed === "function"
-  );
+export interface FieldOptions<T extends DataType = DataType> {
+  datatype: T;
+  dims?: number;
 }

nodejs/lancedb/embedding/index.ts

@@ -1,2 +1,113 @@
-export { EmbeddingFunction, isEmbeddingFunction } from "./embedding_function";
-export { OpenAIEmbeddingFunction } from "./openai";
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { DataType, Field, Schema } from "../arrow";
+import { isDataType } from "../arrow";
+import { sanitizeType } from "../sanitize";
+import { EmbeddingFunction } from "./embedding_function";
+import { EmbeddingFunctionConfig, getRegistry } from "./registry";
+
+export { EmbeddingFunction } from "./embedding_function";
+
+// We need to explicitly export '*' so that the `register` decorator actually registers the class.
+export * from "./openai";
+export * from "./registry";
+
+/**
+ * Create a schema with embedding functions.
+ *
+ * @param fields
+ * @returns Schema
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ * // ...
+ * }
+ * const func = new MyEmbeddingFunction();
+ * const schema = LanceSchema({
+ *   id: new Int32(),
+ *   text: func.sourceField(new Utf8()),
+ *   vector: func.vectorField(),
+ *   // optional: specify the datatype and/or dimensions
+ *   vector2: func.vectorField({ datatype: new Float32(), dims: 3}),
+ * });
+ *
+ * const table = await db.createTable("my_table", data, { schema });
+ * ```
+ */
+export function LanceSchema(
+  fields: Record<string, [object, Map<string, EmbeddingFunction>] | object>,
+): Schema {
+  const arrowFields: Field[] = [];
+
+  const embeddingFunctions = new Map<
+    EmbeddingFunction,
+    Partial<EmbeddingFunctionConfig>
+  >();
+  Object.entries(fields).forEach(([key, value]) => {
+    if (isDataType(value)) {
+      arrowFields.push(new Field(key, sanitizeType(value), true));
+    } else {
+      const [dtype, metadata] = value as [
+        object,
+        Map<string, EmbeddingFunction>,
+      ];
+      arrowFields.push(new Field(key, sanitizeType(dtype), true));
+      parseEmbeddingFunctions(embeddingFunctions, key, metadata);
+    }
+  });
+  const registry = getRegistry();
+  const metadata = registry.getTableMetadata(
+    Array.from(embeddingFunctions.values()) as EmbeddingFunctionConfig[],
+  );
+  const schema = new Schema(arrowFields, metadata);
+  return schema;
+}
+
+function parseEmbeddingFunctions(
+  embeddingFunctions: Map<EmbeddingFunction, Partial<EmbeddingFunctionConfig>>,
+  key: string,
+  metadata: Map<string, EmbeddingFunction>,
+): void {
+  if (metadata.has("source_column_for")) {
+    const embedFunction = metadata.get("source_column_for")!;
+    const current = embeddingFunctions.get(embedFunction);
+    if (current !== undefined) {
+      embeddingFunctions.set(embedFunction, {
+        ...current,
+        sourceColumn: key,
+      });
+    } else {
+      embeddingFunctions.set(embedFunction, {
+        sourceColumn: key,
+        function: embedFunction,
+      });
+    }
+  } else if (metadata.has("vector_column_for")) {
+    const embedFunction = metadata.get("vector_column_for")!;
+
+    const current = embeddingFunctions.get(embedFunction);
+    if (current !== undefined) {
+      embeddingFunctions.set(embedFunction, {
+        ...current,
+        vectorColumn: key,
+      });
+    } else {
+      embeddingFunctions.set(embedFunction, {
+        vectorColumn: key,
+        function: embedFunction,
+      });
+    }
+  }
+}

nodejs/lancedb/embedding/openai.ts

@@ -13,17 +13,31 @@
 // limitations under the License.
 
 import type OpenAI from "openai";
-import { type EmbeddingFunction } from "./embedding_function";
+import { Float, Float32 } from "../arrow";
+import { EmbeddingFunction } from "./embedding_function";
+import { register } from "./registry";
 
-export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
-  private readonly _openai: OpenAI;
-  private readonly _modelName: string;
+export type OpenAIOptions = {
+  apiKey?: string;
+  model?: string;
+};
+
+@register("openai")
+export class OpenAIEmbeddingFunction extends EmbeddingFunction<
+  string,
+  OpenAIOptions
+> {
+  #openai: OpenAI;
+  #modelName: string;
+
+  constructor(options: OpenAIOptions = { model: "text-embedding-ada-002" }) {
+    super();
+    const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
+    if (!openAIKey) {
+      throw new Error("OpenAI API key is required");
+    }
+    const modelName = options?.model ?? "text-embedding-ada-002";
 
-  constructor(
-    sourceColumn: string,
-    openAIKey: string,
-    modelName: string = "text-embedding-ada-002",
-  ) {
     /**
      * @type {import("openai").default}
      */
@@ -36,18 +50,40 @@ export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
       throw new Error("please install openai@^4.24.1 using npm install openai");
     }
 
-    this.sourceColumn = sourceColumn;
     const configuration = {
       apiKey: openAIKey,
     };
 
-    this._openai = new Openai(configuration);
-    this._modelName = modelName;
+    this.#openai = new Openai(configuration);
+    this.#modelName = modelName;
   }
 
-  async embed(data: string[]): Promise<number[][]> {
-    const response = await this._openai.embeddings.create({
-      model: this._modelName,
+  toJSON() {
+    return {
+      model: this.#modelName,
+    };
+  }
+
+  ndims(): number {
+    switch (this.#modelName) {
+      case "text-embedding-ada-002":
+        return 1536;
+      case "text-embedding-3-large":
+        return 3072;
+      case "text-embedding-3-small":
+        return 1536;
+      default:
+        return null as never;
+    }
+  }
+
+  embeddingDataType(): Float {
+    return new Float32();
+  }
+
+  async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
+    const response = await this.#openai.embeddings.create({
+      model: this.#modelName,
       input: data,
     });
 
@@ -58,5 +94,15 @@ export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
     return embeddings;
   }
 
-  sourceColumn: string;
+  async computeQueryEmbeddings(data: string): Promise<number[]> {
+    if (typeof data !== "string") {
+      throw new Error("Data must be a string");
+    }
+    const response = await this.#openai.embeddings.create({
+      model: this.#modelName,
+      input: data,
+    });
+
+    return response.data[0].embedding;
+  }
 }

nodejs/lancedb/embedding/registry.ts

@@ -0,0 +1,176 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import type { EmbeddingFunction } from "./embedding_function";
+import "reflect-metadata";
+
+export interface EmbeddingFunctionOptions {
+  [key: string]: unknown;
+}
+
+export interface EmbeddingFunctionFactory<
+  T extends EmbeddingFunction = EmbeddingFunction,
+> {
+  new (modelOptions?: EmbeddingFunctionOptions): T;
+}
+
+interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
+  create(options?: EmbeddingFunctionOptions): T;
+}
+
+/**
+ * This is a singleton class used to register embedding functions
+ * and fetch them by name. It also handles serializing and deserializing.
+ * You can implement your own embedding function by subclassing EmbeddingFunction
+ * or TextEmbeddingFunction and registering it with the registry
+ */
+export class EmbeddingFunctionRegistry {
+  #functions: Map<string, EmbeddingFunctionFactory> = new Map();
+
+  /**
+   * Register an embedding function
+   * @param name The name of the function
+   * @param func The function to register
+   * @throws Error if the function is already registered
+   */
+  register<T extends EmbeddingFunctionFactory = EmbeddingFunctionFactory>(
+    this: EmbeddingFunctionRegistry,
+    alias?: string,
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  ): (ctor: T) => any {
+    const self = this;
+    return function (ctor: T) {
+      if (!alias) {
+        alias = ctor.name;
+      }
+      if (self.#functions.has(alias)) {
+        throw new Error(
+          `Embedding function with alias "${alias}" already exists`,
+        );
+      }
+      self.#functions.set(alias, ctor);
+      Reflect.defineMetadata("lancedb::embedding::name", alias, ctor);
+      return ctor;
+    };
+  }
+
+  /**
+   * Fetch an embedding function by name
+   * @param name The name of the function
+   */
+  get<T extends EmbeddingFunction<unknown> = EmbeddingFunction>(
+    name: string,
+  ): EmbeddingFunctionCreate<T> | undefined {
+    const factory = this.#functions.get(name);
+    if (!factory) {
+      return undefined;
+    }
+    return {
+      create: function (options: EmbeddingFunctionOptions) {
+        return new factory(options) as unknown as T;
+      },
+    };
+  }
+
+  /**
+   * reset the registry to the initial state
+   */
+  reset(this: EmbeddingFunctionRegistry) {
+    this.#functions.clear();
+  }
+
+  /**
+   * @ignore
+   */
+  parseFunctions(
+    this: EmbeddingFunctionRegistry,
+    metadata: Map<string, string>,
+  ): Map<string, EmbeddingFunctionConfig> {
+    if (!metadata.has("embedding_functions")) {
+      return new Map();
+    } else {
+      type FunctionConfig = {
+        name: string;
+        sourceColumn: string;
+        vectorColumn: string;
+        model: EmbeddingFunctionOptions;
+      };
+      const functions = <FunctionConfig[]>(
+        JSON.parse(metadata.get("embedding_functions")!)
+      );
+      return new Map(
+        functions.map((f) => {
+          const fn = this.get(f.name);
+          if (!fn) {
+            throw new Error(`Function "${f.name}" not found in registry`);
+          }
+          return [
+            f.name,
+            {
+              sourceColumn: f.sourceColumn,
+              vectorColumn: f.vectorColumn,
+              function: this.get(f.name)!.create(f.model),
+            },
+          ];
+        }),
+      );
+    }
+  }
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any> {
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    const metadata: Record<string, any> = {};
+    const name = Reflect.getMetadata(
+      "lancedb::embedding::name",
+      conf.function.constructor,
+    );
+    metadata["sourceColumn"] = conf.sourceColumn;
+    metadata["vectorColumn"] = conf.vectorColumn ?? "vector";
+    metadata["name"] = name ?? conf.function.constructor.name;
+    metadata["model"] = conf.function.toJSON();
+    return metadata;
+  }
+
+  getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string> {
+    const metadata = new Map<string, string>();
+    const jsonData = functions.map((conf) => this.functionToMetadata(conf));
+    metadata.set("embedding_functions", JSON.stringify(jsonData));
+
+    return metadata;
+  }
+}
+
+const _REGISTRY = new EmbeddingFunctionRegistry();
+
+export function register(name?: string) {
+  return _REGISTRY.register(name);
+}
+
+/**
+ * Utility function to get the global instance of the registry
+ * @returns `EmbeddingFunctionRegistry` The global instance of the registry
+ * @example
+ * ```ts
+ * const registry = getRegistry();
+ * const openai = registry.get("openai").create();
+ */
+export function getRegistry(): EmbeddingFunctionRegistry {
+  return _REGISTRY;
+}
+
+export interface EmbeddingFunctionConfig {
+  sourceColumn: string;
+  vectorColumn?: string;
+  function: EmbeddingFunction;
+}

nodejs/lancedb/query.ts

@@ -12,7 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Table as ArrowTable, RecordBatch, tableFromIPC } from "apache-arrow";
+import {
+  Table as ArrowTable,
+  type IntoVector,
+  RecordBatch,
+  tableFromIPC,
+} from "./arrow";
 import { type IvfPqOptions } from "./indices";
 import {
   RecordBatchIterator as NativeBatchIterator,
@@ -50,6 +55,39 @@ export class RecordBatchIterator implements AsyncIterator<RecordBatch> {
 }
 /* eslint-enable */
 
+class RecordBatchIterable<
+  NativeQueryType extends NativeQuery | NativeVectorQuery,
+> implements AsyncIterable<RecordBatch>
+{
+  private inner: NativeQueryType;
+  private options?: QueryExecutionOptions;
+
+  constructor(inner: NativeQueryType, options?: QueryExecutionOptions) {
+    this.inner = inner;
+    this.options = options;
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: skip
+  [Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>, any, undefined> {
+    return new RecordBatchIterator(
+      this.inner.execute(this.options?.maxBatchLength),
+    );
+  }
+}
+
+/**
+ * Options that control the behavior of a particular query execution
+ */
+export interface QueryExecutionOptions {
+  /**
+   * The maximum number of rows to return in a single batch
+   *
+   * Batches may have fewer rows if the underlying data is stored
+   * in smaller chunks.
+   */
+  maxBatchLength?: number;
+}
+
 /** Common methods supported by all query types */
 export class QueryBase<
   NativeQueryType extends NativeQuery | NativeVectorQuery,
@@ -108,9 +146,12 @@ export class QueryBase<
    * object insertion order is easy to get wrong and `Map` is more foolproof.
    */
   select(
-    columns: string[] | Map<string, string> | Record<string, string>,
+    columns: string[] | Map<string, string> | Record<string, string> | string,
   ): QueryType {
     let columnTuples: [string, string][];
+    if (typeof columns === "string") {
+      columns = [columns];
+    }
     if (Array.isArray(columns)) {
       columnTuples = columns.map((c) => [c, c]);
     } else if (columns instanceof Map) {
@@ -133,8 +174,10 @@ export class QueryBase<
     return this as unknown as QueryType;
   }
 
-  protected nativeExecute(): Promise<NativeBatchIterator> {
-    return this.inner.execute();
+  protected nativeExecute(
+    options?: Partial<QueryExecutionOptions>,
+  ): Promise<NativeBatchIterator> {
+    return this.inner.execute(options?.maxBatchLength);
   }
 
   /**
@@ -148,8 +191,10 @@ export class QueryBase<
    * single query)
    *
    */
-  protected execute(): RecordBatchIterator {
-    return new RecordBatchIterator(this.nativeExecute());
+  protected execute(
+    options?: Partial<QueryExecutionOptions>,
+  ): RecordBatchIterator {
+    return new RecordBatchIterator(this.nativeExecute(options));
   }
 
   // biome-ignore lint/suspicious/noExplicitAny: skip
@@ -159,18 +204,18 @@ export class QueryBase<
   }
 
   /** Collect the results as an Arrow @see {@link ArrowTable}. */
-  async toArrow(): Promise<ArrowTable> {
+  async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
     const batches = [];
-    for await (const batch of this) {
+    for await (const batch of new RecordBatchIterable(this.inner, options)) {
       batches.push(batch);
     }
     return new ArrowTable(batches);
   }
 
   /** Collect the results as an array of objects. */
-  async toArray(): Promise<unknown[]> {
-    const tbl = await this.toArrow();
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+  // biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
+  async toArray(options?: Partial<QueryExecutionOptions>): Promise<any[]> {
+    const tbl = await this.toArrow(options);
     return tbl.toArray();
   }
 }
@@ -369,9 +414,8 @@ export class Query extends QueryBase<NativeQuery, Query> {
    * Vector searches always have a `limit`.  If `limit` has not been called then
    * a default `limit` of 10 will be used.  @see {@link Query#limit}
    */
-  nearestTo(vector: unknown): VectorQuery {
-    // biome-ignore lint/suspicious/noExplicitAny: skip
-    const vectorQuery = this.inner.nearestTo(Float32Array.from(vector as any));
+  nearestTo(vector: IntoVector): VectorQuery {
+    const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
     return new VectorQuery(vectorQuery);
   }
 }

nodejs/lancedb/sanitize.ts

@@ -20,6 +20,7 @@
 // comes from the exact same library instance.  This is not always the case
 // and so we must sanitize the input to ensure that it is compatible.
 
+import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
 import {
   Binary,
   Bool,
@@ -75,10 +76,9 @@ import {
   Uint64,
   Union,
   Utf8,
-} from "apache-arrow";
-import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
+} from "./arrow";
 
-function sanitizeMetadata(
+export function sanitizeMetadata(
   metadataLike?: unknown,
 ): Map<string, string> | undefined {
   if (metadataLike === undefined || metadataLike === null) {
@@ -97,7 +97,7 @@ function sanitizeMetadata(
   return metadataLike as Map<string, string>;
 }
 
-function sanitizeInt(typeLike: object) {
+export function sanitizeInt(typeLike: object) {
   if (
     !("bitWidth" in typeLike) ||
     typeof typeLike.bitWidth !== "number" ||
@@ -111,14 +111,14 @@ function sanitizeInt(typeLike: object) {
   return new Int(typeLike.isSigned, typeLike.bitWidth as IntBitWidth);
 }
 
-function sanitizeFloat(typeLike: object) {
+export function sanitizeFloat(typeLike: object) {
   if (!("precision" in typeLike) || typeof typeLike.precision !== "number") {
     throw Error("Expected a Float Type to have a `precision` property");
   }
   return new Float(typeLike.precision as Precision);
 }
 
-function sanitizeDecimal(typeLike: object) {
+export function sanitizeDecimal(typeLike: object) {
   if (
     !("scale" in typeLike) ||
     typeof typeLike.scale !== "number" ||
@@ -134,14 +134,14 @@ function sanitizeDecimal(typeLike: object) {
   return new Decimal(typeLike.scale, typeLike.precision, typeLike.bitWidth);
 }
 
-function sanitizeDate(typeLike: object) {
+export function sanitizeDate(typeLike: object) {
   if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
     throw Error("Expected a Date type to have a `unit` property");
   }
   return new Date_(typeLike.unit as DateUnit);
 }
 
-function sanitizeTime(typeLike: object) {
+export function sanitizeTime(typeLike: object) {
   if (
     !("unit" in typeLike) ||
     typeof typeLike.unit !== "number" ||
@@ -155,7 +155,7 @@ function sanitizeTime(typeLike: object) {
   return new Time(typeLike.unit, typeLike.bitWidth as TimeBitWidth);
 }
 
-function sanitizeTimestamp(typeLike: object) {
+export function sanitizeTimestamp(typeLike: object) {
   if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
     throw Error("Expected a Timestamp type to have a `unit` property");
   }
@@ -166,7 +166,7 @@ function sanitizeTimestamp(typeLike: object) {
   return new Timestamp(typeLike.unit, timezone);
 }
 
-function sanitizeTypedTimestamp(
+export function sanitizeTypedTimestamp(
   typeLike: object,
   // eslint-disable-next-line @typescript-eslint/naming-convention
   Datatype:
@@ -182,14 +182,14 @@ function sanitizeTypedTimestamp(
   return new Datatype(timezone);
 }
 
-function sanitizeInterval(typeLike: object) {
+export function sanitizeInterval(typeLike: object) {
   if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
     throw Error("Expected an Interval type to have a `unit` property");
   }
   return new Interval(typeLike.unit);
 }
 
-function sanitizeList(typeLike: object) {
+export function sanitizeList(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a List type to have an array-like `children` property",
@@ -201,7 +201,7 @@ function sanitizeList(typeLike: object) {
   return new List(sanitizeField(typeLike.children[0]));
 }
 
-function sanitizeStruct(typeLike: object) {
+export function sanitizeStruct(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a Struct type to have an array-like `children` property",
@@ -210,7 +210,7 @@ function sanitizeStruct(typeLike: object) {
   return new Struct(typeLike.children.map((child) => sanitizeField(child)));
 }
 
-function sanitizeUnion(typeLike: object) {
+export function sanitizeUnion(typeLike: object) {
   if (
     !("typeIds" in typeLike) ||
     !("mode" in typeLike) ||
@@ -234,7 +234,7 @@ function sanitizeUnion(typeLike: object) {
   );
 }
 
-function sanitizeTypedUnion(
+export function sanitizeTypedUnion(
   typeLike: object,
   // eslint-disable-next-line @typescript-eslint/naming-convention
   UnionType: typeof DenseUnion | typeof SparseUnion,
@@ -256,7 +256,7 @@ function sanitizeTypedUnion(
   );
 }
 
-function sanitizeFixedSizeBinary(typeLike: object) {
+export function sanitizeFixedSizeBinary(typeLike: object) {
   if (!("byteWidth" in typeLike) || typeof typeLike.byteWidth !== "number") {
     throw Error(
       "Expected a FixedSizeBinary type to have a `byteWidth` property",
@@ -265,7 +265,7 @@ function sanitizeFixedSizeBinary(typeLike: object) {
   return new FixedSizeBinary(typeLike.byteWidth);
 }
 
-function sanitizeFixedSizeList(typeLike: object) {
+export function sanitizeFixedSizeList(typeLike: object) {
   if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
     throw Error("Expected a FixedSizeList type to have a `listSize` property");
   }
@@ -283,7 +283,7 @@ function sanitizeFixedSizeList(typeLike: object) {
   );
 }
 
-function sanitizeMap(typeLike: object) {
+export function sanitizeMap(typeLike: object) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a Map type to have an array-like `children` property",
@@ -300,14 +300,14 @@ function sanitizeMap(typeLike: object) {
   );
 }
 
-function sanitizeDuration(typeLike: object) {
+export function sanitizeDuration(typeLike: object) {
   if (!("unit" in typeLike) || typeof typeLike.unit !== "number") {
     throw Error("Expected a Duration type to have a `unit` property");
   }
   return new Duration(typeLike.unit);
 }
 
-function sanitizeDictionary(typeLike: object) {
+export function sanitizeDictionary(typeLike: object) {
   if (!("id" in typeLike) || typeof typeLike.id !== "number") {
     throw Error("Expected a Dictionary type to have an `id` property");
   }
@@ -329,7 +329,7 @@ function sanitizeDictionary(typeLike: object) {
 }
 
 // biome-ignore lint/suspicious/noExplicitAny: skip
-function sanitizeType(typeLike: unknown): DataType<any> {
+export function sanitizeType(typeLike: unknown): DataType<any> {
   if (typeof typeLike !== "object" || typeLike === null) {
     throw Error("Expected a Type but object was null/undefined");
   }
@@ -449,7 +449,7 @@ function sanitizeType(typeLike: unknown): DataType<any> {
   }
 }
 
-function sanitizeField(fieldLike: unknown): Field {
+export function sanitizeField(fieldLike: unknown): Field {
   if (fieldLike instanceof Field) {
     return fieldLike;
   }

nodejs/lancedb/table.ts

@@ -12,8 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Schema, tableFromIPC } from "apache-arrow";
-import { Data, fromDataToBuffer } from "./arrow";
+import {
+  Table as ArrowTable,
+  Data,
+  IntoVector,
+  Schema,
+  fromDataToBuffer,
+  tableFromIPC,
+} from "./arrow";
+
+import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { IndexOptions } from "./indices";
 import {
   AddColumnsSql,
@@ -23,8 +31,8 @@ import {
   Table as _NativeTable,
 } from "./native";
 import { Query, VectorQuery } from "./query";
-
 export { IndexConfig } from "./native";
+
 /**
  * Options for adding data to a table.
  */
@@ -109,6 +117,14 @@ export class Table {
     return this.inner.display();
   }
 
+  async #getEmbeddingFunctions(): Promise<
+    Map<string, EmbeddingFunctionConfig>
+  > {
+    const schema = await this.schema();
+    const registry = getRegistry();
+    return registry.parseFunctions(schema.metadata);
+  }
+
   /** Get the schema of the table. */
   async schema(): Promise<Schema> {
     const schemaBuf = await this.inner.schema();
@@ -122,8 +138,15 @@ export class Table {
    */
   async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
     const mode = options?.mode ?? "append";
+    const schema = await this.schema();
+    const registry = getRegistry();
+    const functions = registry.parseFunctions(schema.metadata);
 
-    const buffer = await fromDataToBuffer(data);
+    const buffer = await fromDataToBuffer(
+      data,
+      functions.values().next().value,
+      schema,
+    );
     await this.inner.add(buffer, mode);
   }
 
@@ -263,6 +286,40 @@ export class Table {
     return new Query(this.inner);
   }
 
+  /**
+   * Create a search query to find the nearest neighbors
+   * of the given query vector
+   * @param {string} query - the query. This will be converted to a vector using the table's provided embedding function
+   * @rejects {Error} If no embedding functions are defined in the table
+   */
+  search(query: string): Promise<VectorQuery>;
+  /**
+   * Create a search query to find the nearest neighbors
+   * of the given query vector
+   * @param {IntoVector} query - the query vector
+   */
+  search(query: IntoVector): VectorQuery;
+  search(query: string | IntoVector): Promise<VectorQuery> | VectorQuery {
+    if (typeof query !== "string") {
+      return this.vectorSearch(query);
+    } else {
+      return this.#getEmbeddingFunctions().then(async (functions) => {
+        // TODO: Support multiple embedding functions
+        const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
+          .values()
+          .next().value;
+        if (!embeddingFunc) {
+          return Promise.reject(
+            new Error("No embedding functions are defined in the table"),
+          );
+        }
+        const embeddings =
+          await embeddingFunc.function.computeQueryEmbeddings(query);
+        return this.query().nearestTo(embeddings);
+      });
+    }
+  }
+
   /**
    * Search the table with a given query vector.
    *
@@ -270,7 +327,7 @@ export class Table {
    * is the same thing as calling `nearestTo` on the builder returned
    * by `query`.  @see {@link Query#nearestTo} for more details.
    */
-  vectorSearch(vector: unknown): VectorQuery {
+  vectorSearch(vector: IntoVector): VectorQuery {
     return this.query().nearestTo(vector);
   }
 
@@ -416,4 +473,9 @@ export class Table {
   async listIndices(): Promise<IndexConfig[]> {
     return await this.inner.listIndices();
   }
+
+  /** Return the table as an arrow table */
+  async toArrow(): Promise<ArrowTable> {
+    return await this.query().toArrow();
+  }
 }

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.5.0",
+	"version": "0.5.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.5.0",
+	"version": "0.5.1",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.5.0",
+	"version": "0.5.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.5.0",
+	"version": "0.5.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.5.0",
+	"version": "0.5.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -1,8 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.5.0",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "version": "0.5.1",
+  "main": "dist/index.js",
+  "exports": {
+    ".": "./dist/index.js",
+    "./embedding": "./dist/embedding/index.js"
+  },
+  "types": "dist/index.d.ts",
   "napi": {
     "name": "lancedb",
     "triples": {
@@ -62,6 +66,7 @@
   },
   "dependencies": {
     "apache-arrow": "^15.0.0",
-    "openai": "^4.29.2"
+    "openai": "^4.29.2",
+    "reflect-metadata": "^0.2.2"
   }
 }

nodejs/src/connection.rs

@@ -126,6 +126,7 @@ impl Connection {
         buf: Buffer,
         mode: String,
         storage_options: Option<HashMap<String, String>>,
+        use_legacy_format: Option<bool>,
     ) -> napi::Result<Table> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
@@ -136,6 +137,9 @@ impl Connection {
                 builder = builder.storage_option(key, value);
             }
         }
+        if let Some(use_legacy_format) = use_legacy_format {
+            builder = builder.use_legacy_format(use_legacy_format);
+        }
         let tbl = builder
             .execute()
             .await
@@ -150,6 +154,7 @@ impl Connection {
         schema_buf: Buffer,
         mode: String,
         storage_options: Option<HashMap<String, String>>,
+        use_legacy_format: Option<bool>,
     ) -> napi::Result<Table> {
         let schema = ipc_file_to_schema(schema_buf.to_vec()).map_err(|e| {
             napi::Error::from_reason(format!("Failed to marshal schema from JS to Rust: {}", e))
@@ -164,6 +169,9 @@ impl Connection {
                 builder = builder.storage_option(key, value);
             }
         }
+        if let Some(use_legacy_format) = use_legacy_format {
+            builder = builder.use_legacy_format(use_legacy_format);
+        }
         let tbl = builder
             .execute()
             .await

nodejs/src/lib.rs

@@ -56,6 +56,7 @@ pub enum WriteMode {
 /// Write options when creating a Table.
 #[napi(object)]
 pub struct WriteOptions {
+    /// Write mode for writing to a table.
     pub mode: Option<WriteMode>,
 }
 

nodejs/src/query.rs

@@ -15,6 +15,7 @@
 use lancedb::query::ExecutableQuery;
 use lancedb::query::Query as LanceDbQuery;
 use lancedb::query::QueryBase;
+use lancedb::query::QueryExecutionOptions;
 use lancedb::query::Select;
 use lancedb::query::VectorQuery as LanceDbVectorQuery;
 use napi::bindgen_prelude::*;
@@ -62,10 +63,21 @@ impl Query {
     }
 
     #[napi]
-    pub async fn execute(&self) -> napi::Result<RecordBatchIterator> {
-        let inner_stream = self.inner.execute().await.map_err(|e| {
-            napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
-        })?;
+    pub async fn execute(
+        &self,
+        max_batch_length: Option<u32>,
+    ) -> napi::Result<RecordBatchIterator> {
+        let mut execution_opts = QueryExecutionOptions::default();
+        if let Some(max_batch_length) = max_batch_length {
+            execution_opts.max_batch_length = max_batch_length;
+        }
+        let inner_stream = self
+            .inner
+            .execute_with_options(execution_opts)
+            .await
+            .map_err(|e| {
+                napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
+            })?;
         Ok(RecordBatchIterator::new(inner_stream))
     }
 }
@@ -125,10 +137,21 @@ impl VectorQuery {
     }
 
     #[napi]
-    pub async fn execute(&self) -> napi::Result<RecordBatchIterator> {
-        let inner_stream = self.inner.execute().await.map_err(|e| {
-            napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
-        })?;
+    pub async fn execute(
+        &self,
+        max_batch_length: Option<u32>,
+    ) -> napi::Result<RecordBatchIterator> {
+        let mut execution_opts = QueryExecutionOptions::default();
+        if let Some(max_batch_length) = max_batch_length {
+            execution_opts.max_batch_length = max_batch_length;
+        }
+        let inner_stream = self
+            .inner
+            .execute_with_options(execution_opts)
+            .await
+            .map_err(|e| {
+                napi::Error::from_reason(format!("Failed to execute query stream: {}", e))
+            })?;
         Ok(RecordBatchIterator::new(inner_stream))
     }
 }

nodejs/tsconfig.json

@@ -7,7 +7,9 @@
     "outDir": "./dist",
     "strict": true,
     "allowJs": true,
-    "resolveJsonModule": true
+    "resolveJsonModule": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true
   },
   "exclude": ["./dist/*"],
   "typedocOptions": {

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.8.0"
+current_version = "0.8.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.8.0"
+version = "0.8.2"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true

python/pyproject.toml

@@ -3,14 +3,14 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
     "deprecation",
-    "pylance==0.11.0",
+    "pylance==0.12.1",
     "ratelimiter~=1.0",
     "requests>=2.31.0",
     "retry>=0.9.2",
     "tqdm>=4.27.0",
     "pydantic>=1.10",
     "attrs>=21.3.0",
-    "semver",
+    "packaging",
     "cachetools",
     "overrides>=0.7",
 ]

python/python/lancedb/_lancedb.pyi

@@ -24,6 +24,7 @@ class Connection(object):
         mode: str,
         data: pa.RecordBatchReader,
         storage_options: Optional[Dict[str, str]] = None,
+        use_legacy_format: Optional[bool] = None,
     ) -> Table: ...
     async def create_empty_table(
         self,
@@ -31,6 +32,7 @@ class Connection(object):
         mode: str,
         schema: pa.Schema,
         storage_options: Optional[Dict[str, str]] = None,
+        use_legacy_format: Optional[bool] = None,
     ) -> Table: ...
 
 class Table:
@@ -72,7 +74,7 @@ class Query:
     def select(self, columns: Tuple[str, str]): ...
     def limit(self, limit: int): ...
     def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
-    async def execute(self) -> RecordBatchStream: ...
+    async def execute(self, max_batch_legnth: Optional[int]) -> RecordBatchStream: ...
 
 class VectorQuery:
     async def execute(self) -> RecordBatchStream: ...

python/python/lancedb/db.py

@@ -509,7 +509,7 @@ class AsyncConnection(object):
         return self._inner.__repr__()
 
     def __enter__(self):
-        self
+        return self
 
     def __exit__(self, *_):
         self.close()
@@ -558,6 +558,8 @@ class AsyncConnection(object):
         on_bad_vectors: Optional[str] = None,
         fill_value: Optional[float] = None,
         storage_options: Optional[Dict[str, str]] = None,
+        *,
+        use_legacy_format: Optional[bool] = None,
     ) -> AsyncTable:
         """Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
 
@@ -600,6 +602,9 @@ class AsyncConnection(object):
             connection will be inherited by the table, but can be overridden here.
             See available options at
             https://lancedb.github.io/lancedb/guides/storage/
+        use_legacy_format: bool, optional, default True
+            If True, use the legacy format for the table. If False, use the new format.
+            The default is True while the new format is in beta.
 
 
         Returns
@@ -761,7 +766,11 @@ class AsyncConnection(object):
 
         if data is None:
             new_table = await self._inner.create_empty_table(
-                name, mode, schema, storage_options=storage_options
+                name,
+                mode,
+                schema,
+                storage_options=storage_options,
+                use_legacy_format=use_legacy_format,
             )
         else:
             data = data_to_reader(data, schema)
@@ -770,6 +779,7 @@ class AsyncConnection(object):
                 mode,
                 data,
                 storage_options=storage_options,
+                use_legacy_format=use_legacy_format,
             )
 
         return AsyncTable(new_table)
@@ -779,7 +789,7 @@ class AsyncConnection(object):
         name: str,
         storage_options: Optional[Dict[str, str]] = None,
         index_cache_size: Optional[int] = None,
-    ) -> Table:
+    ) -> AsyncTable:
         """Open a Lance Table in the database.
 
         Parameters

python/python/lancedb/embeddings/base.py

@@ -153,7 +153,7 @@ class TextEmbeddingFunction(EmbeddingFunction):
 
     @abstractmethod
     def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
+        self, texts: Union[List[str], np.ndarray], *args, **kwargs
     ) -> List[np.array]:
         """
         Generate the embeddings for the given texts

python/python/lancedb/embeddings/bedrock.py

@@ -73,8 +73,10 @@ class BedRockText(TextEmbeddingFunction):
     assumed_role: Union[str, None] = None
     profile_name: Union[str, None] = None
     role_session_name: str = "lancedb-embeddings"
+    source_input_type: str = "search_document"
+    query_input_type: str = "search_query"
 
-    if PYDANTIC_VERSION < (2, 0):  # Pydantic 1.x compat
+    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
 
         class Config:
             keep_untouched = (cached_property,)
@@ -87,21 +89,29 @@ class BedRockText(TextEmbeddingFunction):
         # TODO: fix hardcoding
         if self.name == "amazon.titan-embed-text-v1":
             return 1536
-        elif self.name in {"cohere.embed-english-v3", "cohere.embed-multilingual-v3"}:
+        elif self.name in [
+            "amazon.titan-embed-text-v2:0",
+            "cohere.embed-english-v3",
+            "cohere.embed-multilingual-v3",
+        ]:
+            # TODO: "amazon.titan-embed-text-v2:0" model supports dynamic ndims
             return 1024
         else:
-            raise ValueError(f"Unknown model name: {self.name}")
+            raise ValueError(f"Model {self.name} not supported")
 
     def compute_query_embeddings(
         self, query: str, *args, **kwargs
     ) -> List[List[float]]:
-        return self.compute_source_embeddings(query)
+        return self.compute_source_embeddings(query, input_type=self.query_input_type)
 
     def compute_source_embeddings(
         self, texts: TEXT, *args, **kwargs
     ) -> List[List[float]]:
         texts = self.sanitize_input(texts)
-        return self.generate_embeddings(texts)
+        # assume source input type if not passed by `compute_query_embeddings`
+        kwargs["input_type"] = kwargs.get("input_type") or self.source_input_type
+
+        return self.generate_embeddings(texts, **kwargs)
 
     def generate_embeddings(
         self, texts: Union[List[str], np.ndarray], *args, **kwargs
@@ -121,11 +131,11 @@ class BedRockText(TextEmbeddingFunction):
         """
         results = []
         for text in texts:
-            response = self._generate_embedding(text)
+            response = self._generate_embedding(text, *args, **kwargs)
             results.append(response)
         return results
 
-    def _generate_embedding(self, text: str) -> List[float]:
+    def _generate_embedding(self, text: str, *args, **kwargs) -> List[float]:
         """
         Get the embeddings for the given texts
 
@@ -141,14 +151,12 @@ class BedRockText(TextEmbeddingFunction):
         """
         # format input body for provider
         provider = self.name.split(".")[0]
-        _model_kwargs = {}
-        input_body = {**_model_kwargs}
+        input_body = {**kwargs}
         if provider == "cohere":
-            if "input_type" not in input_body.keys():
-                input_body["input_type"] = "search_document"
             input_body["texts"] = [text]
         else:
             # includes common provider == "amazon"
+            input_body.pop("input_type", None)
             input_body["inputText"] = text
         body = json.dumps(input_body)
 

python/python/lancedb/embeddings/cohere.py

@@ -19,7 +19,7 @@ import numpy as np
 from ..util import attempt_import_or_raise
 from .base import TextEmbeddingFunction
 from .registry import register
-from .utils import api_key_not_found_help
+from .utils import api_key_not_found_help, TEXT
 
 
 @register("cohere")
@@ -32,8 +32,36 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
     Parameters
     ----------
     name: str, default "embed-multilingual-v2.0"
-        The name of the model to use. See the Cohere documentation for
-        a list of available models.
+        The name of the model to use. List of acceptable models:
+
+            * embed-english-v3.0
+            * embed-multilingual-v3.0
+            * embed-english-light-v3.0
+            * embed-multilingual-light-v3.0
+            * embed-english-v2.0
+            * embed-english-light-v2.0
+            * embed-multilingual-v2.0
+
+    source_input_type: str, default "search_document"
+        The input type for the source column in the database
+
+    query_input_type: str, default "search_query"
+        The input type for the query column in the database
+
+    Cohere supports following input types:
+
+    | Input Type               | Description                          |
+    |-------------------------|---------------------------------------|
+    | "`search_document`"     | Used for embeddings stored in a vector|
+    |                         | database for search use-cases.        |
+    | "`search_query`"        | Used for embeddings of search queries |
+    |                         | run against a vector DB               |
+    | "`semantic_similarity`" | Specifies the given text will be used |
+    |                         | for Semantic Textual Similarity (STS) |
+    | "`classification`"      | Used for embeddings passed through a  |
+    |                         | text classifier.                      |
+    | "`clustering`"          | Used for the embeddings run through a |
+    |                         | clustering algorithm                  |
 
     Examples
     --------
@@ -61,14 +89,39 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
     """
 
     name: str = "embed-multilingual-v2.0"
+    source_input_type: str = "search_document"
+    query_input_type: str = "search_query"
     client: ClassVar = None
 
     def ndims(self):
         # TODO: fix hardcoding
-        return 768
+        if self.name in [
+            "embed-english-v3.0",
+            "embed-multilingual-v3.0",
+            "embed-english-light-v2.0",
+        ]:
+            return 1024
+        elif self.name in ["embed-english-light-v3.0", "embed-multilingual-light-v3.0"]:
+            return 384
+        elif self.name == "embed-english-v2.0":
+            return 4096
+        elif self.name == "embed-multilingual-v2.0":
+            return 768
+        else:
+            raise ValueError(f"Model {self.name} not supported")
+
+    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
+        return self.compute_source_embeddings(query, input_type=self.query_input_type)
+
+    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
+        texts = self.sanitize_input(texts)
+        input_type = (
+            kwargs.get("input_type") or self.source_input_type
+        )  # assume source input type if not passed by `compute_query_embeddings`
+        return self.generate_embeddings(texts, input_type=input_type)
 
     def generate_embeddings(
-        self, texts: Union[List[str], np.ndarray]
+        self, texts: Union[List[str], np.ndarray], *args, **kwargs
     ) -> List[np.array]:
         """
         Get the embeddings for the given texts
@@ -78,9 +131,10 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
         texts: list[str] or np.ndarray (of str)
             The texts to embed
         """
-        # TODO retry, rate limit, token limit
         self._init_client()
-        rs = CohereEmbeddingFunction.client.embed(texts=texts, model=self.name)
+        rs = CohereEmbeddingFunction.client.embed(
+            texts=texts, model=self.name, **kwargs
+        )
 
         return [emb for emb in rs.embeddings]
 

python/python/lancedb/embeddings/gemini_text.py

@@ -90,7 +90,7 @@ class GeminiText(TextEmbeddingFunction):
     query_task_type: str = "retrieval_query"
     source_task_type: str = "retrieval_document"
 
-    if PYDANTIC_VERSION < (2, 0):  # Pydantic 1.x compat
+    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
 
         class Config:
             keep_untouched = (cached_property,)

python/python/lancedb/embeddings/imagebind.py

@@ -40,7 +40,7 @@ class ImageBindEmbeddings(EmbeddingFunction):
     device: str = "cpu"
     normalize: bool = False
 
-    if PYDANTIC_VERSION < (2, 0):  # Pydantic 1.x compat
+    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
 
         class Config:
             keep_untouched = (cached_property,)

python/python/lancedb/embeddings/transformers.py

@@ -54,7 +54,7 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
         self._tokenizer = transformers.AutoTokenizer.from_pretrained(self.name)
         self._model = transformers.AutoModel.from_pretrained(self.name)
 
-    if PYDANTIC_VERSION < (2, 0):  # Pydantic 1.x compat
+    if PYDANTIC_VERSION.major < 2:  # Pydantic 1.x compat
 
         class Config:
             keep_untouched = (cached_property,)

python/python/lancedb/pydantic.py

@@ -35,13 +35,13 @@ from typing import (
 import numpy as np
 import pyarrow as pa
 import pydantic
-import semver
+from packaging.version import Version
 
-PYDANTIC_VERSION = semver.parse_version_info(pydantic.__version__)
+PYDANTIC_VERSION = Version(pydantic.__version__)
 try:
     from pydantic_core import CoreSchema, core_schema
 except ImportError:
-    if PYDANTIC_VERSION >= (2,):
+    if PYDANTIC_VERSION.major >= 2:
         raise
 
 if TYPE_CHECKING:
@@ -144,7 +144,7 @@ def Vector(
                 raise TypeError("A list of numbers or numpy.ndarray is needed")
             return cls(v)
 
-        if PYDANTIC_VERSION < (2, 0):
+        if PYDANTIC_VERSION.major < 2:
 
             @classmethod
             def __modify_schema__(cls, field_schema: Dict[str, Any]):

python/python/lancedb/query.py

@@ -1113,11 +1113,22 @@ class AsyncQueryBase(object):
         self._inner.limit(limit)
         return self
 
-    async def to_batches(self) -> AsyncRecordBatchReader:
+    async def to_batches(
+        self, *, max_batch_length: Optional[int] = None
+    ) -> AsyncRecordBatchReader:
         """
         Execute the query and return the results as an Apache Arrow RecordBatchReader.
+
+        Parameters
+        ----------
+
+        max_batch_length: Optional[int]
+            The maximum number of selected records in a single RecordBatch object.
+            If not specified, a default batch length is used.
+            It is possible for batches to be smaller than the provided length if the
+            underlying data is stored in smaller chunks.
         """
-        return AsyncRecordBatchReader(await self._inner.execute())
+        return AsyncRecordBatchReader(await self._inner.execute(max_batch_length))
 
     async def to_arrow(self) -> pa.Table:
         """

python/python/lancedb/rerankers/cohere.py

@@ -1,5 +1,5 @@
 import os
-import semver
+from packaging.version import Version
 from functools import cached_property
 from typing import Union
 
@@ -44,9 +44,8 @@ class CohereReranker(Reranker):
     def _client(self):
         cohere = attempt_import_or_raise("cohere")
         # ensure version is at least 0.5.0
-        if (
-            hasattr(cohere, "__version__")
-            and semver.compare(cohere.__version__, "5.0.0") < 0
+        if hasattr(cohere, "__version__") and Version(cohere.__version__) < Version(
+            "0.5.0"
         ):
             raise ValueError(
                 f"cohere version must be at least 0.5.0, found {cohere.__version__}"

python/python/tests/test_db.py

@@ -296,6 +296,13 @@ async def test_close(tmp_path):
         await db.table_names()
 
 
+@pytest.mark.asyncio
+async def test_context_manager(tmp_path):
+    with await lancedb.connect_async(tmp_path) as db:
+        assert db.is_open()
+    assert not db.is_open()
+
+
 @pytest.mark.asyncio
 async def test_create_mode_async(tmp_path):
     db = await lancedb.connect_async(tmp_path)
@@ -500,6 +507,52 @@ def test_empty_or_nonexistent_table(tmp_path):
     assert test.schema == test2.schema
 
 
+@pytest.mark.asyncio
+async def test_create_in_v2_mode(tmp_path):
+    def make_data():
+        for i in range(10):
+            yield pa.record_batch([pa.array([x for x in range(1024)])], names=["x"])
+
+    def make_table():
+        return pa.table([pa.array([x for x in range(10 * 1024)])], names=["x"])
+
+    schema = pa.schema([pa.field("x", pa.int64())])
+
+    db = await lancedb.connect_async(tmp_path)
+
+    # Create table in v1 mode
+    tbl = await db.create_table("test", data=make_data(), schema=schema)
+
+    async def is_in_v2_mode(tbl):
+        batches = await tbl.query().to_batches(max_batch_length=1024 * 10)
+        num_batches = 0
+        async for batch in batches:
+            num_batches += 1
+        return num_batches < 10
+
+    assert not await is_in_v2_mode(tbl)
+
+    # Create table in v2 mode
+    tbl = await db.create_table(
+        "test_v2", data=make_data(), schema=schema, use_legacy_format=False
+    )
+
+    assert await is_in_v2_mode(tbl)
+
+    # Add data (should remain in v2 mode)
+    await tbl.add(make_table())
+
+    assert await is_in_v2_mode(tbl)
+
+    # Create empty table in v2 mode and add data
+    tbl = await db.create_table(
+        "test_empty_v2", data=None, schema=schema, use_legacy_format=False
+    )
+    await tbl.add(make_table())
+
+    assert await is_in_v2_mode(tbl)
+
+
 def test_replace_index(tmp_path):
     db = lancedb.connect(uri=tmp_path)
     table = db.create_table(

python/python/tests/test_pydantic.py

@@ -178,7 +178,7 @@ def test_fixed_size_list_field():
         li: List[int]
 
     data = TestModel(vec=list(range(16)), li=[1, 2, 3])
-    if PYDANTIC_VERSION >= (2,):
+    if PYDANTIC_VERSION.major >= 2:
         assert json.loads(data.model_dump_json()) == {
             "vec": list(range(16)),
             "li": [1, 2, 3],
@@ -197,7 +197,7 @@ def test_fixed_size_list_field():
         ]
     )
 
-    if PYDANTIC_VERSION >= (2,):
+    if PYDANTIC_VERSION.major >= 2:
         json_schema = TestModel.model_json_schema()
     else:
         json_schema = TestModel.schema()

python/src/connection.rs

@@ -91,6 +91,7 @@ impl Connection {
         mode: &str,
         data: &PyAny,
         storage_options: Option<HashMap<String, String>>,
+        use_legacy_format: Option<bool>,
     ) -> PyResult<&'a PyAny> {
         let inner = self_.get_inner()?.clone();
 
@@ -103,6 +104,10 @@ impl Connection {
             builder = builder.storage_options(storage_options);
         }
 
+        if let Some(use_legacy_format) = use_legacy_format {
+            builder = builder.use_legacy_format(use_legacy_format);
+        }
+
         future_into_py(self_.py(), async move {
             let table = builder.execute().await.infer_error()?;
             Ok(Table::new(table))
@@ -115,6 +120,7 @@ impl Connection {
         mode: &str,
         schema: &PyAny,
         storage_options: Option<HashMap<String, String>>,
+        use_legacy_format: Option<bool>,
     ) -> PyResult<&'a PyAny> {
         let inner = self_.get_inner()?.clone();
 
@@ -128,6 +134,10 @@ impl Connection {
             builder = builder.storage_options(storage_options);
         }
 
+        if let Some(use_legacy_format) = use_legacy_format {
+            builder = builder.use_legacy_format(use_legacy_format);
+        }
+
         future_into_py(self_.py(), async move {
             let table = builder.execute().await.infer_error()?;
             Ok(Table::new(table))

python/src/query.rs

@@ -15,6 +15,7 @@
 use arrow::array::make_array;
 use arrow::array::ArrayData;
 use arrow::pyarrow::FromPyArrow;
+use lancedb::query::QueryExecutionOptions;
 use lancedb::query::{
     ExecutableQuery, Query as LanceDbQuery, QueryBase, Select, VectorQuery as LanceDbVectorQuery,
 };
@@ -61,10 +62,14 @@ impl Query {
         Ok(VectorQuery { inner })
     }
 
-    pub fn execute(self_: PyRef<'_, Self>) -> PyResult<&PyAny> {
+    pub fn execute(self_: PyRef<'_, Self>, max_batch_length: Option<u32>) -> PyResult<&PyAny> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
-            let inner_stream = inner.execute().await.infer_error()?;
+            let mut opts = QueryExecutionOptions::default();
+            if let Some(max_batch_length) = max_batch_length {
+                opts.max_batch_length = max_batch_length;
+            }
+            let inner_stream = inner.execute_with_options(opts).await.infer_error()?;
             Ok(RecordBatchStream::new(inner_stream))
         })
     }
@@ -115,10 +120,14 @@ impl VectorQuery {
         self.inner = self.inner.clone().bypass_vector_index()
     }
 
-    pub fn execute(self_: PyRef<'_, Self>) -> PyResult<&PyAny> {
+    pub fn execute(self_: PyRef<'_, Self>, max_batch_length: Option<u32>) -> PyResult<&PyAny> {
         let inner = self_.inner.clone();
         future_into_py(self_.py(), async move {
-            let inner_stream = inner.execute().await.infer_error()?;
+            let mut opts = QueryExecutionOptions::default();
+            if let Some(max_batch_length) = max_batch_length {
+                opts.max_batch_length = max_batch_length;
+            }
+            let inner_stream = inner.execute_with_options(opts).await.infer_error()?;
             Ok(RecordBatchStream::new(inner_stream))
         })
     }

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-node"
-version = "0.5.0"
+version = "0.5.1"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.5.0"
+version = "0.5.1"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -19,11 +19,13 @@ arrow-ord = { workspace = true }
 arrow-cast = { workspace = true }
 arrow-ipc.workspace = true
 chrono = { workspace = true }
+datafusion-physical-plan.workspace = true
 object_store = { workspace = true }
 snafu = { workspace = true }
 half = { workspace = true }
 lazy_static.workspace = true
 lance = { workspace = true }
+lance-datafusion.workspace = true
 lance-index = { workspace = true }
 lance-linalg = { workspace = true }
 lance-testing = { workspace = true }
@@ -38,10 +40,12 @@ url.workspace = true
 regex.workspace = true
 serde = { version = "^1" }
 serde_json = { version = "1" }
+async-openai = { version = "0.20.0", optional = true }
+serde_with = { version = "3.8.1" }
 # For remote feature
 reqwest = { version = "0.11.24", features = ["gzip", "json"], optional = true }
 polars-arrow = { version = ">=0.37,<0.40.0", optional = true }
-polars = { version = ">=0.37,<0.40.0", optional = true}
+polars = { version = ">=0.37,<0.40.0", optional = true }
 
 [dev-dependencies]
 tempfile = "3.5.0"
@@ -61,4 +65,10 @@ default = []
 remote = ["dep:reqwest"]
 fp16kernels = ["lance-linalg/fp16kernels"]
 s3-test = []
+openai = ["dep:async-openai", "dep:reqwest"]
 polars = ["dep:polars-arrow", "dep:polars"]
+
+
+[[example]]
+name = "openai"
+required-features = ["openai"]

rust/lancedb/examples/openai.rs

@@ -0,0 +1,82 @@
+use std::{iter::once, sync::Arc};
+
+use arrow_array::{Float64Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use futures::StreamExt;
+use lancedb::{
+    arrow::IntoArrow,
+    connect,
+    embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},
+    query::{ExecutableQuery, QueryBase},
+    Result,
+};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let tempdir = tempfile::tempdir().unwrap();
+    let tempdir = tempdir.path().to_str().unwrap();
+    let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set");
+    let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(
+        api_key,
+        "text-embedding-3-large",
+    )?);
+
+    let db = connect(tempdir).execute().await?;
+    db.embedding_registry()
+        .register("openai", embedding.clone())?;
+
+    let table = db
+        .create_table("vectors", make_data())
+        .add_embedding(EmbeddingDefinition::new(
+            "text",
+            "openai",
+            Some("embeddings"),
+        ))?
+        .execute()
+        .await?;
+
+    // there is no equivalent to '.search(<query>)' yet
+    let query = Arc::new(StringArray::from_iter_values(once("something warm")));
+    let query_vector = embedding.compute_query_embeddings(query)?;
+    let mut results = table
+        .vector_search(query_vector)?
+        .limit(1)
+        .execute()
+        .await?;
+
+    let rb = results.next().await.unwrap()?;
+    let out = rb
+        .column_by_name("text")
+        .unwrap()
+        .as_any()
+        .downcast_ref::<StringArray>()
+        .unwrap();
+    let text = out.iter().next().unwrap().unwrap();
+    println!("Closest match: {}", text);
+
+    Ok(())
+}
+
+fn make_data() -> impl IntoArrow {
+    let schema = Schema::new(vec![
+        Field::new("id", DataType::Int32, true),
+        Field::new("text", DataType::Utf8, false),
+        Field::new("price", DataType::Float64, false),
+    ]);
+
+    let id = Int32Array::from(vec![1, 2, 3, 4]);
+    let text = StringArray::from_iter_values(vec![
+        "Black T-Shirt",
+        "Leather Jacket",
+        "Winter Parka",
+        "Hooded Sweatshirt",
+    ]);
+    let price = Float64Array::from(vec![10.0, 50.0, 100.0, 30.0]);
+    let schema = Arc::new(schema);
+    let rb = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(id), Arc::new(text), Arc::new(price)],
+    )
+    .unwrap();
+    Box::new(RecordBatchIterator::new(vec![Ok(rb)], schema))
+}

rust/lancedb/src/connection.rs

@@ -140,6 +140,7 @@ pub struct CreateTableBuilder<const HAS_DATA: bool, T: IntoArrow> {
     pub(crate) write_options: WriteOptions,
     pub(crate) table_definition: Option<TableDefinition>,
     pub(crate) embeddings: Vec<(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)>,
+    pub(crate) use_legacy_format: bool,
 }
 
 // Builder methods that only apply when we have initial data
@@ -153,6 +154,7 @@ impl<T: IntoArrow> CreateTableBuilder<true, T> {
             write_options: WriteOptions::default(),
             table_definition: None,
             embeddings: Vec::new(),
+            use_legacy_format: true,
         }
     }
 
@@ -184,6 +186,7 @@ impl<T: IntoArrow> CreateTableBuilder<true, T> {
             mode: self.mode,
             write_options: self.write_options,
             embeddings: self.embeddings,
+            use_legacy_format: self.use_legacy_format,
         };
         Ok((data, builder))
     }
@@ -217,6 +220,7 @@ impl CreateTableBuilder<false, NoData> {
             mode: CreateTableMode::default(),
             write_options: WriteOptions::default(),
             embeddings: Vec::new(),
+            use_legacy_format: false,
         }
     }
 
@@ -278,6 +282,20 @@ impl<const HAS_DATA: bool, T: IntoArrow> CreateTableBuilder<HAS_DATA, T> {
         }
         self
     }
+
+    /// Set to true to use the v1 format for data files
+    ///
+    /// This is currently defaulted to true and can be set to false to opt-in
+    /// to the new format.  This should only be used for experimentation and
+    /// evaluation.  The new format is still in beta and may change in ways that
+    /// are not backwards compatible.
+    ///
+    /// Once the new format is stable, the default will change to `false` for
+    /// several releases and then eventually this option will be removed.
+    pub fn use_legacy_format(mut self, use_legacy_format: bool) -> Self {
+        self.use_legacy_format = use_legacy_format;
+        self
+    }
 }
 
 #[derive(Clone, Debug)]
@@ -943,6 +961,7 @@ impl ConnectionInternal for Database {
         if matches!(&options.mode, CreateTableMode::Overwrite) {
             write_params.mode = WriteMode::Overwrite;
         }
+        write_params.use_legacy_format = options.use_legacy_format;
 
         match NativeTable::create(
             &table_uri,
@@ -1040,8 +1059,12 @@ impl ConnectionInternal for Database {
 #[cfg(test)]
 mod tests {
     use arrow_schema::{DataType, Field, Schema};
+    use futures::TryStreamExt;
+    use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
     use tempfile::tempdir;
 
+    use crate::query::{ExecutableQuery, QueryExecutionOptions};
+
     use super::*;
 
     #[tokio::test]
@@ -1146,6 +1169,58 @@ mod tests {
         assert_eq!(tables, vec!["table1".to_owned()]);
     }
 
+    fn make_data() -> impl RecordBatchReader + Send + 'static {
+        let id = Box::new(IncrementingInt32::new().named("id".to_string()));
+        BatchGenerator::new().col(id).batches(10, 2000)
+    }
+
+    #[tokio::test]
+    async fn test_create_table_v2() {
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+        let db = connect(uri).execute().await.unwrap();
+
+        let tbl = db
+            .create_table("v1_test", make_data())
+            .execute()
+            .await
+            .unwrap();
+
+        // In v1 the row group size will trump max_batch_length
+        let batches = tbl
+            .query()
+            .execute_with_options(QueryExecutionOptions {
+                max_batch_length: 50000,
+            })
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(batches.len(), 20);
+
+        let tbl = db
+            .create_table("v2_test", make_data())
+            .use_legacy_format(false)
+            .execute()
+            .await
+            .unwrap();
+
+        // In v2 the page size is much bigger than 50k so we should get a single batch
+        let batches = tbl
+            .query()
+            .execute_with_options(QueryExecutionOptions {
+                max_batch_length: 50000,
+            })
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        assert_eq!(batches.len(), 1);
+    }
+
     #[tokio::test]
     async fn drop_table() {
         let tmp_dir = tempdir().unwrap();

rust/lancedb/src/embeddings.rs

@@ -11,6 +11,8 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#[cfg(feature = "openai")]
+pub mod openai;
 
 use lance::arrow::RecordBatchExt;
 use std::{
@@ -51,8 +53,10 @@ pub trait EmbeddingFunction: std::fmt::Debug + Send + Sync {
     /// The type of the output data
     /// This should **always** match the output of the `embed` function
     fn dest_type(&self) -> Result<Cow<DataType>>;
-    /// Embed the input
-    fn embed(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>>;
+    /// Compute the embeddings for the source column in the database
+    fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>>;
+    /// Compute the embeddings for a given user query
+    fn compute_query_embeddings(&self, input: Arc<dyn Array>) -> Result<Arc<dyn Array>>;
 }
 
 /// Defines an embedding from input data into a lower-dimensional space
@@ -266,7 +270,7 @@ impl<R: RecordBatchReader> Iterator for WithEmbeddings<R> {
                 // todo: parallelize this
                 for (fld, func) in self.embeddings.iter() {
                     let src_column = batch.column_by_name(&fld.source_column).unwrap();
-                    let embedding = match func.embed(src_column.clone()) {
+                    let embedding = match func.compute_source_embeddings(src_column.clone()) {
                         Ok(embedding) => embedding,
                         Err(e) => {
                             return Some(Err(arrow_schema::ArrowError::ComputeError(format!(

rust/lancedb/src/embeddings/openai.rs

@@ -0,0 +1,257 @@
+use std::{borrow::Cow, fmt::Formatter, str::FromStr, sync::Arc};
+
+use arrow::array::{AsArray, Float32Builder};
+use arrow_array::{Array, ArrayRef, FixedSizeListArray, Float32Array};
+use arrow_data::ArrayData;
+use arrow_schema::DataType;
+use async_openai::{
+    config::OpenAIConfig,
+    types::{CreateEmbeddingRequest, Embedding, EmbeddingInput, EncodingFormat},
+    Client,
+};
+use tokio::{runtime::Handle, task};
+
+use crate::{Error, Result};
+
+use super::EmbeddingFunction;
+
+#[derive(Debug)]
+pub enum EmbeddingModel {
+    TextEmbeddingAda002,
+    TextEmbedding3Small,
+    TextEmbedding3Large,
+}
+
+impl EmbeddingModel {
+    fn ndims(&self) -> usize {
+        match self {
+            Self::TextEmbeddingAda002 => 1536,
+            Self::TextEmbedding3Small => 1536,
+            Self::TextEmbedding3Large => 3072,
+        }
+    }
+}
+
+impl FromStr for EmbeddingModel {
+    type Err = Error;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s {
+            "text-embedding-ada-002" => Ok(Self::TextEmbeddingAda002),
+            "text-embedding-3-small" => Ok(Self::TextEmbedding3Small),
+            "text-embedding-3-large" => Ok(Self::TextEmbedding3Large),
+            _ => Err(Error::InvalidInput {
+                message: "Invalid input. Available models are: 'text-embedding-3-small', 'text-embedding-ada-002', 'text-embedding-3-large' ".to_string()
+            }),
+        }
+    }
+}
+
+impl std::fmt::Display for EmbeddingModel {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        match self {
+            Self::TextEmbeddingAda002 => write!(f, "text-embedding-ada-002"),
+            Self::TextEmbedding3Small => write!(f, "text-embedding-3-small"),
+            Self::TextEmbedding3Large => write!(f, "text-embedding-3-large"),
+        }
+    }
+}
+
+impl TryFrom<&str> for EmbeddingModel {
+    type Error = Error;
+
+    fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
+        value.parse()
+    }
+}
+
+pub struct OpenAIEmbeddingFunction {
+    model: EmbeddingModel,
+    api_key: String,
+    api_base: Option<String>,
+    org_id: Option<String>,
+}
+
+impl std::fmt::Debug for OpenAIEmbeddingFunction {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
+        // let's be safe and not print the full API key
+        let creds_display = if self.api_key.len() > 6 {
+            format!(
+                "{}***{}",
+                &self.api_key[0..2],
+                &self.api_key[self.api_key.len() - 4..]
+            )
+        } else {
+            "[INVALID]".to_string()
+        };
+
+        f.debug_struct("OpenAI")
+            .field("model", &self.model)
+            .field("api_key", &creds_display)
+            .field("api_base", &self.api_base)
+            .field("org_id", &self.org_id)
+            .finish()
+    }
+}
+
+impl OpenAIEmbeddingFunction {
+    /// Create a new OpenAIEmbeddingFunction
+    pub fn new<A: Into<String>>(api_key: A) -> Self {
+        Self::new_impl(api_key.into(), EmbeddingModel::TextEmbeddingAda002)
+    }
+
+    pub fn new_with_model<A: Into<String>, M: TryInto<EmbeddingModel>>(
+        api_key: A,
+        model: M,
+    ) -> crate::Result<Self>
+    where
+        M::Error: Into<crate::Error>,
+    {
+        Ok(Self::new_impl(
+            api_key.into(),
+            model.try_into().map_err(|e| e.into())?,
+        ))
+    }
+
+    /// concrete implementation to reduce monomorphization
+    fn new_impl(api_key: String, model: EmbeddingModel) -> Self {
+        Self {
+            model,
+            api_key,
+            api_base: None,
+            org_id: None,
+        }
+    }
+
+    /// To use a API base url different from default "https://api.openai.com/v1"
+    pub fn api_base<S: Into<String>>(mut self, api_base: S) -> Self {
+        self.api_base = Some(api_base.into());
+        self
+    }
+
+    /// To use a different OpenAI organization id other than default
+    pub fn org_id<S: Into<String>>(mut self, org_id: S) -> Self {
+        self.org_id = Some(org_id.into());
+        self
+    }
+}
+
+impl EmbeddingFunction for OpenAIEmbeddingFunction {
+    fn name(&self) -> &str {
+        "openai"
+    }
+
+    fn source_type(&self) -> Result<Cow<DataType>> {
+        Ok(Cow::Owned(DataType::Utf8))
+    }
+
+    fn dest_type(&self) -> Result<Cow<DataType>> {
+        let n_dims = self.model.ndims();
+        Ok(Cow::Owned(DataType::new_fixed_size_list(
+            DataType::Float32,
+            n_dims as i32,
+            false,
+        )))
+    }
+
+    fn compute_source_embeddings(&self, source: ArrayRef) -> crate::Result<ArrayRef> {
+        let len = source.len();
+        let n_dims = self.model.ndims();
+        let inner = self.compute_inner(source)?;
+
+        let fsl = DataType::new_fixed_size_list(DataType::Float32, n_dims as i32, false);
+
+        // We can't use the FixedSizeListBuilder here because it always adds a null bitmap
+        // and we want to explicitly work with non-nullable arrays.
+        let array_data = ArrayData::builder(fsl)
+            .len(len)
+            .add_child_data(inner.into_data())
+            .build()?;
+
+        Ok(Arc::new(FixedSizeListArray::from(array_data)))
+    }
+
+    fn compute_query_embeddings(&self, input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+        let arr = self.compute_inner(input)?;
+        Ok(Arc::new(arr))
+    }
+}
+impl OpenAIEmbeddingFunction {
+    fn compute_inner(&self, source: Arc<dyn Array>) -> Result<Float32Array> {
+        // OpenAI only supports non-nullable string arrays
+        if source.is_nullable() {
+            return Err(crate::Error::InvalidInput {
+                message: "Expected non-nullable data type".to_string(),
+            });
+        }
+
+        // OpenAI only supports string arrays
+        if !matches!(source.data_type(), DataType::Utf8 | DataType::LargeUtf8) {
+            return Err(crate::Error::InvalidInput {
+                message: "Expected Utf8 data type".to_string(),
+            });
+        };
+
+        let mut creds = OpenAIConfig::new().with_api_key(self.api_key.clone());
+
+        if let Some(api_base) = &self.api_base {
+            creds = creds.with_api_base(api_base.clone());
+        }
+        if let Some(org_id) = &self.org_id {
+            creds = creds.with_org_id(org_id.clone());
+        }
+
+        let input = match source.data_type() {
+            DataType::Utf8 => {
+                let array = source
+                    .as_string::<i32>()
+                    .into_iter()
+                    .map(|s| {
+                        s.expect("we already asserted that the array is non-nullable")
+                            .to_string()
+                    })
+                    .collect::<Vec<String>>();
+                EmbeddingInput::StringArray(array)
+            }
+            DataType::LargeUtf8 => {
+                let array = source
+                    .as_string::<i64>()
+                    .into_iter()
+                    .map(|s| {
+                        s.expect("we already asserted that the array is non-nullable")
+                            .to_string()
+                    })
+                    .collect::<Vec<String>>();
+                EmbeddingInput::StringArray(array)
+            }
+            _ => unreachable!("This should not happen. We already checked the data type."),
+        };
+
+        let client = Client::with_config(creds);
+        let embed = client.embeddings();
+        let req = CreateEmbeddingRequest {
+            model: self.model.to_string(),
+            input,
+            encoding_format: Some(EncodingFormat::Float),
+            user: None,
+            dimensions: None,
+        };
+
+        // TODO: request batching and retry logic
+        task::block_in_place(move || {
+            Handle::current().block_on(async {
+                let mut builder = Float32Builder::new();
+
+                let res = embed.create(req).await.map_err(|e| crate::Error::Runtime {
+                    message: format!("OpenAI embed request failed: {e}"),
+                })?;
+
+                for Embedding { embedding, .. } in res.data.iter() {
+                    builder.append_slice(embedding);
+                }
+
+                Ok(builder.finish())
+            })
+        })
+    }
+}

rust/lancedb/src/index.rs

@@ -14,11 +14,14 @@
 
 use std::sync::Arc;
 
+use serde::Deserialize;
+use serde_with::skip_serializing_none;
+
 use crate::{table::TableInternal, Result};
 
 use self::{
     scalar::BTreeIndexBuilder,
-    vector::{IvfHnswSqIndexBuilder, IvfPqIndexBuilder},
+    vector::{IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder},
 };
 
 pub mod scalar;
@@ -28,6 +31,7 @@ pub enum Index {
     Auto,
     BTree(BTreeIndexBuilder),
     IvfPq(IvfPqIndexBuilder),
+    IvfHnswPq(IvfHnswPqIndexBuilder),
     IvfHnswSq(IvfHnswSqIndexBuilder),
 }
 
@@ -69,6 +73,7 @@ impl IndexBuilder {
 #[derive(Debug, Clone, PartialEq)]
 pub enum IndexType {
     IvfPq,
+    IvfHnswPq,
     IvfHnswSq,
     BTree,
 }
@@ -83,3 +88,19 @@ pub struct IndexConfig {
     /// be more columns to represent composite indices.
     pub columns: Vec<String>,
 }
+
+#[skip_serializing_none]
+#[derive(Debug, Deserialize)]
+pub struct IndexMetadata {
+    pub metric_type: Option<String>,
+    pub index_type: Option<String>,
+}
+
+#[skip_serializing_none]
+#[derive(Debug, Deserialize)]
+pub struct IndexStatistics {
+    pub num_indexed_rows: usize,
+    pub num_unindexed_rows: usize,
+    pub index_type: Option<String>,
+    pub indices: Vec<IndexMetadata>,
+}

rust/lancedb/src/index/vector.rs

@@ -19,8 +19,6 @@
 //! values
 use std::cmp::max;
 
-use serde::Deserialize;
-
 use lance::table::format::{Index, Manifest};
 
 use crate::DistanceType;
@@ -46,18 +44,118 @@ impl VectorIndex {
     }
 }
 
-#[derive(Debug, Deserialize)]
-pub struct VectorIndexMetadata {
-    pub metric_type: String,
-    pub index_type: String,
+macro_rules! impl_distance_type_setter {
+    () => {
+        /// [DistanceType] to use to build the index.
+        ///
+        /// Default value is [DistanceType::L2].
+        ///
+        /// This is used when training the index to calculate the IVF partitions (vectors are
+        /// grouped in partitions with similar vectors according to this distance type) and to
+        /// calculate a subvector's code during quantization.
+        ///
+        /// The metric type used to train an index MUST match the metric type used to search the
+        /// index.  Failure to do so will yield inaccurate results.
+        pub fn distance_type(mut self, distance_type: DistanceType) -> Self {
+            self.distance_type = distance_type;
+            self
+        }
+    };
 }
 
-#[derive(Debug, Deserialize)]
-pub struct VectorIndexStatistics {
-    pub num_indexed_rows: usize,
-    pub num_unindexed_rows: usize,
-    pub index_type: String,
-    pub indices: Vec<VectorIndexMetadata>,
+macro_rules! impl_ivf_params_setter {
+    () => {
+        /// The number of IVF partitions to create.
+        ///
+        /// This value should generally scale with the number of rows in the dataset.  By default
+        /// the number of partitions is the square root of the number of rows.
+        ///
+        /// If this value is too large then the first part of the search (picking the right partition)
+        /// will be slow.  If this value is too small then the second part of the search (searching
+        /// within a partition) will be slow.
+        pub fn num_partitions(mut self, num_partitions: u32) -> Self {
+            self.num_partitions = Some(num_partitions);
+            self
+        }
+
+        /// The rate used to calculate the number of training vectors for kmeans.
+        ///
+        /// When an IVF index is trained, we need to calculate partitions.  These are groups
+        /// of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+        ///
+        /// Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+        /// random sample of the data.  This parameter controls the size of the sample.  The total
+        /// number of vectors used to train the index is `sample_rate * num_partitions`.
+        ///
+        /// Increasing this value might improve the quality of the index but in most cases the
+        /// default should be sufficient.
+        ///
+        /// The default value is 256.
+        pub fn sample_rate(mut self, sample_rate: u32) -> Self {
+            self.sample_rate = sample_rate;
+            self
+        }
+
+        /// Max iterations to train kmeans.
+        ///
+        /// When training an IVF index we use kmeans to calculate the partitions.  This parameter
+        /// controls how many iterations of kmeans to run.
+        ///
+        /// Increasing this might improve the quality of the index but in most cases the parameter
+        /// is unused because kmeans will converge with fewer iterations.  The parameter is only
+        /// used in cases where kmeans does not appear to converge.  In those cases it is unlikely
+        /// that setting this larger will lead to the index converging anyways.
+        ///
+        /// The default value is 50.
+        pub fn max_iterations(mut self, max_iterations: u32) -> Self {
+            self.max_iterations = max_iterations;
+            self
+        }
+    };
+}
+
+macro_rules! impl_pq_params_setter {
+    () => {
+        /// Number of sub-vectors of PQ.
+        ///
+        /// This value controls how much the vector is compressed during the quantization step.
+        /// The more sub vectors there are the less the vector is compressed.  The default is
+        /// the dimension of the vector divided by 16.  If the dimension is not evenly divisible
+        /// by 16 we use the dimension divded by 8.
+        ///
+        /// The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
+        /// us to use efficient SIMD instructions.
+        ///
+        /// If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
+        /// will likely result in poor performance.
+        pub fn num_sub_vectors(mut self, num_sub_vectors: u32) -> Self {
+            self.num_sub_vectors = Some(num_sub_vectors);
+            self
+        }
+    };
+}
+
+macro_rules! impl_hnsw_params_setter {
+    () => {
+        /// The number of neighbors to select for each vector in the HNSW graph.
+        /// This value controls the tradeoff between search speed and accuracy.
+        /// The higher the value the more accurate the search but the slower it will be.
+        /// The default value is 20.
+        pub fn num_edges(mut self, m: u32) -> Self {
+            self.m = m;
+            self
+        }
+
+        /// The number of candidates to evaluate during the construction of the HNSW graph.
+        /// This value controls the tradeoff between build speed and accuracy.
+        /// The higher the value the more accurate the build but the slower it will be.
+        /// This value should be set to a value that is not less than `ef` in the search phase.
+        /// The default value is 300.
+        pub fn ef_construction(mut self, ef_construction: u32) -> Self {
+            self.ef_construction = ef_construction;
+            self
+        }
+    };
 }
 
 /// Builder for an IVF PQ index.
@@ -106,84 +204,9 @@ impl Default for IvfPqIndexBuilder {
 }
 
 impl IvfPqIndexBuilder {
-    /// [DistanceType] to use to build the index.
-    ///
-    /// Default value is [DistanceType::L2].
-    ///
-    /// This is used when training the index to calculate the IVF partitions (vectors are
-    /// grouped in partitions with similar vectors according to this distance type) and to
-    /// calculate a subvector's code during quantization.
-    ///
-    /// The metric type used to train an index MUST match the metric type used to search the
-    /// index.  Failure to do so will yield inaccurate results.
-    pub fn distance_type(mut self, distance_type: DistanceType) -> Self {
-        self.distance_type = distance_type;
-        self
-    }
-
-    /// The number of IVF partitions to create.
-    ///
-    /// This value should generally scale with the number of rows in the dataset.  By default
-    /// the number of partitions is the square root of the number of rows.
-    ///
-    /// If this value is too large then the first part of the search (picking the right partition)
-    /// will be slow.  If this value is too small then the second part of the search (searching
-    /// within a partition) will be slow.
-    pub fn num_partitions(mut self, num_partitions: u32) -> Self {
-        self.num_partitions = Some(num_partitions);
-        self
-    }
-
-    /// Number of sub-vectors of PQ.
-    ///
-    /// This value controls how much the vector is compressed during the quantization step.
-    /// The more sub vectors there are the less the vector is compressed.  The default is
-    /// the dimension of the vector divided by 16.  If the dimension is not evenly divisible
-    /// by 16 we use the dimension divded by 8.
-    ///
-    /// The above two cases are highly preferred.  Having 8 or 16 values per subvector allows
-    /// us to use efficient SIMD instructions.
-    ///
-    /// If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
-    /// will likely result in poor performance.
-    pub fn num_sub_vectors(mut self, num_sub_vectors: u32) -> Self {
-        self.num_sub_vectors = Some(num_sub_vectors);
-        self
-    }
-
-    /// The rate used to calculate the number of training vectors for kmeans.
-    ///
-    /// When an IVF PQ index is trained, we need to calculate partitions.  These are groups
-    /// of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
-    ///
-    /// Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
-    /// random sample of the data.  This parameter controls the size of the sample.  The total
-    /// number of vectors used to train the index is `sample_rate * num_partitions`.
-    ///
-    /// Increasing this value might improve the quality of the index but in most cases the
-    /// default should be sufficient.
-    ///
-    /// The default value is 256.
-    pub fn sample_rate(mut self, sample_rate: u32) -> Self {
-        self.sample_rate = sample_rate;
-        self
-    }
-
-    /// Max iterations to train kmeans.
-    ///
-    /// When training an IVF PQ index we use kmeans to calculate the partitions.  This parameter
-    /// controls how many iterations of kmeans to run.
-    ///
-    /// Increasing this might improve the quality of the index but in most cases the parameter
-    /// is unused because kmeans will converge with fewer iterations.  The parameter is only
-    /// used in cases where kmeans does not appear to converge.  In those cases it is unlikely
-    /// that setting this larger will lead to the index converging anyways.
-    ///
-    /// The default value is 50.
-    pub fn max_iterations(mut self, max_iterations: u32) -> Self {
-        self.max_iterations = max_iterations;
-        self
-    }
+    impl_distance_type_setter!();
+    impl_ivf_params_setter!();
+    impl_pq_params_setter!();
 }
 
 pub(crate) fn suggested_num_partitions(rows: usize) -> u32 {
@@ -206,6 +229,51 @@ pub(crate) fn suggested_num_sub_vectors(dim: u32) -> u32 {
     }
 }
 
+/// Builder for an IVF HNSW PQ index.
+///
+/// This index is a combination of IVF and HNSW.
+/// The IVF part is the same as the IVF PQ index.
+/// For each IVF partition, this builds a HNSW graph, the graph is used to
+/// quickly find the closest vectors to a query vector.
+///
+/// The PQ (product quantizer) is used to compress the vectors as the same as IVF PQ.
+#[derive(Debug, Clone)]
+pub struct IvfHnswPqIndexBuilder {
+    // IVF
+    pub(crate) distance_type: DistanceType,
+    pub(crate) num_partitions: Option<u32>,
+    pub(crate) sample_rate: u32,
+    pub(crate) max_iterations: u32,
+
+    // HNSW
+    pub(crate) m: u32,
+    pub(crate) ef_construction: u32,
+
+    // PQ
+    pub(crate) num_sub_vectors: Option<u32>,
+}
+
+impl Default for IvfHnswPqIndexBuilder {
+    fn default() -> Self {
+        Self {
+            distance_type: DistanceType::L2,
+            num_partitions: None,
+            num_sub_vectors: None,
+            sample_rate: 256,
+            max_iterations: 50,
+            m: 20,
+            ef_construction: 300,
+        }
+    }
+}
+
+impl IvfHnswPqIndexBuilder {
+    impl_distance_type_setter!();
+    impl_ivf_params_setter!();
+    impl_hnsw_params_setter!();
+    impl_pq_params_setter!();
+}
+
 /// Builder for an IVF_HNSW_SQ index.
 ///
 /// This index is a combination of IVF and HNSW.
@@ -244,85 +312,7 @@ impl Default for IvfHnswSqIndexBuilder {
 }
 
 impl IvfHnswSqIndexBuilder {
-    /// [DistanceType] to use to build the index.
-    ///
-    /// Default value is [DistanceType::L2].
-    ///
-    /// This is used when training the index to calculate the IVF partitions (vectors are
-    /// grouped in partitions with similar vectors according to this distance type)
-    ///
-    /// The metric type used to train an index MUST match the metric type used to search the
-    /// index.  Failure to do so will yield inaccurate results.
-    ///
-    /// Now IVF_HNSW_SQ only supports L2 and Cosine distance types.
-    pub fn distance_type(mut self, distance_type: DistanceType) -> Self {
-        self.distance_type = distance_type;
-        self
-    }
-
-    /// The number of IVF partitions to create.
-    ///
-    /// This value should generally scale with the number of rows in the dataset.  By default
-    /// the number of partitions is the square root of the number of rows.
-    ///
-    /// If this value is too large then the first part of the search (picking the right partition)
-    /// will be slow.  If this value is too small then the second part of the search (searching
-    /// within a partition) will be slow.
-    pub fn num_partitions(mut self, num_partitions: u32) -> Self {
-        self.num_partitions = Some(num_partitions);
-        self
-    }
-
-    /// The rate used to calculate the number of training vectors for kmeans and SQ.
-    ///
-    /// When an IVF_HNSW_SQ index is trained, we need to calculate partitions and min/max value of vectors.  These are groups
-    /// of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
-    ///
-    /// Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
-    /// random sample of the data.  This parameter controls the size of the sample.  The total
-    /// number of vectors used to train the IVF is `sample_rate * num_partitions`.
-    ///
-    /// The total number of vectors used to train the SQ is `sample_rate * 2^{num_bits}`.
-    ///
-    /// Increasing this value might improve the quality of the index but in most cases the
-    /// default should be sufficient.
-    ///
-    /// The default value is 256.
-    pub fn sample_rate(mut self, sample_rate: u32) -> Self {
-        self.sample_rate = sample_rate;
-        self
-    }
-
-    /// Max iterations to train kmeans.
-    ///
-    /// When training an IVF index we use kmeans to calculate the partitions.  This parameter
-    /// controls how many iterations of kmeans to run.
-    ///
-    /// Increasing this might improve the quality of the index but in most cases the parameter
-    /// is unused because kmeans will converge with fewer iterations.  The parameter is only
-    /// used in cases where kmeans does not appear to converge.  In those cases it is unlikely
-    /// that setting this larger will lead to the index converging anyways.
-    ///
-    /// The default value is 50.
-    pub fn max_iterations(mut self, max_iterations: u32) -> Self {
-        self.max_iterations = max_iterations;
-        self
-    }
-
-    /// The number of neighbors to select for each vector in the HNSW graph.
-    /// Bumping this number will increase the recall of the search but also increase the build/search time.
-    /// The default value is 20.
-    pub fn m(mut self, m: u32) -> Self {
-        self.m = m;
-        self
-    }
-
-    /// The number of candidates to evaluate during the construction of the HNSW graph.
-    /// Bumping this number will increase the recall of the search but also increase the build/search time.
-    /// This value should be not less than `ef` in the search phase.
-    /// The default value is 300.
-    pub fn ef_construction(mut self, ef_construction: u32) -> Self {
-        self.ef_construction = ef_construction;
-        self
-    }
+    impl_distance_type_setter!();
+    impl_ivf_params_setter!();
+    impl_hnsw_params_setter!();
 }

rust/lancedb/src/query.rs

@@ -17,7 +17,10 @@ use std::sync::Arc;
 
 use arrow_array::{make_array, Array, Float16Array, Float32Array, Float64Array};
 use arrow_schema::DataType;
+use datafusion_physical_plan::ExecutionPlan;
 use half::f16;
+use lance::dataset::scanner::DatasetRecordBatchStream;
+use lance_datafusion::exec::execute_plan;
 
 use crate::arrow::SendableRecordBatchStream;
 use crate::error::{Error, Result};
@@ -425,6 +428,15 @@ impl Default for QueryExecutionOptions {
 /// There are various kinds of queries but they all return results
 /// in the same way.
 pub trait ExecutableQuery {
+    /// Return the Datafusion [ExecutionPlan].
+    ///
+    /// The caller can further optimize the plan or execute it.
+    ///
+    fn create_plan(
+        &self,
+        options: QueryExecutionOptions,
+    ) -> impl Future<Output = Result<Arc<dyn ExecutionPlan>>> + Send;
+
     /// Execute the query with default options and return results
     ///
     /// See [`ExecutableQuery::execute_with_options`] for more details.
@@ -545,6 +557,13 @@ impl HasQuery for Query {
 }
 
 impl ExecutableQuery for Query {
+    async fn create_plan(&self, options: QueryExecutionOptions) -> Result<Arc<dyn ExecutionPlan>> {
+        self.parent
+            .clone()
+            .create_plan(&self.clone().into_vector(), options)
+            .await
+    }
+
     async fn execute_with_options(
         &self,
         options: QueryExecutionOptions,
@@ -718,12 +737,19 @@ impl VectorQuery {
 }
 
 impl ExecutableQuery for VectorQuery {
+    async fn create_plan(&self, options: QueryExecutionOptions) -> Result<Arc<dyn ExecutionPlan>> {
+        self.base.parent.clone().create_plan(self, options).await
+    }
+
     async fn execute_with_options(
         &self,
         options: QueryExecutionOptions,
     ) -> Result<SendableRecordBatchStream> {
         Ok(SendableRecordBatchStream::from(
-            self.base.parent.clone().vector_query(self, options).await?,
+            DatasetRecordBatchStream::new(execute_plan(
+                self.create_plan(options).await?,
+                Default::default(),
+            )?),
         ))
     }
 }
@@ -972,6 +998,30 @@ mod tests {
         }
     }
 
+    fn assert_plan_exists(plan: &Arc<dyn ExecutionPlan>, name: &str) -> bool {
+        if plan.name() == name {
+            return true;
+        }
+        plan.children()
+            .iter()
+            .any(|child| assert_plan_exists(child, name))
+    }
+
+    #[tokio::test]
+    async fn test_create_execute_plan() {
+        let tmp_dir = tempdir().unwrap();
+        let table = make_test_table(&tmp_dir).await;
+        let plan = table
+            .query()
+            .nearest_to(vec![0.1, 0.2, 0.3, 0.4])
+            .unwrap()
+            .create_plan(QueryExecutionOptions::default())
+            .await
+            .unwrap();
+        assert_plan_exists(&plan, "KNNFlatSearch");
+        assert_plan_exists(&plan, "ProjectionExec");
+    }
+
     #[tokio::test]
     async fn query_base_methods_on_vector_query() {
         // Make sure VectorQuery can be used as a QueryBase
@@ -989,5 +1039,18 @@ mod tests {
         let first_batch = results.next().await.unwrap().unwrap();
         assert_eq!(first_batch.num_rows(), 1);
         assert!(results.next().await.is_none());
+
+        // query with wrong vector dimension
+        let error_result = table
+            .vector_search(&[1.0, 2.0, 3.0])
+            .unwrap()
+            .limit(1)
+            .execute()
+            .await;
+        assert!(error_result
+            .err()
+            .unwrap()
+            .to_string()
+            .contains("No vector column found to match with the query vector dimension: 3"));
     }
 }

rust/lancedb/src/remote/table.rs

@@ -1,6 +1,9 @@
+use std::sync::Arc;
+
 use arrow_array::RecordBatchReader;
 use arrow_schema::SchemaRef;
 use async_trait::async_trait;
+use datafusion_physical_plan::ExecutionPlan;
 use lance::dataset::{scanner::DatasetRecordBatchStream, ColumnAlteration, NewColumnTransform};
 
 use crate::{
@@ -71,6 +74,13 @@ impl TableInternal for RemoteTable {
     ) -> Result<()> {
         todo!()
     }
+    async fn create_plan(
+        &self,
+        _query: &VectorQuery,
+        _options: QueryExecutionOptions,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        unimplemented!()
+    }
     async fn plain_query(
         &self,
         _query: &Query,
@@ -78,13 +88,6 @@ impl TableInternal for RemoteTable {
     ) -> Result<DatasetRecordBatchStream> {
         todo!()
     }
-    async fn vector_query(
-        &self,
-        _query: &VectorQuery,
-        _options: QueryExecutionOptions,
-    ) -> Result<DatasetRecordBatchStream> {
-        todo!()
-    }
     async fn update(&self, _update: UpdateBuilder) -> Result<()> {
         todo!()
     }

rust/lancedb/src/table.rs

@@ -23,6 +23,7 @@ use arrow::datatypes::Float32Type;
 use arrow_array::{RecordBatchIterator, RecordBatchReader};
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
 use async_trait::async_trait;
+use datafusion_physical_plan::ExecutionPlan;
 use lance::dataset::builder::DatasetBuilder;
 use lance::dataset::cleanup::RemovalStats;
 use lance::dataset::optimize::{compact_files, CompactionMetrics, IndexRemapperOptions};
@@ -35,8 +36,10 @@ use lance::dataset::{
 };
 use lance::dataset::{MergeInsertBuilder as LanceMergeInsertBuilder, WhenNotMatchedBySource};
 use lance::io::WrappingObjectStore;
+use lance_datafusion::exec::execute_plan;
 use lance_index::vector::hnsw::builder::HnswBuildParams;
 use lance_index::vector::ivf::IvfBuildParams;
+use lance_index::vector::pq::PQBuildParams;
 use lance_index::vector::sq::builder::SQBuildParams;
 use lance_index::DatasetIndexExt;
 use lance_index::IndexType;
@@ -49,9 +52,10 @@ use crate::connection::NoData;
 use crate::embeddings::{EmbeddingDefinition, EmbeddingRegistry, MaybeEmbedded, MemoryRegistry};
 use crate::error::{Error, Result};
 use crate::index::vector::{
-    IvfHnswSqIndexBuilder, IvfPqIndexBuilder, VectorIndex, VectorIndexStatistics,
+    IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder, VectorIndex,
 };
 use crate::index::IndexConfig;
+use crate::index::IndexStatistics;
 use crate::index::{
     vector::{suggested_num_partitions, suggested_num_sub_vectors},
     Index, IndexBuilder,
@@ -229,7 +233,8 @@ pub struct WriteOptions {
     // pub on_bad_vectors: BadVectorHandling,
     /// Advanced parameters that can be used to customize table creation
     ///
-    /// If set, these will take precedence over any overlapping `OpenTableBuilder` options
+    /// Overlapping `OpenTableBuilder` options (e.g. [AddDataBuilder::mode]) will take
+    /// precedence over their counterparts in `WriteOptions` (e.g. [WriteParams::mode]).
     pub lance_write_params: Option<WriteParams>,
 }
 
@@ -364,16 +369,16 @@ pub(crate) trait TableInternal: std::fmt::Display + std::fmt::Debug + Send + Syn
     async fn schema(&self) -> Result<SchemaRef>;
     /// Count the number of rows in this table.
     async fn count_rows(&self, filter: Option<String>) -> Result<usize>;
+    async fn create_plan(
+        &self,
+        query: &VectorQuery,
+        options: QueryExecutionOptions,
+    ) -> Result<Arc<dyn ExecutionPlan>>;
     async fn plain_query(
         &self,
         query: &Query,
         options: QueryExecutionOptions,
     ) -> Result<DatasetRecordBatchStream>;
-    async fn vector_query(
-        &self,
-        query: &VectorQuery,
-        options: QueryExecutionOptions,
-    ) -> Result<DatasetRecordBatchStream>;
     async fn add(
         &self,
         add: AddDataBuilder<NoData>,
@@ -1217,7 +1222,7 @@ impl NativeTable {
 
     pub async fn get_index_type(&self, index_uuid: &str) -> Result<Option<String>> {
         match self.load_index_stats(index_uuid).await? {
-            Some(stats) => Ok(Some(stats.index_type)),
+            Some(stats) => Ok(Some(stats.index_type.unwrap_or_default())),
             None => Ok(None),
         }
     }
@@ -1228,7 +1233,7 @@ impl NativeTable {
                 stats
                     .indices
                     .iter()
-                    .map(|i| i.metric_type.clone())
+                    .filter_map(|i| i.metric_type.clone())
                     .collect(),
             )),
             None => Ok(None),
@@ -1244,7 +1249,7 @@ impl NativeTable {
             .collect())
     }
 
-    async fn load_index_stats(&self, index_uuid: &str) -> Result<Option<VectorIndexStatistics>> {
+    async fn load_index_stats(&self, index_uuid: &str) -> Result<Option<IndexStatistics>> {
         let index = self
             .load_indices()
             .await?
@@ -1255,7 +1260,7 @@ impl NativeTable {
         }
         let dataset = self.dataset.get().await?;
         let index_stats = dataset.index_statistics(&index.unwrap().index_name).await?;
-        let index_stats: VectorIndexStatistics = whatever!(
+        let index_stats: IndexStatistics = whatever!(
             serde_json::from_str(&index_stats),
             "error deserializing index statistics {index_stats}",
         );
@@ -1316,6 +1321,69 @@ impl NativeTable {
         Ok(())
     }
 
+    async fn create_ivf_hnsw_pq_index(
+        &self,
+        index: IvfHnswPqIndexBuilder,
+        field: &Field,
+        replace: bool,
+    ) -> Result<()> {
+        if !Self::supported_vector_data_type(field.data_type()) {
+            return Err(Error::InvalidInput {
+                message: format!(
+                    "An IVF HNSW PQ index cannot be created on the column `{}` which has data type {}",
+                    field.name(),
+                    field.data_type()
+                ),
+            });
+        }
+
+        let num_partitions = if let Some(n) = index.num_partitions {
+            n
+        } else {
+            suggested_num_partitions(self.count_rows(None).await?)
+        };
+        let num_sub_vectors: u32 = if let Some(n) = index.num_sub_vectors {
+            n
+        } else {
+            match field.data_type() {
+                arrow_schema::DataType::FixedSizeList(_, n) => {
+                    Ok::<u32, Error>(suggested_num_sub_vectors(*n as u32))
+                }
+                _ => Err(Error::Schema {
+                    message: format!("Column '{}' is not a FixedSizeList", field.name()),
+                }),
+            }?
+        };
+
+        let mut dataset = self.dataset.get_mut().await?;
+        let mut ivf_params = IvfBuildParams::new(num_partitions as usize);
+        ivf_params.sample_rate = index.sample_rate as usize;
+        ivf_params.max_iters = index.max_iterations as usize;
+        let hnsw_params = HnswBuildParams::default()
+            .num_edges(index.m as usize)
+            .ef_construction(index.ef_construction as usize);
+        let pq_params = PQBuildParams {
+            num_sub_vectors: num_sub_vectors as usize,
+            ..Default::default()
+        };
+        let lance_idx_params = lance::index::vector::VectorIndexParams::with_ivf_hnsw_pq_params(
+            index.distance_type.into(),
+            ivf_params,
+            hnsw_params,
+            pq_params,
+        );
+        dataset
+            .create_index(
+                &[field.name()],
+                IndexType::Vector,
+                None,
+                &lance_idx_params,
+                replace,
+            )
+            .await?;
+        Ok(())
+    }
+
     async fn create_ivf_hnsw_sq_index(
         &self,
         index: IvfHnswSqIndexBuilder,
@@ -1414,79 +1482,11 @@ impl NativeTable {
         query: &VectorQuery,
         options: QueryExecutionOptions,
     ) -> Result<DatasetRecordBatchStream> {
-        let ds_ref = self.dataset.get().await?;
-        let mut scanner: Scanner = ds_ref.scan();
-
-        if let Some(query_vector) = query.query_vector.as_ref() {
-            // If there is a vector query, default to limit=10 if unspecified
-            let column = if let Some(col) = query.column.as_ref() {
-                col.clone()
-            } else {
-                // Infer a vector column with the same dimension of the query vector.
-                let arrow_schema = Schema::from(ds_ref.schema());
-                default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?
-            };
-            let field = ds_ref.schema().field(&column).ok_or(Error::Schema {
-                message: format!("Column {} not found in dataset schema", column),
-            })?;
-            if let arrow_schema::DataType::FixedSizeList(f, dim) = field.data_type() {
-                if !f.data_type().is_floating() {
-                    return Err(Error::InvalidInput {
-                        message: format!(
-                            "The data type of the vector column '{}' is not a floating point type",
-                            column
-                        ),
-                    });
-                }
-                if dim != query_vector.len() as i32 {
-                    return Err(Error::InvalidInput {
-                        message: format!(
-                            "The dimension of the query vector does not match with the dimension of the vector column '{}':
-                                query dim={}, expected vector dim={}",
-                            column,
-                            query_vector.len(),
-                            dim,
-                        ),
-                    });
-                }
-            }
-            let query_vector = query_vector.as_primitive::<Float32Type>();
-            scanner.nearest(
-                &column,
-                query_vector,
-                query.base.limit.unwrap_or(DEFAULT_TOP_K),
-            )?;
-        } else {
-            // If there is no vector query, it's ok to not have a limit
-            scanner.limit(query.base.limit.map(|limit| limit as i64), None)?;
-        }
-        scanner.nprobs(query.nprobes);
-        scanner.use_index(query.use_index);
-        scanner.prefilter(query.prefilter);
-        scanner.batch_size(options.max_batch_length as usize);
-
-        match &query.base.select {
-            Select::Columns(select) => {
-                scanner.project(select.as_slice())?;
-            }
-            Select::Dynamic(select_with_transform) => {
-                scanner.project_with_transform(select_with_transform.as_slice())?;
-            }
-            Select::All => { /* Do nothing */ }
-        }
-
-        if let Some(filter) = &query.base.filter {
-            scanner.filter(filter)?;
-        }
-
-        if let Some(refine_factor) = query.refine_factor {
-            scanner.refine(refine_factor);
-        }
-
-        if let Some(distance_type) = query.distance_type {
-            scanner.distance_metric(distance_type.into());
-        }
-        Ok(scanner.try_into_stream().await?)
+        let plan = self.create_plan(query, options).await?;
+        Ok(DatasetRecordBatchStream::new(execute_plan(
+            plan,
+            Default::default(),
+        )?))
     }
 }
 
@@ -1610,6 +1610,10 @@ impl TableInternal for NativeTable {
             Index::Auto => self.create_auto_index(field, opts).await,
             Index::BTree(_) => self.create_btree_index(field, opts).await,
             Index::IvfPq(ivf_pq) => self.create_ivf_pq_index(ivf_pq, field, opts.replace).await,
+            Index::IvfHnswPq(ivf_hnsw_pq) => {
+                self.create_ivf_hnsw_pq_index(ivf_hnsw_pq, field, opts.replace)
+                    .await
+            }
             Index::IvfHnswSq(ivf_hnsw_sq) => {
                 self.create_ivf_hnsw_sq_index(ivf_hnsw_sq, field, opts.replace)
                     .await
@@ -1634,6 +1638,86 @@ impl TableInternal for NativeTable {
         Ok(())
     }
 
+    async fn create_plan(
+        &self,
+        query: &VectorQuery,
+        options: QueryExecutionOptions,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let ds_ref = self.dataset.get().await?;
+        let mut scanner: Scanner = ds_ref.scan();
+
+        if let Some(query_vector) = query.query_vector.as_ref() {
+            // If there is a vector query, default to limit=10 if unspecified
+            let column = if let Some(col) = query.column.as_ref() {
+                col.clone()
+            } else {
+                // Infer a vector column with the same dimension of the query vector.
+                let arrow_schema = Schema::from(ds_ref.schema());
+                default_vector_column(&arrow_schema, Some(query_vector.len() as i32))?
+            };
+            let field = ds_ref.schema().field(&column).ok_or(Error::Schema {
+                message: format!("Column {} not found in dataset schema", column),
+            })?;
+            if let arrow_schema::DataType::FixedSizeList(f, dim) = field.data_type() {
+                if !f.data_type().is_floating() {
+                    return Err(Error::InvalidInput {
+                        message: format!(
+                            "The data type of the vector column '{}' is not a floating point type",
+                            column
+                        ),
+                    });
+                }
+                if dim != query_vector.len() as i32 {
+                    return Err(Error::InvalidInput {
+                        message: format!(
+                            "The dimension of the query vector does not match with the dimension of the vector column '{}': \
+                                query dim={}, expected vector dim={}",
+                            column,
+                            query_vector.len(),
+                            dim,
+                        ),
+                    });
+                }
+            }
+            let query_vector = query_vector.as_primitive::<Float32Type>();
+            scanner.nearest(
+                &column,
+                query_vector,
+                query.base.limit.unwrap_or(DEFAULT_TOP_K),
+            )?;
+        } else {
+            // If there is no vector query, it's ok to not have a limit
+            scanner.limit(query.base.limit.map(|limit| limit as i64), None)?;
+        }
+        scanner.nprobs(query.nprobes);
+        scanner.use_index(query.use_index);
+        scanner.prefilter(query.prefilter);
+        scanner.batch_size(options.max_batch_length as usize);
+
+        match &query.base.select {
+            Select::Columns(select) => {
+                scanner.project(select.as_slice())?;
+            }
+            Select::Dynamic(select_with_transform) => {
+                scanner.project_with_transform(select_with_transform.as_slice())?;
+            }
+            Select::All => { /* Do nothing */ }
+        }
+
+        if let Some(filter) = &query.base.filter {
+            scanner.filter(filter)?;
+        }
+
+        if let Some(refine_factor) = query.refine_factor {
+            scanner.refine(refine_factor);
+        }
+
+        if let Some(distance_type) = query.distance_type {
+            scanner.distance_metric(distance_type.into());
+        }
+        Ok(scanner.create_plan().await?)
+    }
+
     async fn plain_query(
         &self,
         query: &Query,
@@ -1643,14 +1727,6 @@ impl TableInternal for NativeTable {
             .await
     }
 
-    async fn vector_query(
-        &self,
-        query: &VectorQuery,
-        options: QueryExecutionOptions,
-    ) -> Result<DatasetRecordBatchStream> {
-        self.generic_query(query, options).await
-    }
-
     async fn merge_insert(
         &self,
         params: MergeInsertBuilder,
@@ -1682,7 +1758,7 @@ impl TableInternal for NativeTable {
             builder.when_not_matched_by_source(WhenNotMatchedBySource::Keep);
         }
         let job = builder.try_build()?;
-        let new_dataset = job.execute_reader(new_data).await?;
+        let (new_dataset, _stats) = job.execute_reader(new_data).await?;
         self.dataset.set_latest(new_dataset.as_ref().clone()).await;
         Ok(())
     }
@@ -2475,6 +2551,24 @@ mod tests {
                 .unwrap(),
             Some(0)
         );
+        assert_eq!(
+            table
+                .as_native()
+                .unwrap()
+                .get_index_type(index_uuid)
+                .await
+                .unwrap(),
+            Some("IVF".to_string())
+        );
+        assert_eq!(
+            table
+                .as_native()
+                .unwrap()
+                .get_distance_type(index_uuid)
+                .await
+                .unwrap(),
+            Some(crate::DistanceType::L2.to_string())
+        );
     }
 
     #[tokio::test]
@@ -2573,6 +2667,102 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_create_index_ivf_hnsw_pq() {
+        use arrow_array::RecordBatch;
+        use arrow_schema::{DataType, Field, Schema as ArrowSchema};
+        use rand;
+        use std::iter::repeat_with;
+
+        use arrow_array::Float32Array;
+
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+        let conn = connect(uri).execute().await.unwrap();
+
+        let dimension = 16;
+        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "embeddings",
+            DataType::FixedSizeList(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                dimension,
+            ),
+            false,
+        )]));
+
+        let mut rng = rand::thread_rng();
+        let float_arr = Float32Array::from(
+            repeat_with(|| rng.gen::<f32>())
+                .take(512 * dimension as usize)
+                .collect::<Vec<f32>>(),
+        );
+
+        let vectors = Arc::new(create_fixed_size_list(float_arr, dimension).unwrap());
+        let batches = RecordBatchIterator::new(
+            vec![RecordBatch::try_new(schema.clone(), vec![vectors.clone()]).unwrap()]
+                .into_iter()
+                .map(Ok),
+            schema,
+        );
+
+        let table = conn.create_table("test", batches).execute().await.unwrap();
+
+        assert_eq!(
+            table
+                .as_native()
+                .unwrap()
+                .count_indexed_rows("my_index")
+                .await
+                .unwrap(),
+            None
+        );
+        assert_eq!(
+            table
+                .as_native()
+                .unwrap()
+                .count_unindexed_rows("my_index")
+                .await
+                .unwrap(),
+            None
+        );
+
+        let index = IvfHnswPqIndexBuilder::default();
+        table
+            .create_index(&["embeddings"], Index::IvfHnswPq(index))
+            .execute()
+            .await
+            .unwrap();
+
+        let index_configs = table.list_indices().await.unwrap();
+        assert_eq!(index_configs.len(), 1);
+        let index = index_configs.into_iter().next().unwrap();
+        assert_eq!(index.index_type, crate::index::IndexType::IvfPq);
+        assert_eq!(index.columns, vec!["embeddings".to_string()]);
+        assert_eq!(table.count_rows(None).await.unwrap(), 512);
+        assert_eq!(table.name(), "test");
+
+        let indices = table.as_native().unwrap().load_indices().await.unwrap();
+        let index_uuid = &indices[0].index_uuid;
+        assert_eq!(
+            table
+                .as_native()
+                .unwrap()
+                .count_indexed_rows(index_uuid)
+                .await
+                .unwrap(),
+            Some(512)
+        );
+        assert_eq!(
+            table
+                .as_native()
+                .unwrap()
+                .count_unindexed_rows(index_uuid)
+                .await
+                .unwrap(),
+            Some(0)
+        );
+    }
+
     fn create_fixed_size_list<T: Array>(values: T, list_size: i32) -> Result<FixedSizeListArray> {
         let list_type = DataType::FixedSizeList(
             Arc::new(Field::new("item", values.data_type().clone(), true)),
@@ -2644,6 +2834,27 @@ mod tests {
         let index = index_configs.into_iter().next().unwrap();
         assert_eq!(index.index_type, crate::index::IndexType::BTree);
         assert_eq!(index.columns, vec!["i".to_string()]);
+
+        let indices = table.as_native().unwrap().load_indices().await.unwrap();
+        let index_uuid = &indices[0].index_uuid;
+        assert_eq!(
+            table
+                .as_native()
+                .unwrap()
+                .count_indexed_rows(index_uuid)
+                .await
+                .unwrap(),
+            Some(1)
+        );
+        assert_eq!(
+            table
+                .as_native()
+                .unwrap()
+                .count_unindexed_rows(index_uuid)
+                .await
+                .unwrap(),
+            Some(0)
+        );
     }
 
     #[tokio::test]

rust/lancedb/src/table/dataset.rs

@@ -66,6 +66,19 @@ impl DatasetRef {
         Ok(())
     }
 
+    fn is_latest(&self) -> bool {
+        matches!(self, Self::Latest { .. })
+    }
+
+    async fn need_reload(&self) -> Result<bool> {
+        Ok(match self {
+            Self::Latest { dataset, .. } => {
+                dataset.latest_version_id().await? != dataset.version().version
+            }
+            Self::TimeTravel { dataset, version } => dataset.version().version != *version,
+        })
+    }
+
     async fn as_latest(&mut self, read_consistency_interval: Option<Duration>) -> Result<()> {
         match self {
             Self::Latest { .. } => Ok(()),
@@ -129,7 +142,7 @@ impl DatasetConsistencyWrapper {
         Self(Arc::new(RwLock::new(DatasetRef::Latest {
             dataset,
             read_consistency_interval,
-            last_consistency_check: None,
+            last_consistency_check: Some(Instant::now()),
         })))
     }
 
@@ -163,11 +176,16 @@ impl DatasetConsistencyWrapper {
 
     /// Convert into a wrapper in latest version mode
     pub async fn as_latest(&self, read_consistency_interval: Option<Duration>) -> Result<()> {
-        self.0
-            .write()
-            .await
-            .as_latest(read_consistency_interval)
-            .await
+        if self.0.read().await.is_latest() {
+            return Ok(());
+        }
+
+        let mut write_guard = self.0.write().await;
+        if write_guard.is_latest() {
+            return Ok(());
+        }
+
+        write_guard.as_latest(read_consistency_interval).await
     }
 
     pub async fn as_time_travel(&self, target_version: u64) -> Result<()> {
@@ -183,7 +201,18 @@ impl DatasetConsistencyWrapper {
     }
 
     pub async fn reload(&self) -> Result<()> {
-        self.0.write().await.reload().await
+        if !self.0.read().await.need_reload().await? {
+            return Ok(());
+        }
+
+        let mut write_guard = self.0.write().await;
+        // on lock escalation -- check if someone else has already reloaded
+        if !write_guard.need_reload().await? {
+            return Ok(());
+        }
+
+        // actually need reloading
+        write_guard.reload().await
     }
 
     /// Returns the version, if in time travel mode, or None otherwise

rust/lancedb/src/utils.rs

@@ -101,7 +101,7 @@ pub fn validate_table_name(name: &str) -> Result<()> {
     Ok(())
 }
 
-/// Find one default column to create index.
+/// Find one default column to create index or perform vector query.
 pub(crate) fn default_vector_column(schema: &Schema, dim: Option<i32>) -> Result<String> {
     // Try to find one fixed size list array column.
     let candidates = schema
@@ -118,14 +118,17 @@ pub(crate) fn default_vector_column(schema: &Schema, dim: Option<i32>) -> Result
         })
         .collect::<Vec<_>>();
     if candidates.is_empty() {
-        Err(Error::Schema {
-            message: "No vector column found to create index".to_string(),
+        Err(Error::InvalidInput {
+            message: format!(
+                "No vector column found to match with the query vector dimension: {}",
+                dim.unwrap_or_default()
+            ),
         })
     } else if candidates.len() != 1 {
         Err(Error::Schema {
             message: format!(
                 "More than one vector columns found, \
-                    please specify which column to create index: {:?}",
+                    please specify which column to create index or query: {:?}",
                 candidates
             ),
         })

rust/lancedb/tests/embedding_registry_test.rs

@@ -302,7 +302,7 @@ impl EmbeddingFunction for MockEmbed {
     fn dest_type(&self) -> Result<Cow<DataType>> {
         Ok(Cow::Borrowed(&self.dest_type))
     }
-    fn embed(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+    fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
         // We can't use the FixedSizeListBuilder here because it always adds a null bitmap
         // and we want to explicitly work with non-nullable arrays.
         let len = source.len();
@@ -317,4 +317,9 @@ impl EmbeddingFunction for MockEmbed {
 
         Ok(Arc::new(arr))
     }
+
+    #[allow(unused_variables)]
+    fn compute_query_embeddings(&self, input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+        unimplemented!()
+    }
 }