Bump version: 0.15.1-beta.1 → 0.15.1-beta.2

* Adds a CI job to check the cross compiled Windows ARM build.
* Didn't replace the test build because we need native build to run
tests. But for some reason (I forget why) we need cross compiled for
nodejs.
* Pinned crunchy to workaround
https://github.com/eira-fransham/crunchy/issues/13

This is needed to fix failure from
https://github.com/lancedb/lancedb/actions/runs/13020773184/job/36320719331

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.15.0"
+current_version = "0.15.1-beta.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/license-header-check.yml

@@ -0,0 +1,31 @@
+name: Check license headers
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - rust/**
+      - python/**
+      - nodejs/**
+      - java/**
+      - .github/workflows/license-header-check.yml
+jobs:
+  check-licenses:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Install license-header-checker
+        working-directory: /tmp
+        run: |
+            curl -s https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh | bash
+            mv /tmp/bin/license-header-checker /usr/local/bin/
+      - name: Check license headers (rust)
+        run: license-header-checker -a -v ./rust/license_header.txt ./ rs && [[ -z `git status -s` ]]
+      - name: Check license headers (python)
+        run: license-header-checker -a -v ./python/license_header.txt python py && [[ -z `git status -s` ]]
+      - name: Check license headers (typescript)
+        run: license-header-checker -a -v ./nodejs/license_header.txt nodejs ts && [[ -z `git status -s` ]]
+      - name: Check license headers (java)
+        run: license-header-checker -a -v ./nodejs/license_header.txt java java && [[ -z `git status -s` ]]

.github/workflows/rust.yml

@@ -22,6 +22,7 @@ env:
   # "1" means line tables only, which is useful for panic tracebacks.
   RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
+  CARGO_INCREMENTAL: 0
 
 jobs:
   lint:
@@ -51,6 +52,28 @@ jobs:
       - name: Run clippy
         run: cargo clippy --workspace --tests --all-features -- -D warnings
 
+  build-no-lock:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      # Need up-to-date compilers for kernels
+      CC: clang
+      CXX: clang++
+    steps:
+      - uses: actions/checkout@v4
+      # Remote cargo.lock to force a fresh build
+      - name: Remove Cargo.lock
+        run: rm -f Cargo.lock
+      - uses: rui314/setup-mold@v1
+      - uses: Swatinem/rust-cache@v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Build all
+        run: |
+          cargo build --benches --all-features --tests
+
   linux:
     timeout-minutes: 30
     # To build all features, we need more disk space than is available
@@ -75,8 +98,11 @@ jobs:
           workspaces: rust
       - name: Install dependencies
         run: |
-          sudo apt update
+          # This shaves 2 minutes off this step in CI. This doesn't seem to be
+          # necessary in standard runners, but it is in the 4x runners.
+          sudo rm /var/lib/man-db/auto-update
           sudo apt install -y protobuf-compiler libssl-dev
+      - uses: rui314/setup-mold@v1
       - name: Make Swap
         run: |
           sudo fallocate -l 16G /swapfile
@@ -87,11 +113,11 @@ jobs:
         working-directory: .
         run: docker compose up --detach --wait
       - name: Build
-        run: cargo build --all-features
+        run: cargo build --all-features --tests --locked --examples
       - name: Run tests
-        run: cargo test --all-features
+        run: cargo test --all-features --locked
       - name: Run examples
-        run: cargo run --example simple
+        run: cargo run --example simple --locked
 
   macos:
     timeout-minutes: 30
@@ -115,11 +141,14 @@ jobs:
           workspaces: rust
       - name: Install dependencies
         run: brew install protobuf
-      - name: Build
-        run: cargo build --all-features
       - name: Run tests
-        # Run with everything except the integration tests.
+        run: |
-        run: cargo test --features remote,fp16kernels
+          # Don't run the s3 integration tests since docker isn't available
+          # on this image.
+          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
+            | jq -r '.packages[] | .features | keys | .[]' \
+            | grep -v s3-test | sort | uniq | paste -s -d "," -`
+          cargo test --features $ALL_FEATURES --locked
 
   windows:
     runs-on: windows-2022
@@ -140,8 +169,38 @@ jobs:
       - name: Run tests
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build
+          cargo test --features remote --locked
-          cargo test
+
+  windows-arm64-cross:
+    # We cross compile in Node releases, so we want to make sure
+    # this can run successfully.
+    runs-on: ubuntu-latest
+    container: alpine:edge
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          set -e
+          apk add protobuf-dev curl clang lld llvm19 grep npm bash msitools sed
+
+          curl --proto '=https' --tlsv1.3 -sSf https://raw.githubusercontent.com/rust-lang/rustup/refs/heads/master/rustup-init.sh | sh -s -- -y
+          source $HOME/.cargo/env
+          rustup target add aarch64-pc-windows-msvc
+
+          mkdir -p sysroot
+          cd sysroot
+          sh ../ci/sysroot-aarch64-pc-windows-msvc.sh
+      - name: Check
+        env:
+          CC: clang
+          AR: llvm-ar
+          C_INCLUDE_PATH: /usr/aarch64-pc-windows-msvc/usr/include
+          CARGO_BUILD_TARGET: aarch64-pc-windows-msvc
+          RUSTFLAGS: -Ctarget-feature=+crt-static,+neon,+fp16,+fhm,+dotprod -Clinker=lld -Clink-arg=/LIBPATH:/usr/aarch64-pc-windows-msvc/usr/lib -Clink-arg=arm64rt.lib
+        run: |
+          source $HOME/.cargo/env
+          cargo check --features remote --locked
 
   windows-arm64:
     runs-on: windows-4x-arm
@@ -236,8 +295,7 @@ jobs:
       - name: Run tests
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build --target aarch64-pc-windows-msvc
+          cargo test --target aarch64-pc-windows-msvc --features remote --locked
-          cargo test --target aarch64-pc-windows-msvc
 
   msrv:
     # Check the minimum supported Rust version

.gitignore

@@ -9,7 +9,6 @@ venv
 .vscode
 .zed
 rust/target
-rust/Cargo.lock
 
 site
 
@@ -42,5 +41,3 @@ dist
 target
 
 **/sccache.log
-
-Cargo.lock

Cargo.toml

@@ -23,14 +23,14 @@ rust-version = "1.78.0"
 [workspace.dependencies]
 lance = { "version" = "=0.23.0", "features" = [
     "dynamodb",
-], git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.2" }
+], git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-io = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.2" }
+lance-io = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-index = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.2" }
+lance-index = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-linalg = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.2" }
+lance-linalg = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-table = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.2" }
+lance-table = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-testing = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.2" }
+lance-testing = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-datafusion = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.2" }
+lance-datafusion = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
-lance-encoding = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.2" }
+lance-encoding = { version = "=0.23.0", git = "https://github.com/lancedb/lance.git", tag = "v0.23.0-beta.3" }
 # Note that this one does not include pyarrow
 arrow = { version = "53.2", optional = false }
 arrow-array = "53.2"
@@ -59,3 +59,6 @@ num-traits = "0.2"
 rand = "0.8"
 regex = "1.10"
 lazy_static = "1"
+
+# Workaround for: https://github.com/eira-fransham/crunchy/issues/13
+crunchy = "=0.2.2"

docs/src/notebooks/diffusiondb/datagen.py

@@ -1,17 +1,6 @@
 #!/usr/bin/env python
 #
-#  Copyright 2023 LanceDB Developers
+
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 """Dataset hf://poloclub/diffusiondb
 """

docs/src/notebooks/tables_guide.ipynb

@@ -114,14 +114,17 @@
     }
    ],
    "source": [
-    "data = [\n",
+    "import pandas as pd\n",
-    "    {\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n",
-    "    {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1},\n",
-    "]\n",
     "\n",
-    "db.create_table(\"table2\", data)\n",
+    "data = pd.DataFrame(\n",
-    "\n",
+    "    {\n",
-    "db[\"table2\"].head() "
+    "        \"vector\": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],\n",
+    "        \"lat\": [45.5, 40.1],\n",
+    "        \"long\": [-122.7, -74.1],\n",
+    "    }\n",
+    ")\n",
+    "db.create_table(\"my_table_pandas\", data)\n",
+    "db[\"my_table_pandas\"].head()"
    ]
   },
   {
@@ -164,7 +167,7 @@
     "import pyarrow as pa\n",
     "\n",
     "custom_schema = pa.schema([\n",
-    "pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n",
+    "pa.field(\"vector\", pa.list_(pa.float32(), 4)),\n",
     "pa.field(\"lat\", pa.float32()),\n",
     "pa.field(\"long\", pa.float32())\n",
     "])\n",

docs/src/sql.md

@@ -64,6 +64,7 @@ const tbl = await db.createTable('myVectors', data)
         ```ts
         --8<-- "docs/src/sql_legacy.ts:search"
         ```
+
 !!! note
 
     Creating a [scalar index](guides/scalar_index.md) accelerates filtering.
@@ -118,14 +119,14 @@ For example, the following filter string is acceptable:
         --8<-- "docs/src/sql_legacy.ts:vec_search"
         ```
 
-If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
+If your column name contains special characters, upper-case characters, or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
 you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
 path must be wrapped in backticks.
 
 === "SQL"
 
     ```sql
-    `CUBE` = 10 AND `column name with space` IS NOT NULL
+    `CUBE` = 10 AND `UpperCaseName` = '3' AND `column name with space` IS NOT NULL
       AND `nested with space`.`inner with space` < 2
     ```
 

java/core/lancedb-jni/src/connection.rs

@@ -1,3 +1,6 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 use crate::ffi::JNIEnvExt;
 use crate::traits::IntoJava;
 use crate::{Error, RT};

java/core/lancedb-jni/src/error.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use std::str::Utf8Error;
 

java/core/lancedb-jni/src/ffi.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use core::slice;
 

java/core/lancedb-jni/src/lib.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use lazy_static::lazy_static;
 

java/core/lancedb-jni/src/traits.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use jni::objects::{JMap, JObject, JString, JValue};
 use jni::JNIEnv;

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lancedb-parent</artifactId>
-        <version>0.15.0-final.0</version>
+        <version>0.15.1-beta.2</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/core/src/main/java/com/lancedb/lancedb/Connection.java

@@ -1,16 +1,5 @@
-/*
+// SPDX-License-Identifier: Apache-2.0
- * Licensed under the Apache License, Version 2.0 (the "License");
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 
 package com.lancedb.lancedb;
 
@@ -117,4 +106,4 @@ public class Connection implements Closeable {
   private native void releaseNativeConnection(long handle);
 
   private Connection() {}
-}
+}

java/core/src/test/java/com/lancedb/lancedb/ConnectionTest.java

@@ -1,16 +1,5 @@
-/*
+// SPDX-License-Identifier: Apache-2.0
- * Licensed under the Apache License, Version 2.0 (the "License");
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 package com.lancedb.lancedb;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;

java/license_header.txt

@@ -0,0 +1,4 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * SPDX-FileCopyrightText: Copyright The LanceDB Authors
+ */

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.15.0-final.0</version>
+    <version>0.15.1-beta.2</version>
     <packaging>pom</packaging>
 
     <name>LanceDB Parent</name>

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.15.0",
+  "version": "0.15.1-beta.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.15.0",
+      "version": "0.15.1-beta.1",
       "cpu": [
         "x64",
         "arm64"
@@ -52,14 +52,14 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.15.0",
+        "@lancedb/vectordb-darwin-arm64": "0.15.1-beta.1",
-        "@lancedb/vectordb-darwin-x64": "0.15.0",
+        "@lancedb/vectordb-darwin-x64": "0.15.1-beta.1",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.15.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.15.1-beta.1",
-        "@lancedb/vectordb-linux-arm64-musl": "0.15.0",
+        "@lancedb/vectordb-linux-arm64-musl": "0.15.1-beta.1",
-        "@lancedb/vectordb-linux-x64-gnu": "0.15.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.15.1-beta.1",
-        "@lancedb/vectordb-linux-x64-musl": "0.15.0",
+        "@lancedb/vectordb-linux-x64-musl": "0.15.1-beta.1",
-        "@lancedb/vectordb-win32-arm64-msvc": "0.15.0",
+        "@lancedb/vectordb-win32-arm64-msvc": "0.15.1-beta.1",
-        "@lancedb/vectordb-win32-x64-msvc": "0.15.0"
+        "@lancedb/vectordb-win32-x64-msvc": "0.15.1-beta.1"
       },
       "peerDependencies": {
         "@apache-arrow/ts": "^14.0.2",
@@ -329,110 +329,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.10"
       }
     },
-    "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.15.0.tgz",
-      "integrity": "sha512-FnBRsCrxvecjhkMQus9M9RQpXyhu1jxQjYGDaqqRIfcUd3ew7ahIR4qk9FyALHmjpPd72xJZgNLjliHtsIX4/w==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.15.0.tgz",
-      "integrity": "sha512-zy+nt1WBCabVI16u2t3sqGUXBOmnF5ZXMsHa9TWYEXVnbw5112K7/1783DTNA/ZBI/WziUa5jqYQ0GOwkgruqA==",
-      "cpu": [
-        "x64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.15.0.tgz",
-      "integrity": "sha512-2Pbw+z5Ij5QBvmBxmjaT5F2lNHftVWlarDM1bDc4JtgodJ3Js729qnVLQ0yehnlt+hM6aGFEyn8bH5vf6gEvpQ==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-musl": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-musl/-/vectordb-linux-arm64-musl-0.15.0.tgz",
-      "integrity": "sha512-WIvgd2EY2maCdYNHPC0C9RprjNWL83FkQKtn591xixltFk3XKgvBQ2USZW2tXndH/WVdvFQvystmZ3dgUrh8DQ==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.15.0.tgz",
-      "integrity": "sha512-Pet3aPE+yQT13Gm0+fh11pgHvImS4X8Uf0zRdzsx0eja7x8j15VrVcZTEVTT4QdBNiZrhXBuiq482NJBsqe6vw==",
-      "cpu": [
-        "x64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-x64-musl": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-musl/-/vectordb-linux-x64-musl-0.15.0.tgz",
-      "integrity": "sha512-BC1RvIoEmyOr7ENp618vs9F05gdN7aKlToJNZnGIoi++hRZ25y39B1xxMXQHDnUL8G+Ur9kJObfQ43nVWqueTQ==",
-      "cpu": [
-        "x64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-win32-arm64-msvc": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-arm64-msvc/-/vectordb-win32-arm64-msvc-0.15.0.tgz",
-      "integrity": "sha512-H9BeryZl1aLxldtVP0XyiQJyzKStkuxS6SmIg+zaANr9Dns+LmVxYCz429JLC0DlvBWoYjTfK9WJTgMSZXr0Cg==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.15.0",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.15.0.tgz",
-      "integrity": "sha512-J8JICux2M82OR27i/4YAbEPlvszuE7EnGIU5jmm2+RTFaptKOCshH1C4D4jEXDAaHcUkVgsxyc9lGmGJCkGLhg==",
-      "cpu": [
-        "x64"
-      ],
-      "license": "Apache-2.0",
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/@neon-rs/cli": {
       "version": "0.0.160",
       "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.15.0",
+  "version": "0.15.1-beta.2",
   "description": " Serverless, low-latency vector database for AI applications",
   "private": false,
   "main": "dist/index.js",
@@ -92,13 +92,13 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-x64": "0.15.0",
+    "@lancedb/vectordb-darwin-x64": "0.15.1-beta.2",
-    "@lancedb/vectordb-darwin-arm64": "0.15.0",
+    "@lancedb/vectordb-darwin-arm64": "0.15.1-beta.2",
-    "@lancedb/vectordb-linux-x64-gnu": "0.15.0",
+    "@lancedb/vectordb-linux-x64-gnu": "0.15.1-beta.2",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.15.0",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.15.1-beta.2",
-    "@lancedb/vectordb-linux-x64-musl": "0.15.0",
+    "@lancedb/vectordb-linux-x64-musl": "0.15.1-beta.2",
-    "@lancedb/vectordb-linux-arm64-musl": "0.15.0",
+    "@lancedb/vectordb-linux-arm64-musl": "0.15.1-beta.2",
-    "@lancedb/vectordb-win32-x64-msvc": "0.15.0",
+    "@lancedb/vectordb-win32-x64-msvc": "0.15.1-beta.2",
-    "@lancedb/vectordb-win32-arm64-msvc": "0.15.0"
+    "@lancedb/vectordb-win32-arm64-msvc": "0.15.1-beta.2"
   }
 }

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.15.0"
+version = "0.15.1-beta.2"
 license.workspace = true
 description.workspace = true
 repository.workspace = true

nodejs/__test__/arrow.test.ts

@@ -1,17 +1,7 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import { Schema } from "apache-arrow";
-// Copyright 2024 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";

nodejs/__test__/connection.test.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import { readdirSync } from "fs";
 import { Field, Float64, Schema } from "apache-arrow";

nodejs/__test__/embedding.test.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import * as tmp from "tmp";
 
@@ -83,6 +72,74 @@ describe("embedding functions", () => {
     expect(vector0).toEqual([1, 2, 3]);
   });
 
+  it("should be able to append and upsert using embedding function", async () => {
+    @register()
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      toJSON(): object {
+        return {};
+      }
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+    const func = new MockEmbeddingFunction();
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable(
+      "test",
+      [
+        { id: 1, text: "hello" },
+        { id: 2, text: "world" },
+      ],
+      {
+        embeddingFunction: {
+          function: func,
+          sourceColumn: "text",
+        },
+      },
+    );
+
+    const schema = await table.schema();
+    expect(schema.metadata.get("embedding_functions")).toBeDefined();
+
+    // Append some new data
+    const data1 = [
+      { id: 3, text: "forest" },
+      { id: 4, text: "mountain" },
+    ];
+    await table.add(data1);
+
+    // Upsert some data
+    const data2 = [
+      { id: 5, text: "river" },
+      { id: 2, text: "canyon" },
+    ];
+    await table
+      .mergeInsert("id")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute(data2);
+
+    const rows = await table.query().toArray();
+    rows.sort((a, b) => a.id - b.id);
+    const texts = rows.map((row) => row.text);
+    expect(texts).toEqual(["hello", "canyon", "forest", "mountain", "river"]);
+    const vectorsDefined = rows.map(
+      (row) => row.vector !== undefined && row.vector !== null,
+    );
+    expect(vectorsDefined).toEqual(new Array(5).fill(true));
+  });
+
   it("should be able to create an empty table with an embedding function", async () => {
     @register()
     class MockEmbeddingFunction extends EmbeddingFunction<string> {

nodejs/__test__/registry.test.ts

@@ -1,17 +1,8 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import * as apiArrow from "apache-arrow";
-// Copyright 2024 Lance Developers.
+
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 import * as arrow15 from "apache-arrow-15";
 import * as arrow16 from "apache-arrow-16";
 import * as arrow17 from "apache-arrow-17";

nodejs/__test__/remote.test.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import * as http from "http";
 import { RequestListener } from "http";

nodejs/__test__/s3_integration.test.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 /* eslint-disable @typescript-eslint/naming-convention */
 

nodejs/__test__/table.test.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import * as fs from "fs";
 import * as path from "path";

nodejs/__test__/util.test.ts

@@ -1,3 +1,6 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import { IntoSql, toSQL } from "../lancedb/util";
 test.each([
   ["string", "'string'"],

nodejs/build.rs

@@ -1,3 +1,6 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 extern crate napi_build;
 
 fn main() {

nodejs/lancedb/arrow.ts

@@ -1,16 +1,5 @@
-// Copyright 2023 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import {
   Table as ArrowTable,
@@ -609,6 +598,14 @@ async function applyEmbeddings<T>(
     return table;
   }
 
+  let schemaMetadata = schema?.metadata || new Map<string, string>();
+
+  if (!(embeddings == null || embeddings === undefined)) {
+    const registry = getRegistry();
+    const embeddingMetadata = registry.getTableMetadata([embeddings]);
+    schemaMetadata = new Map([...schemaMetadata, ...embeddingMetadata]);
+  }
+
   // Convert from ArrowTable to Record<String, Vector>
   const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
     const name = table.schema.fields[idx].name;
@@ -677,15 +674,21 @@ async function applyEmbeddings<T>(
     newColumns[destColumn] = makeVector(vectors, destType);
   }
 
-  const newTable = new ArrowTable(newColumns);
+  let newTable = new ArrowTable(newColumns);
   if (schema != null) {
     if (schema.fields.find((f) => f.name === destColumn) === undefined) {
       throw new Error(
         `When using embedding functions and specifying a schema the schema should include the embedding column but the column ${destColumn} was missing`,
       );
     }
-    return alignTable(newTable, schema as Schema);
+    newTable = alignTable(newTable, schema as Schema);
   }
+
+  newTable = new ArrowTable(
+    new Schema(newTable.schema.fields, schemaMetadata),
+    newTable.batches,
+  );
+
   return newTable;
 }
 

nodejs/lancedb/connection.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import { Data, Schema, SchemaLike, TableLike } from "./arrow";
 import { fromTableToBuffer, makeEmptyTable } from "./arrow";

nodejs/lancedb/embedding/embedding_function.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import "reflect-metadata";
 import {

nodejs/lancedb/embedding/index.ts

@@ -1,16 +1,5 @@
-// Copyright 2023 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import { Field, Schema } from "../arrow";
 import { sanitizeType } from "../sanitize";

nodejs/lancedb/embedding/openai.ts

@@ -1,16 +1,5 @@
-// Copyright 2023 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import type OpenAI from "openai";
 import type { EmbeddingCreateParams } from "openai/resources/index";

nodejs/lancedb/embedding/registry.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import {
   type EmbeddingFunction,

nodejs/lancedb/embedding/transformers.ts

@@ -1,16 +1,5 @@
-// Copyright 2023 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import { Float, Float32 } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";

nodejs/lancedb/index.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import {
   Connection,

nodejs/lancedb/indices.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import { Index as LanceDbIndex } from "./native";
 

nodejs/lancedb/merge.ts

@@ -1,13 +1,20 @@
-import { Data, fromDataToBuffer } from "./arrow";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+import { Data, Schema, fromDataToBuffer } from "./arrow";
 import { NativeMergeInsertBuilder } from "./native";
 
 /** A builder used to create and run a merge insert operation */
 export class MergeInsertBuilder {
   #native: NativeMergeInsertBuilder;
+  #schema: Schema | Promise<Schema>;
 
   /** Construct a MergeInsertBuilder. __Internal use only.__ */
-  constructor(native: NativeMergeInsertBuilder) {
+  constructor(
+    native: NativeMergeInsertBuilder,
+    schema: Schema | Promise<Schema>,
+  ) {
     this.#native = native;
+    this.#schema = schema;
   }
 
   /**
@@ -35,6 +42,7 @@ export class MergeInsertBuilder {
   whenMatchedUpdateAll(options?: { where: string }): MergeInsertBuilder {
     return new MergeInsertBuilder(
       this.#native.whenMatchedUpdateAll(options?.where),
+      this.#schema,
     );
   }
   /**
@@ -42,7 +50,10 @@ export class MergeInsertBuilder {
    * be inserted into the target table.
    */
   whenNotMatchedInsertAll(): MergeInsertBuilder {
-    return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll());
+    return new MergeInsertBuilder(
+      this.#native.whenNotMatchedInsertAll(),
+      this.#schema,
+    );
   }
   /**
    * Rows that exist only in the target table (old data) will be
@@ -56,6 +67,7 @@ export class MergeInsertBuilder {
   }): MergeInsertBuilder {
     return new MergeInsertBuilder(
       this.#native.whenNotMatchedBySourceDelete(options?.where),
+      this.#schema,
     );
   }
   /**
@@ -64,7 +76,14 @@ export class MergeInsertBuilder {
    * Nothing is returned but the `Table` is updated
    */
   async execute(data: Data): Promise<void> {
-    const buffer = await fromDataToBuffer(data);
+    let schema: Schema;
+    if (this.#schema instanceof Promise) {
+      schema = await this.#schema;
+      this.#schema = schema; // In case of future calls
+    } else {
+      schema = this.#schema;
+    }
+    const buffer = await fromDataToBuffer(data, undefined, schema);
     await this.#native.execute(buffer);
   }
 }

nodejs/lancedb/query.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import {
   Table as ArrowTable,

nodejs/lancedb/sanitize.ts

@@ -1,16 +1,5 @@
-// Copyright 2023 LanceDB Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 // The utilities in this file help sanitize data from the user's arrow
 // library into the types expected by vectordb's arrow library.  Node

nodejs/lancedb/table.ts

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 import {
   Table as ArrowTable,
@@ -520,14 +509,8 @@ export class LocalTable extends Table {
   async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
     const mode = options?.mode ?? "append";
     const schema = await this.schema();
-    const registry = getRegistry();
-    const functions = await registry.parseFunctions(schema.metadata);
 
-    const buffer = await fromDataToBuffer(
+    const buffer = await fromDataToBuffer(data, undefined, schema);
-      data,
-      functions.values().next().value,
-      schema,
-    );
     await this.inner.add(buffer, mode);
   }
 
@@ -733,7 +716,7 @@ export class LocalTable extends Table {
   }
   mergeInsert(on: string | string[]): MergeInsertBuilder {
     on = Array.isArray(on) ? on : [on];
-    return new MergeInsertBuilder(this.inner.mergeInsert(on));
+    return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());
   }
 
   /**

nodejs/lancedb/util.ts

@@ -1,3 +1,6 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 export type IntoSql =
   | string
   | number

nodejs/license_header.txt

@@ -0,0 +1,2 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.15.0",
+	"version": "0.15.1-beta.2",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.15.0",
+	"version": "0.15.1-beta.2",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.15.0",
+	"version": "0.15.1-beta.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.15.0",
+	"version": "0.15.1-beta.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.15.0",
+	"version": "0.15.1-beta.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.15.0",
+	"version": "0.15.1-beta.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.15.0",
+  "version": "0.15.1-beta.2",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.15.0",
+	"version": "0.15.1-beta.2",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.15.0",
+  "version": "0.15.1-beta.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.15.0",
+      "version": "0.15.1-beta.1",
       "cpu": [
         "x64",
         "arm64"

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.15.0",
+  "version": "0.15.1-beta.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

nodejs/src/connection.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use std::collections::HashMap;
 use std::str::FromStr;

nodejs/src/error.rs

@@ -1,3 +1,6 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 pub type Result<T> = napi::Result<T>;
 
 pub trait NapiErrorExt<T> {

nodejs/src/index.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use std::sync::Mutex;
 

nodejs/src/iterator.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use futures::StreamExt;
 use lancedb::arrow::SendableRecordBatchStream;

nodejs/src/lib.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use std::collections::HashMap;
 

nodejs/src/merge.rs

@@ -1,3 +1,6 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 use lancedb::{arrow::IntoArrow, ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
 use napi::bindgen_prelude::*;
 use napi_derive::napi;

nodejs/src/query.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use std::sync::Arc;
 

nodejs/src/remote.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use napi_derive::*;
 

nodejs/src/table.rs

@@ -1,16 +1,5 @@
-// Copyright 2024 Lance Developers.
+// SPDX-License-Identifier: Apache-2.0
-//
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use std::collections::HashMap;
 

nodejs/src/util.rs

@@ -1,3 +1,6 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 use lancedb::DistanceType;
 
 pub fn parse_distance_type(distance_type: impl AsRef<str>) -> napi::Result<DistanceType> {

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.18.1-beta.1"
+current_version = "0.18.1-beta.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.18.1-beta.1"
+version = "0.18.1-beta.2"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true

python/build.rs

@@ -1,3 +1,6 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 fn main() {
     pyo3_build_config::add_extension_module_link_args();
 }

python/license_header.txt

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors

python/pyproject.toml

@@ -4,7 +4,7 @@ name = "lancedb"
 dynamic = ["version"]
 dependencies = [
     "deprecation",
-    "pylance==0.23.0b2",
+    "pylance==0.23.0b3",
     "tqdm>=4.27.0",
     "pydantic>=1.10",
     "packaging",

python/python/lancedb/__init__.py

@@ -1,15 +1,6 @@
-#  Copyright 2023 LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import importlib.metadata
 import os

python/python/lancedb/arrow.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 from typing import List, Optional, Union
 
 import pyarrow as pa

python/python/lancedb/common.py

@@ -1,15 +1,7 @@
-#  Copyright 2023 LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from pathlib import Path
 from typing import Iterable, List, Optional, Union
 

python/python/lancedb/conftest.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import os
 import time
 

python/python/lancedb/context.py

@@ -1,15 +1,7 @@
-#  Copyright 2023 LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from __future__ import annotations
 
 import deprecation

python/python/lancedb/db.py

@@ -1,15 +1,6 @@
-#  Copyright 2023 LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 from __future__ import annotations
 

python/python/lancedb/dependencies.py

@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The Lance Authors
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+
 #
 # The following code is originally from https://github.com/pola-rs/polars/blob/ea4389c31b0e87ddf20a85e4c3797b285966edb6/py-polars/polars/dependencies.py
 # and is licensed under the MIT license:

python/python/lancedb/embeddings/__init__.py

@@ -1,15 +1,7 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 # ruff: noqa: F401
 from .base import EmbeddingFunction, EmbeddingFunctionConfig, TextEmbeddingFunction
 from .bedrock import BedRockText

python/python/lancedb/embeddings/bedrock.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import json
 from functools import cached_property

python/python/lancedb/embeddings/cohere.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import os
 from typing import ClassVar, List, Union

python/python/lancedb/embeddings/gemini_text.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import os
 from functools import cached_property

python/python/lancedb/embeddings/gte.py

@@ -1,15 +1,7 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from typing import List, Union
 
 import numpy as np

python/python/lancedb/embeddings/gte_mlx_model.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import json
 from typing import List, Optional
 

python/python/lancedb/embeddings/imagebind.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 from functools import cached_property
 from typing import List, Union

python/python/lancedb/embeddings/instructor.py

@@ -1,15 +1,7 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from typing import List
 
 import numpy as np

python/python/lancedb/embeddings/jinaai.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import os
 import io

python/python/lancedb/embeddings/open_clip.py

@@ -1,15 +1,7 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 import concurrent.futures
 import io
 import os

python/python/lancedb/embeddings/sentence_transformers.py

@@ -1,15 +1,7 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from typing import List, Union
 
 import numpy as np

python/python/lancedb/embeddings/transformers.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 from functools import cached_property
 from typing import List, Any

python/python/lancedb/embeddings/utils.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import functools
 import math

python/python/lancedb/embeddings/voyageai.py

@@ -1,15 +1,6 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import os
 from typing import ClassVar, TYPE_CHECKING, List, Union

python/python/lancedb/embeddings/watsonx.py

@@ -1,15 +1,7 @@
-#  Copyright (c) 2023. LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 import os
 from functools import cached_property
 from typing import List, Optional, Dict, Union

python/python/lancedb/exceptions.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 """Custom exception handling"""
 
 

python/python/lancedb/index.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 from dataclasses import dataclass
 from typing import Literal, Optional
 

python/python/lancedb/integrations/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors

python/python/lancedb/integrations/pyarrow.py

@@ -1,3 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
 import logging
 from typing import Any, List, Optional, Tuple, Union, Literal
 

python/python/lancedb/merge.py

@@ -1,15 +1,7 @@
-#  Copyright 2023 LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
+
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, List, Optional

python/python/lancedb/query.py

@@ -505,7 +505,7 @@ class LanceQueryBuilder(ABC):
                 "column": self._vector_column,
                 "q": self._query,
                 "k": self._limit,
-                "metric": self._metric,
+                "metric": self._distance_type,
                 "nprobes": self._nprobes,
                 "refine_factor": self._refine_factor,
                 "use_index": self._use_index,
@@ -576,7 +576,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
     >>> db = lancedb.connect("./.lancedb")
     >>> table = db.create_table("my_table", data=data)
     >>> (table.search([0.4, 0.4])
-    ...       .metric("cosine")
+    ...       .distance_type("cosine")
     ...       .where("b < 10")
     ...       .select(["b", "vector"])
     ...       .limit(2)
@@ -596,7 +596,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
     ):
         super().__init__(table)
         self._query = query
-        self._metric = "L2"
+        self._distance_type = "L2"
         self._nprobes = 20
         self._lower_bound = None
         self._upper_bound = None
@@ -610,6 +610,9 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
     def metric(self, metric: Literal["L2", "cosine", "dot"]) -> LanceVectorQueryBuilder:
         """Set the distance metric to use.
 
+        This is an alias for distance_type() and may be deprecated in the future.
+        Please use distance_type() instead.
+
         Parameters
         ----------
         metric: "L2" or "cosine" or "dot"
@@ -620,7 +623,32 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         LanceVectorQueryBuilder
             The LanceQueryBuilder object.
         """
-        self._metric = metric.lower()
+        return self.distance_type(metric)
+
+    def distance_type(
+        self, distance_type: Literal["L2", "cosine", "dot"]
+    ) -> "LanceVectorQueryBuilder":
+        """Set the distance metric to use.
+
+        When performing a vector search we try and find the "nearest" vectors according
+        to some kind of distance metric. This parameter controls which distance metric
+        to use.
+
+        Note: if there is a vector index then the distance type used MUST match the
+        distance type used to train the vector index. If this is not done then the
+        results will be invalid.
+
+        Parameters
+        ----------
+        distance_type: "L2" or "cosine" or "dot"
+            The distance metric to use. By default "L2" is used.
+
+        Returns
+        -------
+        LanceVectorQueryBuilder
+            The LanceQueryBuilder object.
+        """
+        self._distance_type = distance_type.lower()
         return self
 
     def nprobes(self, nprobes: int) -> LanceVectorQueryBuilder:
@@ -745,7 +773,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
             filter=self._where,
             prefilter=self._prefilter,
             k=self._limit,
-            metric=self._metric,
+            metric=self._distance_type,
             columns=self._columns,
             nprobes=self._nprobes,
             lower_bound=self._lower_bound,
@@ -1078,7 +1106,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         self._reranker = RRFReranker()
         self._nprobes = None
         self._refine_factor = None
-        self._metric = None
+        self._distance_type = None
         self._phrase_query = False
 
     def _validate_query(self, query, vector=None, text=None):
@@ -1146,8 +1174,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             self._fts_query.with_row_id(True)
         if self._phrase_query:
             self._fts_query.phrase_query(True)
-        if self._metric:
+        if self._distance_type:
-            self._vector_query.metric(self._metric)
+            self._vector_query.metric(self._distance_type)
         if self._nprobes:
             self._vector_query.nprobes(self._nprobes)
         if self._refine_factor:
@@ -1386,6 +1414,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
     def metric(self, metric: Literal["L2", "cosine", "dot"]) -> LanceHybridQueryBuilder:
         """Set the distance metric to use.
 
+        This is an alias for distance_type() and may be deprecated in the future.
+        Please use distance_type() instead.
+
         Parameters
         ----------
         metric: "L2" or "cosine" or "dot"
@@ -1396,7 +1427,32 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         LanceVectorQueryBuilder
             The LanceQueryBuilder object.
         """
-        self._metric = metric.lower()
+        return self.distance_type(metric)
+
+    def distance_type(
+        self, distance_type: Literal["L2", "cosine", "dot"]
+    ) -> "LanceHybridQueryBuilder":
+        """Set the distance metric to use.
+
+        When performing a vector search we try and find the "nearest" vectors according
+        to some kind of distance metric. This parameter controls which distance metric
+        to use.
+
+        Note: if there is a vector index then the distance type used MUST match the
+        distance type used to train the vector index. If this is not done then the
+        results will be invalid.
+
+        Parameters
+        ----------
+        distance_type: "L2" or "cosine" or "dot"
+            The distance metric to use. By default "L2" is used.
+
+        Returns
+        -------
+        LanceVectorQueryBuilder
+            The LanceQueryBuilder object.
+        """
+        self._distance_type = distance_type.lower()
         return self
 
     def refine_factor(self, refine_factor: int) -> LanceHybridQueryBuilder:

python/python/lancedb/remote/__init__.py

@@ -1,15 +1,6 @@
-#  Copyright 2023 LanceDB Developers
+# SPDX-License-Identifier: Apache-2.0
-#
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-#  Licensed under the Apache License, Version 2.0 (the "License");
+
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 from dataclasses import dataclass, field
 from datetime import timedelta