Merge main into blob concurrency repair

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.40.0-beta.8"
+version = "0.40.0-beta.11"
 publish = false
 license.workspace = true
 description.workspace = true

nodejs/__test__/arrow.test.ts

@@ -601,6 +601,103 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         );
       });
 
+      it.each([
+        ["ArrayBuffer", new Uint8Array([104]).buffer],
+        ["URL", new URL("https://example.com/")],
+        ["empty object", {}],
+        [
+          "class instance with enumerable fields",
+          Object.assign(new Blob(["x"]), { position: 0 }),
+        ],
+      ])("rejects %s values without a schema", (_name, value) => {
+        expect(() => makeArrowTable([{ id: 1, value }])).toThrow(
+          /field value at row 0/,
+        );
+        expect(() => makeArrowTable([{ value }])).toThrow(
+          /field value at row 0/,
+        );
+      });
+
+      it.each([
+        ["Binary", new Binary(), new Uint8Array([104]).buffer],
+        ["Utf8", new Utf8(), new URL("https://example.com/")],
+      ])("rejects %s values with a schema", (_name, type, value) => {
+        const schema = new Schema([new Field("value", type, true)]);
+        expect(() => makeArrowTable([{ value }], { schema })).toThrow(
+          /field value at row 0/,
+        );
+      });
+
+      it("rejects unsupported values in nested struct fields", () => {
+        const schema = new Schema([
+          new Field(
+            "metadata",
+            new Struct([new Field("bytes", new Binary(), true)]),
+            true,
+          ),
+        ]);
+        expect(() =>
+          makeArrowTable(
+            [{ metadata: { bytes: new Uint8Array([104]).buffer } }],
+            { schema },
+          ),
+        ).toThrow(/field metadata\.bytes at row 0/);
+      });
+
+      it("rejects unsupported objects in list elements", () => {
+        const schema = new Schema([
+          new Field(
+            "images",
+            new List(new Field("item", new Binary(), true)),
+            true,
+          ),
+        ]);
+        expect(() =>
+          makeArrowTable([{ images: [new Blob(["abc"])] }], { schema }),
+        ).toThrow(/field images\[0\] at row 0/);
+      });
+
+      it("rejects unsupported objects in nested lists", () => {
+        const schema = new Schema([
+          new Field(
+            "images",
+            new List(
+              new Field(
+                "item",
+                new List(new Field("item", new Binary(), true)),
+                true,
+              ),
+            ),
+            true,
+          ),
+        ]);
+        expect(() =>
+          makeArrowTable([{ images: [[new Blob(["abc"])]] }], { schema }),
+        ).toThrow(/field images\[0\]\[0\] at row 0/);
+      });
+
+      it("rejects unsupported nested struct fields in lists", () => {
+        const schema = new Schema([
+          new Field(
+            "items",
+            new List(
+              new Field(
+                "item",
+                new Struct([new Field("bytes", new Binary(), true)]),
+                true,
+              ),
+            ),
+            true,
+          ),
+        ]);
+        expect(() =>
+          makeArrowTable(
+            [{ items: [{ bytes: new Uint8Array([104]).buffer }] }],
+            { schema },
+          ),
+        ).toThrow(/field items\[0\]\.bytes at row 0/);
+      });
+
       it("will ignore generated dictionary IDs when comparing inferred types", function () {
         const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
           dictionaryEncodeStrings: true,

nodejs/__test__/blob.test.ts

@@ -102,6 +102,27 @@ describe("BlobFile", () => {
 });
 
 describe("makeArrowTable blob columns", () => {
+  it.each([
+    ["ArrayBuffer", new Uint8Array([104]).buffer],
+    ["Blob", new Blob(["hello"])],
+    ["File", new File(["hello"], "hello.txt")],
+    ["URL", new URL("https://example.com/blob")],
+    ["ReadableStream", new ReadableStream()],
+    ["data struct with ArrayBuffer", { data: new Uint8Array([104]).buffer }],
+    ["uri struct with URL", { uri: new URL("https://example.com/blob") }],
+  ])("rejects %s input rather than writing a null blob", (_name, image) => {
+    const schema = new Schema([blob("image")]);
+    expect(() => makeArrowTable([{ image }], { schema })).toThrow(/image/);
+  });
+
+  it("rejects Blob instances with enumerable fields as one invalid value", () => {
+    const schema = new Schema([blob("image")]);
+    const image = Object.assign(new Blob(["hello"]), { position: 0 });
+    expect(() => makeArrowTable([{ image }], { schema })).toThrow(
+      "Unsupported object value for field image at row 0.",
+    );
+  });
+
   it("coerces Buffer input onto a blob field", () => {
     const schema = new Schema([
       new Field("id", new Int64(), true),

nodejs/__test__/materialized_view.test.ts

@@ -52,9 +52,19 @@ describe("materialized views", () => {
       "SELECT * FROM people",
     );
 
+    for (const [format, fn] of [
+      [3, "vector_duplicate_pairs"],
+      [4, "vector_dedup"],
+    ]) {
+      const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`;
+      expect(read(JSON.stringify({ format, query: native })).query).toBe(
+        native,
+      );
+    }
+
     // A newer writer's layout is reported, never guessed at.
     for (const newer of [
-      `{"format":3,"query":${JSON.stringify(query)}}`,
+      `{"format":5,"query":${JSON.stringify(query)}}`,
       '{"kind":"select_v3","source_table":"people"}',
     ]) {
       expect(() => read(newer)).toThrow(/cannot refresh/);

nodejs/__test__/remote.test.ts

@@ -85,6 +85,69 @@ async function withMockDatabase(
 }
 
 describe("remote connection", () => {
+  it("rejects the external blob opt-in without blocking regular adds", async () => {
+    let insertRequests = 0;
+    const describeRequests: string[] = [];
+
+    await withMockDatabase(
+      (req, res) => {
+        const requestPath = req.url ?? "";
+        if (requestPath.endsWith("/describe/")) {
+          describeRequests.push(requestPath);
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "items",
+              version: 1,
+              schema: {
+                fields: [
+                  {
+                    name: "id",
+                    type: { type: "int64" },
+                    nullable: true,
+                  },
+                ],
+              },
+            }),
+          );
+          return;
+        }
+        if (requestPath.endsWith("/insert/")) {
+          insertRequests++;
+          req.resume();
+          req.on("end", () => {
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(JSON.stringify({ version: insertRequests + 1 }));
+          });
+          return;
+        }
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        const table = await db.openTable("items");
+
+        await expect(
+          table.add([{ id: 1n }], {
+            allowExternalBlobOutsideBases: true,
+          }),
+        ).rejects.toThrow("only supported on local tables");
+        expect(insertRequests).toBe(0);
+
+        await expect(table.add([{ id: 2n }])).resolves.toMatchObject({
+          version: 2,
+        });
+        await expect(
+          table.add([{ id: 3n }], {
+            allowExternalBlobOutsideBases: false,
+          }),
+        ).resolves.toMatchObject({ version: 3 });
+      },
+    );
+
+    expect(describeRequests.length).toBeGreaterThan(0);
+    expect(insertRequests).toBe(2);
+  });
+
   it("lists materialized views through the namespace route", async () => {
     await withMockDatabase(
       (req, res) => {
@@ -161,6 +224,37 @@ describe("remote connection", () => {
     );
   });
 
+  it("reports the cleanup job when a view drop is accepted", async () => {
+    await withMockDatabase(
+      (req, res) => {
+        expect(req.method).toBe("POST");
+        expect(req.url).toBe("/v1/view/adults/drop");
+        res
+          .writeHead(202, { "content-type": "application/json" })
+          .end('{"job_id": "j1-do-abc"}');
+      },
+      async (db) => {
+        const job = await db.dropViewAsync("adults");
+        expect(job.id).toBe("j1-do-abc");
+      },
+    );
+  });
+
+  it("reports a finished job when a view drop had nothing to delete", async () => {
+    await withMockDatabase(
+      (req, res) => {
+        expect(req.url).toBe("/v1/view/adults/drop");
+        res.writeHead(200, { "content-type": "application/json" }).end("{}");
+      },
+      async (db) => {
+        // A 200 means the name was not bound, so there is no cleanup to wait on.
+        const job = await db.dropViewAsync("adults");
+        expect(job.id).toBeNull();
+        await job.wait();
+      },
+    );
+  });
+
   it("should accept partial connection options", async () => {
     await connect("db://test", {
       apiKey: "fake",
@@ -1161,6 +1255,24 @@ describe("remote connection jobs surface", () => {
             res
               .writeHead(200, { "Content-Type": "application/json" })
               .end('{"job_id": "job-1"}');
+          } else if (req.url === "/v1/jobs/pause") {
+            if (payload["job_id"] !== "job-1") {
+              res.writeHead(404).end("no such job");
+              return;
+            }
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end('{"job_id": "job-1", "paused": true}');
+          } else if (req.url === "/v1/jobs/resume") {
+            if (payload["job_id"] !== "job-1") {
+              res.writeHead(404).end("no such job");
+              return;
+            }
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(
+                '{"job_id": "job-1", "resumed": false, "still_pausing": true}',
+              );
           } else if (req.url === "/v1/jobs/query_events") {
             queryEventsPayloads.push(payload);
             res
@@ -1182,6 +1294,9 @@ describe("remote connection jobs surface", () => {
         expect(await db.cancelJob("job-1")).toBe(true);
         expect(await db.cancelJob("missing")).toBe(false);
 
+        expect(await db.pauseJob("job-1")).toEqual("pausing");
+        expect(await db.resumeJob("job-1")).toEqual("still_pausing");
+
         // Opening a job hands back a populated handle; a missing one rejects.
         await expect(db.openJob("missing")).rejects.toThrow("not found");
         const finished = await db.openJob("job-2");

nodejs/__test__/table.test.ts

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import * as fs from "fs";
+import { pathToFileURL } from "node:url";
 import * as path from "path";
 import * as tmp from "tmp";
 
@@ -416,6 +417,26 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(await table.countRows("id == 10")).toBe(1);
     });
 
+    it("should let me update float values to NaN and Infinity", async () => {
+      const db = await connect(tmpDir.name);
+      const floats = await db.createTable("floats", [
+        { id: 1, price: 1.5 },
+        { id: 2, price: 2.5 },
+        { id: 3, price: 3.5 },
+        { id: 4, price: 4.5 },
+      ]);
+      await floats.update({ values: { price: NaN }, where: "id = 1" });
+      await floats.update({ values: { price: Infinity }, where: "id = 2" });
+      await floats.update({ values: { price: -Infinity }, where: "id = 3" });
+
+      const rows = await floats.query().toArray();
+      const price = (id: number) => rows.find((row) => row.id === id).price;
+      expect(price(1)).toBeNaN();
+      expect(price(2)).toBe(Infinity);
+      expect(price(3)).toBe(-Infinity);
+      expect(price(4)).toBe(4.5);
+    });
+
     // https://github.com/lancedb/lancedb/issues/1293
     test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
       "can create empty table with non default float type: %s",
@@ -2412,6 +2433,135 @@ describe("when dealing with blob columns", () => {
     tmpDir.removeCallback();
   });
 
+  it("rejects unsupported blob values in createTable and add", async () => {
+    const db = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      blob("image"),
+    ]);
+    const row = { id: 1n, image: new Uint8Array([104]).buffer };
+
+    await expect(db.createTable("invalid", [row], { schema })).rejects.toThrow(
+      /field image at row 0/,
+    );
+
+    const table = await db.createEmptyTable("empty", schema);
+    await expect(table.add([row])).rejects.toThrow(/field image at row 0/);
+    await expect(table.countRows()).resolves.toBe(0);
+  });
+
+  it("requires an explicit opt-in for blob URIs outside registered bases", async () => {
+    const { table, uri } = await openExternalBlobTable();
+
+    await expect(table.add([{ id: 1n, payload: uri }])).rejects.toThrow(
+      "allow_external_blob_outside_bases",
+    );
+    await expect(table.countRows()).resolves.toBe(0);
+
+    await expect(
+      table.add([{ id: 2n, payload: uri }], {
+        allowExternalBlobOutsideBases: false,
+      }),
+    ).rejects.toThrow("allow_external_blob_outside_bases");
+    await expect(table.countRows()).resolves.toBe(0);
+  });
+
+  it.each([
+    ["URI string", (uri: string) => uri],
+    ["URI struct", (uri: string) => ({ uri })],
+  ])(
+    "round-trips an external blob from a %s after reopening",
+    async (_label, blobValue) => {
+      const { db, table, uri, payload } = await openExternalBlobTable();
+
+      const result = await table.add([{ id: 1n, payload: blobValue(uri) }], {
+        allowExternalBlobOutsideBases: true,
+      });
+      expect(result.version).toBeGreaterThan(0);
+      await expect(table.countRows()).resolves.toBe(1);
+
+      table.close();
+      const reopened = await db.openTable("external_blobs");
+      const rows = await reopened.query().withRowId().toArray();
+      expect(rows).toHaveLength(1);
+      expect(rows[0].payload.blob_uri).toBe(uri);
+      const bytes = await reopened.fetchBlobs("payload", [
+        rows[0]._rowid as bigint,
+      ]);
+      expect(bytes[0]).toEqual(payload);
+    },
+  );
+
+  it("applies the external blob opt-in to Arrow table input", async () => {
+    const { table, schema, uri, payload } = await openExternalBlobTable();
+    const input = makeArrowTable([{ id: 1n, payload: uri }], { schema });
+
+    const result = await table.add(input, {
+      allowExternalBlobOutsideBases: true,
+    });
+
+    expect(result.version).toBeGreaterThan(0);
+    const rows = await table.query().withRowId().toArray();
+    expect(rows[0].payload.blob_uri).toBe(uri);
+    const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
+    expect(bytes[0]).toEqual(payload);
+  });
+
+  it("applies the external blob opt-in only to the current add", async () => {
+    const { table, uri } = await openExternalBlobTable();
+
+    await table.add([{ id: 1n, payload: uri }], {
+      allowExternalBlobOutsideBases: true,
+    });
+    await expect(table.countRows()).resolves.toBe(1);
+
+    await expect(table.add([{ id: 2n, payload: uri }])).rejects.toThrow(
+      "allow_external_blob_outside_bases",
+    );
+    await expect(table.countRows()).resolves.toBe(1);
+  });
+
+  it("supports progress and overwrite with the external blob opt-in", async () => {
+    const { table, uri, payload } = await openExternalBlobTable();
+    await table.add([{ id: 1n, payload: Buffer.from("old") }]);
+
+    let resolveFinal!: (progress: import("../lancedb").WriteProgress) => void;
+    let rejectFinal!: (error: Error) => void;
+    const finalProgress = new Promise<import("../lancedb").WriteProgress>(
+      (resolve, reject) => {
+        resolveFinal = resolve;
+        rejectFinal = reject;
+      },
+    );
+    const timeout = setTimeout(
+      () => rejectFinal(new Error("timed out waiting for final progress")),
+      5_000,
+    );
+    try {
+      const result = await table.add([{ id: 2n, payload: uri }], {
+        mode: "overwrite",
+        allowExternalBlobOutsideBases: true,
+        progress: (progress) => {
+          if (progress.done) {
+            resolveFinal(progress);
+          }
+        },
+      });
+      expect(result.version).toBeGreaterThan(0);
+      const progress = await finalProgress;
+      expect(progress.outputRows).toBe(1);
+      expect(progress.totalRows).toBe(1);
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    const rows = await table.query().withRowId().toArray();
+    expect(rows).toHaveLength(1);
+    expect(rows[0].id).toBe(2n);
+    const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
+    expect(bytes[0]).toEqual(payload);
+  });
+
   it("discovers blob columns", async () => {
     const { table } = await openBlobTable();
     expect(await table.blobColumns()).toEqual(["image"]);
@@ -2671,6 +2821,20 @@ describe("when dealing with blob columns", () => {
     const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!);
     return { table, rowIds, alpha, beta };
   }
+
+  async function openExternalBlobTable() {
+    const db = await connect(path.join(tmpDir.name, "db"));
+    const payload = Buffer.from("external-payload");
+    const payloadPath = path.join(tmpDir.name, "payload.bin");
+    fs.writeFileSync(payloadPath, payload);
+    const uri = pathToFileURL(payloadPath).href;
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      blob("payload"),
+    ]);
+    const table = await db.createEmptyTable("external_blobs", schema);
+    return { db, table, schema, uri, payload };
+  }
 });
 
 describe("when dealing with tags", () => {

nodejs/__test__/util.test.ts

@@ -14,6 +14,9 @@ test.each([
   [new ArrayBuffer(8), "X'0000000000000000'"],
   [Buffer.from("hello"), "X'68656c6c6f'"],
   ["Hello 'world'", "'Hello ''world'''"],
+  [NaN, "CAST('NaN' AS DOUBLE)"],
+  [Infinity, "CAST('Infinity' AS DOUBLE)"],
+  [-Infinity, "CAST('-Infinity' AS DOUBLE)"],
 ])("toSQL(%p) === %p", (value, expected) => {
   expect(toSQL(value)).toBe(expected);
 });

nodejs/lancedb/connection.ts

@@ -404,12 +404,22 @@ export abstract class Connection {
   ): Promise<ViewDescription>;
 
   /**
-   * Drop the view named `name`.
+   * Drop the view named `name` and wait for its definition to be deleted.
    *
-   * The tables it reads are untouched: a view holds no rows of its own.
+   * The tables it reads are untouched: a view holds no rows of its own. Use
+   * {@link dropViewAsync} to retain the cleanup job instead of waiting on it.
    */
   abstract dropView(name: string, namespacePath?: string[]): Promise<void>;
 
+  /**
+   * Start dropping the view named `name` and return the job deleting its
+   * definition, without waiting for completion.
+   *
+   * The name is free before this resolves. When nothing was bound to it, the
+   * returned job is already finished and has no id.
+   */
+  abstract dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
+
   /**
    * The names of the views in one namespace.
    *
@@ -640,6 +650,24 @@ export abstract class Connection {
    * such job exists. Cancelling an already-terminal job is a no-op success.
    */
   abstract cancelJob(jobId: string): Promise<boolean>;
+
+  /**
+   * Pause a server-side job by id.
+   *
+   * The job's workers drain and it stays parked until resumed. Resolves to
+   * "pausing", "already_paused", or "committing" -- a job finalizing its
+   * results cannot be parked; retry shortly.
+   */
+  abstract pauseJob(jobId: string): Promise<string>;
+
+  /**
+   * Resume a paused server-side job by id.
+   *
+   * Its workers pick their work back up from checkpoints. Resolves to
+   * "resumed", "still_pausing" -- the pause's worker drain is not confirmed
+   * yet; retry shortly -- or "not_paused".
+   */
+  abstract resumeJob(jobId: string): Promise<string>;
 }
 
 /** @hideconstructor */
@@ -759,6 +787,10 @@ export class LocalConnection extends Connection {
     return this.inner.dropView(name, namespacePath ?? []);
   }
 
+  async dropViewAsync(name: string, namespacePath?: string[]): Promise<Job> {
+    return new Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
+  }
+
   async listViews(namespacePath?: string[]): Promise<string[]> {
     return this.inner.listViews(namespacePath ?? []);
   }
@@ -1035,6 +1067,14 @@ export class LocalConnection extends Connection {
   async cancelJob(jobId: string): Promise<boolean> {
     return this.inner.cancelJob(jobId);
   }
+
+  async pauseJob(jobId: string): Promise<string> {
+    return this.inner.pauseJob(jobId);
+  }
+
+  async resumeJob(jobId: string): Promise<string> {
+    return this.inner.resumeJob(jobId);
+  }
 }
 
 /**

nodejs/lancedb/materialized_view.ts

@@ -9,9 +9,10 @@ export const DEFINITION_META_KEY = "mv.definition";
 
 /**
  * The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
- * format 2 being a query with `GROUP BY`.
+ * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
+ * native dedup results.
  */
-export const DEFINITION_FORMAT = 2;
+export const DEFINITION_FORMAT = 4;
 
 /**
  * The query that defines a materialized view, as stored:

nodejs/lancedb/schema.ts

@@ -62,6 +62,8 @@ class SchemaInferrer {
   }
 
   private observe(path: string[], value: unknown, row: number): void {
+    assertSupportedValue(value, path.join("."), row);
+
     const current = this.fields.get(path);
     if (current === undefined) {
       this.addField(path, value, row);
@@ -430,19 +432,57 @@ function* recordPathsAndValues(
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  const prototype = Object.getPrototypeOf(value);
   return (
-    typeof value === "object" &&
-    value !== null &&
-    !Array.isArray(value) &&
-    !(value instanceof RegExp) &&
-    !(value instanceof Date) &&
-    !(value instanceof Set) &&
-    !(value instanceof Map) &&
-    !(value instanceof Buffer) &&
-    !ArrayBuffer.isView(value)
+    (prototype === Object.prototype || prototype === null) &&
+    Object.keys(value).length > 0
   );
 }
 
+function isUnsupportedObject(value: unknown): boolean {
+  if (
+    typeof value !== "object" ||
+    value === null ||
+    Array.isArray(value) ||
+    isRecord(value)
+  ) {
+    return false;
+  }
+  // Preserve the object types already passed through to Arrow. Other objects
+  // must not disappear while walking struct fields.
+  return !(
+    value instanceof RegExp ||
+    value instanceof Date ||
+    value instanceof Set ||
+    value instanceof Map ||
+    Buffer.isBuffer(value) ||
+    ArrayBuffer.isView(value)
+  );
+}
+
+function assertSupportedValue(
+  value: unknown,
+  field: string,
+  row: number,
+): void {
+  if (Array.isArray(value)) {
+    for (const [index, item] of value.entries()) {
+      assertSupportedValue(item, `${field}[${index}]`, row);
+    }
+  } else if (isRecord(value)) {
+    for (const [name, child] of Object.entries(value)) {
+      assertSupportedValue(child, `${field}.${name}`, row);
+    }
+  } else if (isUnsupportedObject(value)) {
+    throw new Error(
+      `Unsupported object value for field ${field} at row ${row}.`,
+    );
+  }
+}
+
 function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
   let fields = schema.fields;
   let field: Field | undefined;

nodejs/lancedb/table.ts

@@ -124,6 +124,17 @@ export interface AddDataOptions {
    * ```
    */
   progress: (progress: WriteProgress) => void;
+
+  /**
+   * Whether blob URIs outside registered external bases may be written.
+   *
+   * Defaults to `false`. This option is supported only for local/native
+   * tables; remote tables return an error when it is enabled. An enabled write
+   * stores the absolute URI reference without registering a base or copying
+   * the external object into the database. The object must remain accessible
+   * when the blob is read later.
+   */
+  allowExternalBlobOutsideBases?: boolean;
 }
 
 export interface UpdateOptions {
@@ -526,6 +537,10 @@ export abstract class Table {
    * Reads the table's current checkout. IDs from another version can fail after
    * compaction unless stable row ids are enabled. Results keep input order and
    * duplicates. Null blobs are `null`. Empty blobs are empty buffers.
+   * Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
+   * The client splits requests automatically and reads an individual larger
+   * blob through the Range route. This method still materializes all bytes in
+   * memory; use {@link Table.fetchBlobFiles} for large values.
    */
   abstract fetchBlobs(
     column: string,
@@ -1091,7 +1106,12 @@ export class LocalTable extends Table {
           }
         }
       : undefined;
-    return await this.inner.add(buffer, mode, progress);
+    return await this.inner.add(
+      buffer,
+      mode,
+      progress,
+      options?.allowExternalBlobOutsideBases,
+    );
   }
 
   async update(

nodejs/lancedb/util.ts

@@ -15,6 +15,14 @@ export function toSQL(value: IntoSql): string {
   if (typeof value === "string") {
     return `'${value.replace(/'/g, "''")}'`;
   } else if (typeof value === "number") {
+    // toString() gives "NaN" / "Infinity", which SQL reads as column names.
+    if (Number.isNaN(value)) {
+      return "CAST('NaN' AS DOUBLE)";
+    } else if (!Number.isFinite(value)) {
+      return value > 0
+        ? "CAST('Infinity' AS DOUBLE)"
+        : "CAST('-Infinity' AS DOUBLE)";
+    }
     return value.toString();
   } else if (typeof value === "boolean") {
     return value ? "TRUE" : "FALSE";

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.40.0-beta.8",
+	"version": "0.40.0-beta.11",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.40.0-beta.8",
+	"version": "0.40.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.40.0-beta.8",
+	"version": "0.40.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.40.0-beta.8",
+	"version": "0.40.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.40.0-beta.8",
+	"version": "0.40.0-beta.11",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.40.0-beta.8",
+  "version": "0.40.0-beta.11",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.40.0-beta.8",
+	"version": "0.40.0-beta.11",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.40.0-beta.8",
+  "version": "0.40.0-beta.11",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

nodejs/src/connection.rs

@@ -427,7 +427,8 @@ impl Connection {
         ViewDescription::from_inner(view)
     }
 
-    /// Drop a view. The tables it reads are untouched.
+    /// Drop a view and wait for its definition to be deleted. The tables it
+    /// reads are untouched.
     #[napi(catch_unwind)]
     pub async fn drop_view(
         &self,
@@ -441,6 +442,22 @@ impl Connection {
             .default_error()
     }
 
+    /// Start dropping a view and return the job deleting its definition.
+    #[napi(catch_unwind)]
+    pub async fn drop_view_async(
+        &self,
+        name: String,
+        namespace_path: Option<Vec<String>>,
+    ) -> napi::Result<crate::job::Job> {
+        let ns = namespace_path.unwrap_or_default();
+        let job = self
+            .get_inner()?
+            .drop_view_async(&name, &ns)
+            .await
+            .default_error()?;
+        Ok(crate::job::Job::new(job))
+    }
+
     /// The names of the views in one namespace.
     #[napi(catch_unwind)]
     pub async fn list_views(
@@ -583,6 +600,34 @@ impl Connection {
         self.get_inner()?.cancel_job(&job_id).await.default_error()
     }
 
+    /// Pause a server-side job by id: its workers drain and it stays parked
+    /// until resumed. Returns "pausing", "already_paused", or "committing".
+    #[napi(catch_unwind)]
+    pub async fn pause_job(&self, job_id: String) -> napi::Result<String> {
+        let status = self.get_inner()?.pause_job(&job_id).await.default_error()?;
+        Ok(match status {
+            lancedb::database::PauseJobStatus::Pausing => "pausing".to_string(),
+            lancedb::database::PauseJobStatus::AlreadyPaused => "already_paused".to_string(),
+            lancedb::database::PauseJobStatus::Committing => "committing".to_string(),
+        })
+    }
+
+    /// Resume a paused server-side job by id. Returns "resumed",
+    /// "still_pausing", or "not_paused".
+    #[napi(catch_unwind)]
+    pub async fn resume_job(&self, job_id: String) -> napi::Result<String> {
+        let status = self
+            .get_inner()?
+            .resume_job(&job_id)
+            .await
+            .default_error()?;
+        Ok(match status {
+            lancedb::database::ResumeJobStatus::Resumed => "resumed".to_string(),
+            lancedb::database::ResumeJobStatus::StillPausing => "still_pausing".to_string(),
+            lancedb::database::ResumeJobStatus::NotPaused => "not_paused".to_string(),
+        })
+    }
+
     #[napi(catch_unwind)]
     /// Describe a namespace and return its properties.
     pub async fn describe_namespace(

nodejs/src/table.rs

@@ -74,13 +74,14 @@ impl Table {
 
     #[napi(
         catch_unwind,
-        ts_args_type = "buf: Buffer, mode: string, progressCallback?: (progress: WriteProgressInfo) => void"
+        ts_args_type = "buf: Buffer, mode: string, progressCallback?: (progress: WriteProgressInfo) => void, allowExternalBlobOutsideBases?: boolean"
     )]
     pub async fn add(
         &self,
         buf: Buffer,
         mode: String,
         progress_callback: Option<ProgressFn>,
+        allow_external_blob_outside_bases: Option<bool>,
     ) -> napi::Result<AddResult> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
@@ -105,6 +106,9 @@ impl Table {
             return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode)));
         };
 
+        op = op
+            .allow_external_blob_outside_bases(allow_external_blob_outside_bases.unwrap_or(false));
+
         if let Some(tsfn) = progress_callback {
             op = op.progress(move |p| {
                 // NonBlocking: dispatch onto the JS event loop without