feat(nodejs): add col() expression builder for camelCase column names

## Summary

Fixes #3211 by exposing the shared Rust/DataFusion expression builder through Node bindings. Case-sensitive references such as `col("firstName")` can be composed and passed to `where()`, while SQL strings remain supported.

The expression-object representation follows [the design requested on #3290](https://github.com/lancedb/lancedb/pull/3290#discussion_r3632371998). That earlier PR remains open; no claim is made that the issue was unclaimed or that maintainers have selected this implementation.

## Review corrections

- Construct the named native struct correctly and parse casts through `arrow_schema::DataType`.
- Preserve numeric values: use Int64 only for JavaScript safe integers; retain Float64 otherwise. Explicit `litInt` rejects unsupported numbers with a documented `RangeError`.
- Correct test syntax/API calls and align SQL assertions with the shared dialect, including reserved and case-sensitive names.
- Use typed, isolated temporary test tables with cleanup, and cover an actual large-number filter threshold.
- Document the public methods and commit generated API documentation.

Synced with current main `0bceaaf3` through a normal merge. The diff is limited to seven Node source/test/config files and eleven generated documentation files; no Rust core change.

## Validation

[Linux validation run](https://github.com/Ganesh1729-ui/lancedb/actions/runs/38244794288) on Node 24 / Rust 1.97.0 with the repository's `ci` profile and `remote,lancedb/metrics-otel` features:

- Rebuilt native addon, TypeScript build, Biome, Rust formatting and native Clippy pass.
- Expression/query/Arrow/table suites: **1,129 passed**; expression suite repeated: **42 passed**.
- Using the original JavaScript numeric dispatch with the rebuilt addon: **8 regressions fail, 2 controls pass**.
- Strict TypeDoc generation passes. Final local typing/docs/lint checks pass; the PR source equals the validated source except for the separate validation workflow, which is excluded from this PR.

Full provider/model/S3, other feature combinations and OS/Node matrices were not run in this follow-up. Upstream checks and maintainer review remain separate.

## AI assistance

AI was used for implementation, review and testing. No prior human review is claimed.
This commit is contained in:
J Ganesh Kumar Reddy
2026-10-29 02:18:16 +08:00
committed by GitHub
parent ece54d8917
commit 12d86ea251
20 changed files with 1406 additions and 34 deletions
+346
View File
@@ -0,0 +1,346 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as http from "http";
import type { AddressInfo } from "net";
import * as tmp from "tmp";
import { type Table, connect } from "../lancedb";
import {
Table as ArrowTable,
Field,
Int64,
Schema,
Utf8,
tableToIPC,
} from "../lancedb/arrow";
import { Expr, col, func, lit, litBool, litInt } from "../lancedb/expr";
describe("expr", () => {
describe("col", () => {
it("returns an Expr instance", () => {
const e = col("age");
expect(e).toBeInstanceOf(Expr);
});
it("preserves camelCase in toSql()", () => {
expect(col("firstName").toSql()).toBe("`firstName`");
});
it("preserves snake_case without quotes", () => {
expect(col("first_name").toSql()).toBe("first_name");
});
it("Quotes identifiers with spaces", () => {
expect(col("first name").toSql()).toBe("`first name`");
});
it("Quotes identifiers with leading digits", () => {
expect(col("2fast").toSql()).toBe("`2fast`");
});
it("Quotes unicode identifiers", () => {
expect(col("名前").toSql()).toBe("`名前`");
});
});
describe("lit", () => {
it("creates a string literal", () => {
expect(lit("hello").toSql()).toBe("'hello'");
});
it("creates an integer literal", () => {
expect(lit(42).toSql()).toBe("42");
});
it("creates a float literal", () => {
expect(lit(3.14).toSql()).toBe("3.14");
});
});
describe("numeric literals", () => {
it.each([Number.MIN_SAFE_INTEGER, Number.MAX_SAFE_INTEGER])(
"preserves the safe integer boundary %s",
(value) => {
expect(lit(value).toSql()).toBe(String(value));
expect(litInt(value).toSql()).toBe(String(value));
},
);
it.each([1e20, -1e20])("preserves the large number %s", (value) => {
expect(Number(lit(value).toSql())).toBe(value);
expect(col("price").gt(value).toSql()).toBe(
col("price").gt(lit(value)).toSql(),
);
});
it.each([1.5, 1e20, -1e20, Number.NaN, Number.POSITIVE_INFINITY])(
"rejects %s as an explicit integer literal",
(value) => expect(() => litInt(value)).toThrow(RangeError),
);
});
describe("Expr operators", () => {
it("eq", () => {
expect(col("x").eq(lit(1)).toSql()).toBe("(x = 1)");
});
it("ne", () => {
expect(col("x").ne(lit(1)).toSql()).toBe("(x <> 1)");
});
it("lt", () => {
expect(col("age").lt(lit(18)).toSql()).toBe("(age < 18)");
});
it("lte", () => {
expect(col("age").lte(lit(18)).toSql()).toBe("(age <= 18)");
});
it("gt", () => {
expect(col("age").gt(lit(18)).toSql()).toBe("(age > 18)");
});
it("gte", () => {
expect(col("age").gte(lit(18)).toSql()).toBe("(age >= 18)");
});
it("and", () => {
const e = col("age")
.gt(lit(18))
.and(col("status").eq(lit("active")));
expect(e.toSql()).toBe("((age > 18) AND (`status` = 'active'))");
});
it("or", () => {
const e = col("a")
.eq(lit(1))
.or(col("b").eq(lit(2)));
expect(e.toSql()).toBe("((a = 1) OR (b = 2))");
});
it("not", () => {
const e = col("active").eq(litBool(true)).not();
expect(e.toSql()).toBe("NOT (active = true)");
});
it("add", () => {
expect(col("x").add(lit(1)).toSql()).toBe("(x + 1)");
});
it("sub", () => {
expect(col("x").sub(lit(1)).toSql()).toBe("(x - 1)");
});
it("mul", () => {
expect(col("price").mul(lit(1.1)).toSql()).toBe("(price * 1.1)");
});
it("div", () => {
expect(col("total").div(lit(2)).toSql()).toBe("(total / 2)");
});
});
describe("Expr string methods", () => {
it("lower", () => {
expect(col("name").lower().toSql()).toBe("lower(`name`)");
});
it("upper", () => {
expect(col("name").upper().toSql()).toBe("upper(`name`)");
});
it("contains", () => {
expect(col("text").contains(lit("hello")).toSql()).toBe(
"contains(`text`, 'hello')",
);
});
});
describe("membership and casts", () => {
it("accepts scalar list members", () => {
expect(col("age").isIn([18, 21]).toSql()).toBe("age IN (18, 21)");
});
it("casts to an Arrow type", () => {
expect(col("age").cast("Int64").toSql()).toBe("arrow_cast(age, 'Int64')");
});
it("rejects an invalid Arrow type", () => {
expect(() => col("age").cast("invalid")).toThrow("Invalid data type");
});
});
describe("func", () => {
it("lower", () => {
expect(func("lower", col("name")).toSql()).toBe("lower(`name`)");
});
});
describe("camelCase integration", () => {
let db: Awaited<ReturnType<typeof connect>>;
let table: Table;
let tmpDir: tmp.DirResult;
beforeAll(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
db = await connect(tmpDir.name);
table = await db.createTable("test", [
{ id: 1, firstName: "Alice", lastName: "Smith" },
{ id: 2, firstName: "Bob", lastName: "Jones" },
{ id: 3, firstName: "Charlie", lastName: "Brown" },
]);
});
afterAll(() => {
table.close();
db.close();
tmpDir.removeCallback();
});
it("filters on camelCase column", async () => {
const results = await table
.query()
.where(col("firstName").eq(lit("Alice")))
.toArray();
expect(results.length).toBe(1);
expect(results[0].firstName).toBe("Alice");
});
it("filters on multiple camelCase columns", async () => {
const results = await table
.query()
.where(
col("firstName")
.eq(lit("Bob"))
.and(col("lastName").eq(lit("Jones"))),
)
.toArray();
expect(results.length).toBe(1);
expect(results[0].firstName).toBe("Bob");
});
it("preserves an unsafe integer filter threshold", async () => {
const prices = await db.createTable("prices", [
{ price: 9.5e18 },
{ price: 1.1e20 },
]);
try {
const results = await prices
.query()
.where(col("price").gt(1e20))
.toArray();
expect(results.map((row) => row.price)).toEqual([1.1e20]);
} finally {
prices.close();
}
});
it("filters with gt on camelCase column", async () => {
const results = await table
.query()
.where(col("id").gt(lit(1)))
.toArray();
expect(results.length).toBe(2);
});
it.each([
["backslash", "\\"],
["quote", "'"],
["backslash quote", "\\'"],
["OR true payload", "\\') OR true --"],
["line comment payload", "\\' -- OR true"],
["block comment payload", "\\') /* OR true */ --"],
])("filters the exact %s value", async (label, value) => {
const escaped = await db.createTable(
`escaped_${label.replaceAll(" ", "_")}`,
[
{ id: 1, tenant: value },
{ id: 2, tenant: "different tenant" },
],
);
try {
const results = await escaped
.query()
.where(col("tenant").eq(lit(value)))
.toArray();
expect(results).toHaveLength(1);
expect(results[0].id).toBe(1);
expect(results[0].tenant).toBe(value);
} finally {
escaped.close();
}
});
});
it("sends a quoted expression value to remote as one literal", async () => {
const value = "\\') OR true --";
const schema = new Schema([
new Field("id", new Int64(), false),
new Field("tenant", new Utf8(), false),
]);
const response = Buffer.from(tableToIPC(new ArrowTable(schema), "stream"));
let remoteFilter: unknown;
const server = http.createServer((req, res) => {
const path = req.url ?? "";
if (path.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "tenants",
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: false },
{ name: "tenant", type: { type: "string" }, nullable: false },
],
},
}),
);
return;
}
if (path.endsWith("/query/")) {
let body = "";
req.on("data", (chunk) => {
body += chunk;
});
req.on("end", () => {
remoteFilter = JSON.parse(body).filter;
res
.writeHead(200, {
"Content-Type": "application/vnd.apache.arrow.stream",
})
.end(response);
});
return;
}
res.writeHead(404).end();
});
await new Promise<void>((resolve) =>
server.listen(0, "127.0.0.1", resolve),
);
const address = server.address() as AddressInfo;
try {
const db = await connect("db://dev", {
apiKey: "fake",
hostOverride: `http://127.0.0.1:${address.port}`,
});
try {
const remoteTable = await db.openTable("tenants");
await remoteTable
.query()
.where(col("tenant").eq(lit(value)))
.toArrow();
} finally {
db.close();
}
} finally {
await new Promise<void>((resolve) => {
server.closeAllConnections();
server.close(() => resolve());
});
}
expect(remoteFilter).toBe("(tenant = '\\'') OR true --')");
});
});
+182
View File
@@ -0,0 +1,182 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
Expr as NativeExpr,
col as nativeCol,
func as nativeFunc,
lit as nativeLit,
litBool as nativeLitBool,
litInt as nativeLitInt,
} from "./native.js";
export { Expr as NativeExpr } from "./native";
/** Values accepted by expression operators. */
export type ExprLike = Expr | string | number | boolean;
function coerce(value: ExprLike): Expr {
if (value instanceof Expr) {
return value;
}
if (typeof value === "string") {
return lit(value);
}
if (typeof value === "boolean") {
return litBool(value);
}
if (typeof value === "number") {
return lit(value);
}
throw new TypeError(`Unsupported literal type: ${typeof value}`);
}
/**
* A SQL expression backed by the shared LanceDB expression builder.
*
* @example
* ```ts
* table.query().where(col("firstName").eq("Alice")).toArray();
* ```
*/
export class Expr {
inner: NativeExpr;
constructor(inner: NativeExpr) {
this.inner = inner;
}
/** Compare for equality. */
eq(other: ExprLike): Expr {
return new Expr(this.inner.eq(coerce(other).inner));
}
/** Compare for inequality. */
ne(other: ExprLike): Expr {
return new Expr(this.inner.ne(coerce(other).inner));
}
/** Compare for less than. */
lt(other: ExprLike): Expr {
return new Expr(this.inner.lt(coerce(other).inner));
}
/** Compare for less than or equal to. */
lte(other: ExprLike): Expr {
return new Expr(this.inner.lte(coerce(other).inner));
}
/** Compare for greater than. */
gt(other: ExprLike): Expr {
return new Expr(this.inner.gt(coerce(other).inner));
}
/** Compare for greater than or equal to. */
gte(other: ExprLike): Expr {
return new Expr(this.inner.gte(coerce(other).inner));
}
/** Combine two predicates with AND. */
and(other: Expr): Expr {
return new Expr(this.inner.and(other.inner));
}
/** Combine two predicates with OR. */
or(other: Expr): Expr {
return new Expr(this.inner.or(other.inner));
}
/** Negate this predicate. */
not(): Expr {
return new Expr(this.inner.not());
}
/** Add a value. */
add(other: ExprLike): Expr {
return new Expr(this.inner.add(coerce(other).inner));
}
/** Subtract a value. */
sub(other: ExprLike): Expr {
return new Expr(this.inner.sub(coerce(other).inner));
}
/** Multiply by a value. */
mul(other: ExprLike): Expr {
return new Expr(this.inner.mul(coerce(other).inner));
}
/** Divide by a value. */
div(other: ExprLike): Expr {
return new Expr(this.inner.div(coerce(other).inner));
}
/** Convert a string to lowercase. */
lower(): Expr {
return new Expr(this.inner.lower());
}
/** Convert a string to uppercase. */
upper(): Expr {
return new Expr(this.inner.upper());
}
/** Test whether a string contains a substring. */
contains(substr: ExprLike): Expr {
return new Expr(this.inner.contains(coerce(substr).inner));
}
/** Test membership in a list of values. */
isIn(values: ExprLike[]): Expr {
return new Expr(this.inner.isIn(values.map((v) => coerce(v).inner)));
}
/** Cast to an Arrow data type, such as "Int64". */
cast(dataType: string): Expr {
return new Expr(this.inner.cast(dataType));
}
/** Serialize with the LanceDB SQL dialect, preserving column case. */
toSql(): string {
return this.inner.toSql();
}
}
/** Reference a column by its exact, case-sensitive name. */
export function col(name: string): Expr {
return new Expr(nativeCol(name));
}
/** Create a string or numeric literal. Unsafe integers are represented as Float64. */
export function lit(value: string | number): Expr {
if (typeof value === "string") {
return new Expr(nativeLit(value));
}
if (Number.isSafeInteger(value)) {
return new Expr(nativeLitInt(value));
}
return new Expr(nativeLit(value));
}
/** Create a boolean literal. */
export function litBool(value: boolean): Expr {
return new Expr(nativeLitBool(value));
}
/** Create an Int64 literal from a JavaScript safe integer. Other numbers throw RangeError. */
export function litInt(value: number): Expr {
if (!Number.isSafeInteger(value)) {
throw new RangeError("Integer literals must be safe integers");
}
return new Expr(nativeLitInt(value));
}
/** Call a registered SQL function, such as `func("lower", col("firstName"))`. */
export function func(name: string, ...args: ExprLike[]): Expr {
return new Expr(
nativeFunc(
name,
args.map((a) => coerce(a).inner),
),
);
}
+2
View File
@@ -207,6 +207,8 @@ export {
MultiVector,
} from "./arrow";
export { IntoSql, packBits } from "./util";
export { Expr, col, lit, litBool, litInt, func } from "./expr";
export type { ExprLike } from "./expr";
/**
* Options for tokenizing a full-text search query without a table index.
+15 -3
View File
@@ -17,6 +17,7 @@ import {
tableFromIPC,
vectorFromArray,
} from "./arrow";
import { Expr } from "./expr";
import { type IvfPqOptions } from "./indices";
import {
JsFullTextQuery,
@@ -649,11 +650,16 @@ export class StandardQueryBase<
/**
* A filter statement to be applied to this query.
*
* The filter should be supplied as an SQL query string. For example:
* The filter may be an SQL query string or an expression built with `col`,
* which preserves case-sensitive column names. For example:
* @example
* x > 10
* y > 0 AND y < 100
* x > 5 OR y = 'test'
* @example
* ```ts
* table.query().where(col("firstName").eq("Alice"));
* ```
*
* Filtering performance can often be improved by creating a scalar index
* on the filter column(s).
@@ -661,8 +667,14 @@ export class StandardQueryBase<
* Calling this multiple times combines the filters with a logical AND rather
* than replacing the previous filter.
*/
where(predicate: string): this {
this.doCall((inner: NativeQueryType) => inner.onlyIf(predicate));
where(predicate: string | Expr): this {
this.doCall((inner: NativeQueryType) => {
if (predicate instanceof Expr) {
inner.onlyIf(predicate.toSql());
} else {
inner.onlyIf(predicate);
}
});
return this;
}
/**
+221
View File
@@ -0,0 +1,221 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Napi bindings for the LanceDB expression builder API.
use std::ops::{Add, Div, Mul, Not, Sub};
use lancedb::expr::{
DfExpr, col as ldb_col, contains, expr_cast, is_in, lit as df_lit, lower, upper,
};
use napi::bindgen_prelude::*;
use napi_derive::napi;
#[napi]
#[derive(Clone)]
pub struct Expr {
pub(crate) inner: DfExpr,
}
#[napi]
impl Expr {
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn eq(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().eq(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn ne(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().not_eq(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn lt(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().lt(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn lte(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().lt_eq(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn gt(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().gt(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn gte(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().gt_eq(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn and(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().and(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn or(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().or(other.inner.clone()),
}
}
#[napi]
pub fn not(&self) -> Self {
Self {
inner: self.inner.clone().not(),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn add(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().add(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn sub(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().sub(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn mul(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().mul(other.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn div(&self, other: &Expr) -> Self {
Self {
inner: self.inner.clone().div(other.inner.clone()),
}
}
#[napi]
pub fn lower(&self) -> Self {
Self {
inner: lower(self.inner.clone()),
}
}
#[napi]
pub fn upper(&self) -> Self {
Self {
inner: upper(self.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn contains(&self, substr: &Expr) -> Self {
Self {
inner: contains(self.inner.clone(), substr.inner.clone()),
}
}
#[napi]
#[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it
pub fn is_in(&self, list: Vec<&Expr>) -> Self {
let items: Vec<DfExpr> = list.into_iter().map(|e| e.inner.clone()).collect();
Self {
inner: is_in(self.inner.clone(), items),
}
}
#[napi]
pub fn cast(&self, data_type: String) -> napi::Result<Self> {
let arrow_type = data_type
.parse::<arrow_schema::DataType>()
.map_err(|e| napi::Error::from_reason(format!("Invalid data type: {}", e)))?;
Ok(Self {
inner: expr_cast(self.inner.clone(), arrow_type),
})
}
#[napi]
pub fn to_sql(&self) -> napi::Result<String> {
lancedb::expr::expr_to_sql_string(&self.inner)
.map_err(|e| napi::Error::from_reason(e.to_string()))
}
}
/// Create a column reference expression.
///
/// The column name is preserved exactly as given (case-sensitive), so
/// `col("firstName")` correctly references a field named `firstName`.
#[napi]
#[allow(dead_code)]
pub fn col(name: String) -> Expr {
Expr {
inner: ldb_col(name),
}
}
/// Create a literal (constant) value expression.
#[napi]
#[allow(dead_code)]
pub fn lit(value: napi::bindgen_prelude::Either<String, f64>) -> napi::Result<Expr> {
match value {
Either::A(s) => Ok(Expr { inner: df_lit(s) }),
Either::B(f) => Ok(Expr { inner: df_lit(f) }),
}
}
/// Create a literal boolean value expression.
#[napi]
#[allow(dead_code)]
pub fn lit_bool(value: bool) -> Expr {
Expr {
inner: df_lit(value),
}
}
/// Create a literal integer value expression.
#[napi]
#[allow(dead_code)]
pub fn lit_int(value: i64) -> Expr {
Expr {
inner: df_lit(value),
}
}
/// Call an arbitrary registered SQL function by name.
#[napi]
#[allow(dead_code)]
pub fn func(name: String, args: Vec<&Expr>) -> napi::Result<Expr> {
let df_args: Vec<DfExpr> = args.into_iter().map(|e| e.inner.clone()).collect();
lancedb::expr::func(&name, df_args)
.map(|inner| Expr { inner })
.map_err(|e| napi::Error::from_reason(e.to_string()))
}
+1
View File
@@ -14,6 +14,7 @@ mod blob;
mod catalog;
mod connection;
mod error;
mod expr;
mod header;
mod index;
mod iterator;
+1
View File
@@ -1,6 +1,7 @@
{
"intentionallyNotExported": [
"lancedb/native.d.ts:Query",
"lancedb/native.d.ts:Expr",
"lancedb/native.d.ts:VectorQuery",
"lancedb/native.d.ts:TakeQuery",
"lancedb/native.d.ts:RecordBatchIterator",