diff --git a/docs/src/js/classes/AutoQuery.md b/docs/src/js/classes/AutoQuery.md index f12668b90..60f568e07 100644 --- a/docs/src/js/classes/AutoQuery.md +++ b/docs/src/js/classes/AutoQuery.md @@ -469,29 +469,34 @@ where(predicate): this A filter statement to be applied to this query. -The filter should be supplied as an SQL query string. For example: +The filter may be an SQL query string or an expression built with `col`, +which preserves case-sensitive column names. For example: #### Parameters -* **predicate**: `string` +* **predicate**: `string` \| [`Expr`](Expr.md) #### Returns `this` -#### Example +#### Examples ```ts x > 10 y > 0 AND y < 100 x > 5 OR y = 'test' +``` + +```ts +table.query().where(col("firstName").eq("Alice")); +``` Filtering performance can often be improved by creating a scalar index on the filter column(s). Calling this multiple times combines the filters with a logical AND rather than replacing the previous filter. -``` #### Inherited from diff --git a/docs/src/js/classes/Expr.md b/docs/src/js/classes/Expr.md new file mode 100644 index 000000000..7342cf793 --- /dev/null +++ b/docs/src/js/classes/Expr.md @@ -0,0 +1,365 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / Expr + +# Class: Expr + +A SQL expression backed by the shared LanceDB expression builder. + +## Example + +```ts +table.query().where(col("firstName").eq("Alice")).toArray(); +``` + +## Constructors + +### new Expr() + +```ts +new Expr(inner): Expr +``` + +#### Parameters + +* **inner**: `Expr` + +#### Returns + +[`Expr`](Expr.md) + +## Properties + +### inner + +```ts +inner: Expr; +``` + +## Methods + +### add() + +```ts +add(other): Expr +``` + +Add a value. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### and() + +```ts +and(other): Expr +``` + +Combine two predicates with AND. + +#### Parameters + +* **other**: [`Expr`](Expr.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### cast() + +```ts +cast(dataType): Expr +``` + +Cast to an Arrow data type, such as "Int64". + +#### Parameters + +* **dataType**: `string` + +#### Returns + +[`Expr`](Expr.md) + +*** + +### contains() + +```ts +contains(substr): Expr +``` + +Test whether a string contains a substring. + +#### Parameters + +* **substr**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### div() + +```ts +div(other): Expr +``` + +Divide by a value. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### eq() + +```ts +eq(other): Expr +``` + +Compare for equality. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### gt() + +```ts +gt(other): Expr +``` + +Compare for greater than. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### gte() + +```ts +gte(other): Expr +``` + +Compare for greater than or equal to. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### isIn() + +```ts +isIn(values): Expr +``` + +Test membership in a list of values. + +#### Parameters + +* **values**: [`ExprLike`](../type-aliases/ExprLike.md)[] + +#### Returns + +[`Expr`](Expr.md) + +*** + +### lower() + +```ts +lower(): Expr +``` + +Convert a string to lowercase. + +#### Returns + +[`Expr`](Expr.md) + +*** + +### lt() + +```ts +lt(other): Expr +``` + +Compare for less than. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### lte() + +```ts +lte(other): Expr +``` + +Compare for less than or equal to. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### mul() + +```ts +mul(other): Expr +``` + +Multiply by a value. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### ne() + +```ts +ne(other): Expr +``` + +Compare for inequality. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### not() + +```ts +not(): Expr +``` + +Negate this predicate. + +#### Returns + +[`Expr`](Expr.md) + +*** + +### or() + +```ts +or(other): Expr +``` + +Combine two predicates with OR. + +#### Parameters + +* **other**: [`Expr`](Expr.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### sub() + +```ts +sub(other): Expr +``` + +Subtract a value. + +#### Parameters + +* **other**: [`ExprLike`](../type-aliases/ExprLike.md) + +#### Returns + +[`Expr`](Expr.md) + +*** + +### toSql() + +```ts +toSql(): string +``` + +Serialize with the LanceDB SQL dialect, preserving column case. + +#### Returns + +`string` + +*** + +### upper() + +```ts +upper(): Expr +``` + +Convert a string to uppercase. + +#### Returns + +[`Expr`](Expr.md) diff --git a/docs/src/js/classes/Query.md b/docs/src/js/classes/Query.md index 8e216d1f4..7434e7b8e 100644 --- a/docs/src/js/classes/Query.md +++ b/docs/src/js/classes/Query.md @@ -543,29 +543,34 @@ where(predicate): this A filter statement to be applied to this query. -The filter should be supplied as an SQL query string. For example: +The filter may be an SQL query string or an expression built with `col`, +which preserves case-sensitive column names. For example: #### Parameters -* **predicate**: `string` +* **predicate**: `string` \| [`Expr`](Expr.md) #### Returns `this` -#### Example +#### Examples ```ts x > 10 y > 0 AND y < 100 x > 5 OR y = 'test' +``` + +```ts +table.query().where(col("firstName").eq("Alice")); +``` Filtering performance can often be improved by creating a scalar index on the filter column(s). Calling this multiple times combines the filters with a logical AND rather than replacing the previous filter. -``` #### Inherited from diff --git a/docs/src/js/classes/VectorQuery.md b/docs/src/js/classes/VectorQuery.md index 04ced2ef4..d98f05893 100644 --- a/docs/src/js/classes/VectorQuery.md +++ b/docs/src/js/classes/VectorQuery.md @@ -825,29 +825,34 @@ where(predicate): this A filter statement to be applied to this query. -The filter should be supplied as an SQL query string. For example: +The filter may be an SQL query string or an expression built with `col`, +which preserves case-sensitive column names. For example: #### Parameters -* **predicate**: `string` +* **predicate**: `string` \| [`Expr`](Expr.md) #### Returns `this` -#### Example +#### Examples ```ts x > 10 y > 0 AND y < 100 x > 5 OR y = 'test' +``` + +```ts +table.query().where(col("firstName").eq("Alice")); +``` Filtering performance can often be improved by creating a scalar index on the filter column(s). Calling this multiple times combines the filters with a logical AND rather than replacing the previous filter. -``` #### Inherited from diff --git a/docs/src/js/functions/col.md b/docs/src/js/functions/col.md new file mode 100644 index 000000000..6cff91cf7 --- /dev/null +++ b/docs/src/js/functions/col.md @@ -0,0 +1,21 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / col + +# Function: col() + +```ts +function col(name): Expr +``` + +Reference a column by its exact, case-sensitive name. + +## Parameters + +* **name**: `string` + +## Returns + +[`Expr`](../classes/Expr.md) diff --git a/docs/src/js/functions/func.md b/docs/src/js/functions/func.md new file mode 100644 index 000000000..05f6901aa --- /dev/null +++ b/docs/src/js/functions/func.md @@ -0,0 +1,23 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / func + +# Function: func() + +```ts +function func(name, ...args): Expr +``` + +Call a registered SQL function, such as `func("lower", col("firstName"))`. + +## Parameters + +* **name**: `string` + +* ...**args**: [`ExprLike`](../type-aliases/ExprLike.md)[] + +## Returns + +[`Expr`](../classes/Expr.md) diff --git a/docs/src/js/functions/lit.md b/docs/src/js/functions/lit.md new file mode 100644 index 000000000..c9db2cf4d --- /dev/null +++ b/docs/src/js/functions/lit.md @@ -0,0 +1,21 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / lit + +# Function: lit() + +```ts +function lit(value): Expr +``` + +Create a string or numeric literal. Unsafe integers are represented as Float64. + +## Parameters + +* **value**: `string` \| `number` + +## Returns + +[`Expr`](../classes/Expr.md) diff --git a/docs/src/js/functions/litBool.md b/docs/src/js/functions/litBool.md new file mode 100644 index 000000000..d857d8b74 --- /dev/null +++ b/docs/src/js/functions/litBool.md @@ -0,0 +1,21 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / litBool + +# Function: litBool() + +```ts +function litBool(value): Expr +``` + +Create a boolean literal. + +## Parameters + +* **value**: `boolean` + +## Returns + +[`Expr`](../classes/Expr.md) diff --git a/docs/src/js/functions/litInt.md b/docs/src/js/functions/litInt.md new file mode 100644 index 000000000..6556f7572 --- /dev/null +++ b/docs/src/js/functions/litInt.md @@ -0,0 +1,21 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / litInt + +# Function: litInt() + +```ts +function litInt(value): Expr +``` + +Create an Int64 literal from a JavaScript safe integer. Other numbers throw RangeError. + +## Parameters + +* **value**: `number` + +## Returns + +[`Expr`](../classes/Expr.md) diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index e3830e9ad..25a32b29c 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -28,6 +28,7 @@ - [Catalog](classes/Catalog.md) - [Connection](classes/Connection.md) - [DatabaseNames](classes/DatabaseNames.md) +- [Expr](classes/Expr.md) - [HeaderProvider](classes/HeaderProvider.md) - [Index](classes/Index.md) - [Job](classes/Job.md) @@ -165,6 +166,7 @@ - [Data](type-aliases/Data.md) - [DataLike](type-aliases/DataLike.md) - [DataTypeLike](type-aliases/DataTypeLike.md) +- [ExprLike](type-aliases/ExprLike.md) - [FieldLike](type-aliases/FieldLike.md) - [IntoSql](type-aliases/IntoSql.md) - [IntoVector](type-aliases/IntoVector.md) @@ -179,11 +181,16 @@ - [RecordBatchIterator](functions/RecordBatchIterator.md) - [blob](functions/blob.md) +- [col](functions/col.md) - [connect](functions/connect.md) - [connectCatalog](functions/connectCatalog.md) - [connectNamespace](functions/connectNamespace.md) +- [func](functions/func.md) - [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md) - [isBlobField](functions/isBlobField.md) +- [lit](functions/lit.md) +- [litBool](functions/litBool.md) +- [litInt](functions/litInt.md) - [makeArrowTable](functions/makeArrowTable.md) - [makeJsonField](functions/makeJsonField.md) - [packBits](functions/packBits.md) diff --git a/docs/src/js/type-aliases/ExprLike.md b/docs/src/js/type-aliases/ExprLike.md new file mode 100644 index 000000000..d0628c982 --- /dev/null +++ b/docs/src/js/type-aliases/ExprLike.md @@ -0,0 +1,13 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ExprLike + +# Type Alias: ExprLike + +```ts +type ExprLike: Expr | string | number | boolean; +``` + +Values accepted by expression operators. diff --git a/nodejs/__test__/expr.test.ts b/nodejs/__test__/expr.test.ts new file mode 100644 index 000000000..492929c04 --- /dev/null +++ b/nodejs/__test__/expr.test.ts @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import * as http from "http"; +import type { AddressInfo } from "net"; +import * as tmp from "tmp"; + +import { type Table, connect } from "../lancedb"; +import { + Table as ArrowTable, + Field, + Int64, + Schema, + Utf8, + tableToIPC, +} from "../lancedb/arrow"; +import { Expr, col, func, lit, litBool, litInt } from "../lancedb/expr"; + +describe("expr", () => { + describe("col", () => { + it("returns an Expr instance", () => { + const e = col("age"); + expect(e).toBeInstanceOf(Expr); + }); + + it("preserves camelCase in toSql()", () => { + expect(col("firstName").toSql()).toBe("`firstName`"); + }); + + it("preserves snake_case without quotes", () => { + expect(col("first_name").toSql()).toBe("first_name"); + }); + + it("Quotes identifiers with spaces", () => { + expect(col("first name").toSql()).toBe("`first name`"); + }); + + it("Quotes identifiers with leading digits", () => { + expect(col("2fast").toSql()).toBe("`2fast`"); + }); + + it("Quotes unicode identifiers", () => { + expect(col("名前").toSql()).toBe("`名前`"); + }); + }); + + describe("lit", () => { + it("creates a string literal", () => { + expect(lit("hello").toSql()).toBe("'hello'"); + }); + + it("creates an integer literal", () => { + expect(lit(42).toSql()).toBe("42"); + }); + + it("creates a float literal", () => { + expect(lit(3.14).toSql()).toBe("3.14"); + }); + }); + + describe("numeric literals", () => { + it.each([Number.MIN_SAFE_INTEGER, Number.MAX_SAFE_INTEGER])( + "preserves the safe integer boundary %s", + (value) => { + expect(lit(value).toSql()).toBe(String(value)); + expect(litInt(value).toSql()).toBe(String(value)); + }, + ); + + it.each([1e20, -1e20])("preserves the large number %s", (value) => { + expect(Number(lit(value).toSql())).toBe(value); + expect(col("price").gt(value).toSql()).toBe( + col("price").gt(lit(value)).toSql(), + ); + }); + + it.each([1.5, 1e20, -1e20, Number.NaN, Number.POSITIVE_INFINITY])( + "rejects %s as an explicit integer literal", + (value) => expect(() => litInt(value)).toThrow(RangeError), + ); + }); + + describe("Expr operators", () => { + it("eq", () => { + expect(col("x").eq(lit(1)).toSql()).toBe("(x = 1)"); + }); + + it("ne", () => { + expect(col("x").ne(lit(1)).toSql()).toBe("(x <> 1)"); + }); + + it("lt", () => { + expect(col("age").lt(lit(18)).toSql()).toBe("(age < 18)"); + }); + + it("lte", () => { + expect(col("age").lte(lit(18)).toSql()).toBe("(age <= 18)"); + }); + + it("gt", () => { + expect(col("age").gt(lit(18)).toSql()).toBe("(age > 18)"); + }); + + it("gte", () => { + expect(col("age").gte(lit(18)).toSql()).toBe("(age >= 18)"); + }); + + it("and", () => { + const e = col("age") + .gt(lit(18)) + .and(col("status").eq(lit("active"))); + expect(e.toSql()).toBe("((age > 18) AND (`status` = 'active'))"); + }); + + it("or", () => { + const e = col("a") + .eq(lit(1)) + .or(col("b").eq(lit(2))); + expect(e.toSql()).toBe("((a = 1) OR (b = 2))"); + }); + + it("not", () => { + const e = col("active").eq(litBool(true)).not(); + expect(e.toSql()).toBe("NOT (active = true)"); + }); + + it("add", () => { + expect(col("x").add(lit(1)).toSql()).toBe("(x + 1)"); + }); + + it("sub", () => { + expect(col("x").sub(lit(1)).toSql()).toBe("(x - 1)"); + }); + + it("mul", () => { + expect(col("price").mul(lit(1.1)).toSql()).toBe("(price * 1.1)"); + }); + + it("div", () => { + expect(col("total").div(lit(2)).toSql()).toBe("(total / 2)"); + }); + }); + + describe("Expr string methods", () => { + it("lower", () => { + expect(col("name").lower().toSql()).toBe("lower(`name`)"); + }); + + it("upper", () => { + expect(col("name").upper().toSql()).toBe("upper(`name`)"); + }); + + it("contains", () => { + expect(col("text").contains(lit("hello")).toSql()).toBe( + "contains(`text`, 'hello')", + ); + }); + }); + + describe("membership and casts", () => { + it("accepts scalar list members", () => { + expect(col("age").isIn([18, 21]).toSql()).toBe("age IN (18, 21)"); + }); + it("casts to an Arrow type", () => { + expect(col("age").cast("Int64").toSql()).toBe("arrow_cast(age, 'Int64')"); + }); + it("rejects an invalid Arrow type", () => { + expect(() => col("age").cast("invalid")).toThrow("Invalid data type"); + }); + }); + + describe("func", () => { + it("lower", () => { + expect(func("lower", col("name")).toSql()).toBe("lower(`name`)"); + }); + }); + + describe("camelCase integration", () => { + let db: Awaited>; + let table: Table; + let tmpDir: tmp.DirResult; + + beforeAll(async () => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + db = await connect(tmpDir.name); + table = await db.createTable("test", [ + { id: 1, firstName: "Alice", lastName: "Smith" }, + { id: 2, firstName: "Bob", lastName: "Jones" }, + { id: 3, firstName: "Charlie", lastName: "Brown" }, + ]); + }); + + afterAll(() => { + table.close(); + db.close(); + tmpDir.removeCallback(); + }); + + it("filters on camelCase column", async () => { + const results = await table + .query() + .where(col("firstName").eq(lit("Alice"))) + .toArray(); + expect(results.length).toBe(1); + expect(results[0].firstName).toBe("Alice"); + }); + + it("filters on multiple camelCase columns", async () => { + const results = await table + .query() + .where( + col("firstName") + .eq(lit("Bob")) + .and(col("lastName").eq(lit("Jones"))), + ) + .toArray(); + expect(results.length).toBe(1); + expect(results[0].firstName).toBe("Bob"); + }); + + it("preserves an unsafe integer filter threshold", async () => { + const prices = await db.createTable("prices", [ + { price: 9.5e18 }, + { price: 1.1e20 }, + ]); + try { + const results = await prices + .query() + .where(col("price").gt(1e20)) + .toArray(); + expect(results.map((row) => row.price)).toEqual([1.1e20]); + } finally { + prices.close(); + } + }); + + it("filters with gt on camelCase column", async () => { + const results = await table + .query() + .where(col("id").gt(lit(1))) + .toArray(); + expect(results.length).toBe(2); + }); + + it.each([ + ["backslash", "\\"], + ["quote", "'"], + ["backslash quote", "\\'"], + ["OR true payload", "\\') OR true --"], + ["line comment payload", "\\' -- OR true"], + ["block comment payload", "\\') /* OR true */ --"], + ])("filters the exact %s value", async (label, value) => { + const escaped = await db.createTable( + `escaped_${label.replaceAll(" ", "_")}`, + [ + { id: 1, tenant: value }, + { id: 2, tenant: "different tenant" }, + ], + ); + try { + const results = await escaped + .query() + .where(col("tenant").eq(lit(value))) + .toArray(); + expect(results).toHaveLength(1); + expect(results[0].id).toBe(1); + expect(results[0].tenant).toBe(value); + } finally { + escaped.close(); + } + }); + }); + + it("sends a quoted expression value to remote as one literal", async () => { + const value = "\\') OR true --"; + const schema = new Schema([ + new Field("id", new Int64(), false), + new Field("tenant", new Utf8(), false), + ]); + const response = Buffer.from(tableToIPC(new ArrowTable(schema), "stream")); + let remoteFilter: unknown; + + const server = http.createServer((req, res) => { + const path = req.url ?? ""; + if (path.endsWith("/describe/")) { + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "tenants", + version: 1, + schema: { + fields: [ + { name: "id", type: { type: "int64" }, nullable: false }, + { name: "tenant", type: { type: "string" }, nullable: false }, + ], + }, + }), + ); + return; + } + if (path.endsWith("/query/")) { + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", () => { + remoteFilter = JSON.parse(body).filter; + res + .writeHead(200, { + "Content-Type": "application/vnd.apache.arrow.stream", + }) + .end(response); + }); + return; + } + res.writeHead(404).end(); + }); + + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + const address = server.address() as AddressInfo; + + try { + const db = await connect("db://dev", { + apiKey: "fake", + hostOverride: `http://127.0.0.1:${address.port}`, + }); + try { + const remoteTable = await db.openTable("tenants"); + await remoteTable + .query() + .where(col("tenant").eq(lit(value))) + .toArrow(); + } finally { + db.close(); + } + } finally { + await new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }); + } + + expect(remoteFilter).toBe("(tenant = '\\'') OR true --')"); + }); +}); diff --git a/nodejs/lancedb/expr.ts b/nodejs/lancedb/expr.ts new file mode 100644 index 000000000..5880bec60 --- /dev/null +++ b/nodejs/lancedb/expr.ts @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { + Expr as NativeExpr, + col as nativeCol, + func as nativeFunc, + lit as nativeLit, + litBool as nativeLitBool, + litInt as nativeLitInt, +} from "./native.js"; + +export { Expr as NativeExpr } from "./native"; + +/** Values accepted by expression operators. */ +export type ExprLike = Expr | string | number | boolean; + +function coerce(value: ExprLike): Expr { + if (value instanceof Expr) { + return value; + } + if (typeof value === "string") { + return lit(value); + } + if (typeof value === "boolean") { + return litBool(value); + } + if (typeof value === "number") { + return lit(value); + } + throw new TypeError(`Unsupported literal type: ${typeof value}`); +} + +/** + * A SQL expression backed by the shared LanceDB expression builder. + * + * @example + * ```ts + * table.query().where(col("firstName").eq("Alice")).toArray(); + * ``` + */ +export class Expr { + inner: NativeExpr; + + constructor(inner: NativeExpr) { + this.inner = inner; + } + + /** Compare for equality. */ + eq(other: ExprLike): Expr { + return new Expr(this.inner.eq(coerce(other).inner)); + } + + /** Compare for inequality. */ + ne(other: ExprLike): Expr { + return new Expr(this.inner.ne(coerce(other).inner)); + } + + /** Compare for less than. */ + lt(other: ExprLike): Expr { + return new Expr(this.inner.lt(coerce(other).inner)); + } + + /** Compare for less than or equal to. */ + lte(other: ExprLike): Expr { + return new Expr(this.inner.lte(coerce(other).inner)); + } + + /** Compare for greater than. */ + gt(other: ExprLike): Expr { + return new Expr(this.inner.gt(coerce(other).inner)); + } + + /** Compare for greater than or equal to. */ + gte(other: ExprLike): Expr { + return new Expr(this.inner.gte(coerce(other).inner)); + } + + /** Combine two predicates with AND. */ + and(other: Expr): Expr { + return new Expr(this.inner.and(other.inner)); + } + + /** Combine two predicates with OR. */ + or(other: Expr): Expr { + return new Expr(this.inner.or(other.inner)); + } + + /** Negate this predicate. */ + not(): Expr { + return new Expr(this.inner.not()); + } + + /** Add a value. */ + add(other: ExprLike): Expr { + return new Expr(this.inner.add(coerce(other).inner)); + } + + /** Subtract a value. */ + sub(other: ExprLike): Expr { + return new Expr(this.inner.sub(coerce(other).inner)); + } + + /** Multiply by a value. */ + mul(other: ExprLike): Expr { + return new Expr(this.inner.mul(coerce(other).inner)); + } + + /** Divide by a value. */ + div(other: ExprLike): Expr { + return new Expr(this.inner.div(coerce(other).inner)); + } + + /** Convert a string to lowercase. */ + lower(): Expr { + return new Expr(this.inner.lower()); + } + + /** Convert a string to uppercase. */ + upper(): Expr { + return new Expr(this.inner.upper()); + } + + /** Test whether a string contains a substring. */ + contains(substr: ExprLike): Expr { + return new Expr(this.inner.contains(coerce(substr).inner)); + } + + /** Test membership in a list of values. */ + isIn(values: ExprLike[]): Expr { + return new Expr(this.inner.isIn(values.map((v) => coerce(v).inner))); + } + + /** Cast to an Arrow data type, such as "Int64". */ + cast(dataType: string): Expr { + return new Expr(this.inner.cast(dataType)); + } + + /** Serialize with the LanceDB SQL dialect, preserving column case. */ + toSql(): string { + return this.inner.toSql(); + } +} + +/** Reference a column by its exact, case-sensitive name. */ +export function col(name: string): Expr { + return new Expr(nativeCol(name)); +} + +/** Create a string or numeric literal. Unsafe integers are represented as Float64. */ +export function lit(value: string | number): Expr { + if (typeof value === "string") { + return new Expr(nativeLit(value)); + } + if (Number.isSafeInteger(value)) { + return new Expr(nativeLitInt(value)); + } + return new Expr(nativeLit(value)); +} + +/** Create a boolean literal. */ +export function litBool(value: boolean): Expr { + return new Expr(nativeLitBool(value)); +} + +/** Create an Int64 literal from a JavaScript safe integer. Other numbers throw RangeError. */ +export function litInt(value: number): Expr { + if (!Number.isSafeInteger(value)) { + throw new RangeError("Integer literals must be safe integers"); + } + return new Expr(nativeLitInt(value)); +} + +/** Call a registered SQL function, such as `func("lower", col("firstName"))`. */ +export function func(name: string, ...args: ExprLike[]): Expr { + return new Expr( + nativeFunc( + name, + args.map((a) => coerce(a).inner), + ), + ); +} diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 0f1b5f333..b9c334a08 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -207,6 +207,8 @@ export { MultiVector, } from "./arrow"; export { IntoSql, packBits } from "./util"; +export { Expr, col, lit, litBool, litInt, func } from "./expr"; +export type { ExprLike } from "./expr"; /** * Options for tokenizing a full-text search query without a table index. diff --git a/nodejs/lancedb/query.ts b/nodejs/lancedb/query.ts index b96e4fda6..27133a326 100644 --- a/nodejs/lancedb/query.ts +++ b/nodejs/lancedb/query.ts @@ -17,6 +17,7 @@ import { tableFromIPC, vectorFromArray, } from "./arrow"; +import { Expr } from "./expr"; import { type IvfPqOptions } from "./indices"; import { JsFullTextQuery, @@ -649,11 +650,16 @@ export class StandardQueryBase< /** * A filter statement to be applied to this query. * - * The filter should be supplied as an SQL query string. For example: + * The filter may be an SQL query string or an expression built with `col`, + * which preserves case-sensitive column names. For example: * @example * x > 10 * y > 0 AND y < 100 * x > 5 OR y = 'test' + * @example + * ```ts + * table.query().where(col("firstName").eq("Alice")); + * ``` * * Filtering performance can often be improved by creating a scalar index * on the filter column(s). @@ -661,8 +667,14 @@ export class StandardQueryBase< * Calling this multiple times combines the filters with a logical AND rather * than replacing the previous filter. */ - where(predicate: string): this { - this.doCall((inner: NativeQueryType) => inner.onlyIf(predicate)); + where(predicate: string | Expr): this { + this.doCall((inner: NativeQueryType) => { + if (predicate instanceof Expr) { + inner.onlyIf(predicate.toSql()); + } else { + inner.onlyIf(predicate); + } + }); return this; } /** diff --git a/nodejs/src/expr.rs b/nodejs/src/expr.rs new file mode 100644 index 000000000..c12f81a96 --- /dev/null +++ b/nodejs/src/expr.rs @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Napi bindings for the LanceDB expression builder API. + +use std::ops::{Add, Div, Mul, Not, Sub}; + +use lancedb::expr::{ + DfExpr, col as ldb_col, contains, expr_cast, is_in, lit as df_lit, lower, upper, +}; +use napi::bindgen_prelude::*; +use napi_derive::napi; + +#[napi] +#[derive(Clone)] +pub struct Expr { + pub(crate) inner: DfExpr, +} + +#[napi] +impl Expr { + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn eq(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().eq(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn ne(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().not_eq(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn lt(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().lt(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn lte(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().lt_eq(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn gt(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().gt(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn gte(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().gt_eq(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn and(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().and(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn or(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().or(other.inner.clone()), + } + } + + #[napi] + pub fn not(&self) -> Self { + Self { + inner: self.inner.clone().not(), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn add(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().add(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn sub(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().sub(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn mul(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().mul(other.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn div(&self, other: &Expr) -> Self { + Self { + inner: self.inner.clone().div(other.inner.clone()), + } + } + + #[napi] + pub fn lower(&self) -> Self { + Self { + inner: lower(self.inner.clone()), + } + } + + #[napi] + pub fn upper(&self) -> Self { + Self { + inner: upper(self.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn contains(&self, substr: &Expr) -> Self { + Self { + inner: contains(self.inner.clone(), substr.inner.clone()), + } + } + + #[napi] + #[allow(clippy::use_self)] // NAPI doesn't allow Self here but clippy reports it + pub fn is_in(&self, list: Vec<&Expr>) -> Self { + let items: Vec = list.into_iter().map(|e| e.inner.clone()).collect(); + Self { + inner: is_in(self.inner.clone(), items), + } + } + + #[napi] + pub fn cast(&self, data_type: String) -> napi::Result { + let arrow_type = data_type + .parse::() + .map_err(|e| napi::Error::from_reason(format!("Invalid data type: {}", e)))?; + Ok(Self { + inner: expr_cast(self.inner.clone(), arrow_type), + }) + } + + #[napi] + pub fn to_sql(&self) -> napi::Result { + lancedb::expr::expr_to_sql_string(&self.inner) + .map_err(|e| napi::Error::from_reason(e.to_string())) + } +} + +/// Create a column reference expression. +/// +/// The column name is preserved exactly as given (case-sensitive), so +/// `col("firstName")` correctly references a field named `firstName`. +#[napi] +#[allow(dead_code)] +pub fn col(name: String) -> Expr { + Expr { + inner: ldb_col(name), + } +} + +/// Create a literal (constant) value expression. +#[napi] +#[allow(dead_code)] +pub fn lit(value: napi::bindgen_prelude::Either) -> napi::Result { + match value { + Either::A(s) => Ok(Expr { inner: df_lit(s) }), + Either::B(f) => Ok(Expr { inner: df_lit(f) }), + } +} + +/// Create a literal boolean value expression. +#[napi] +#[allow(dead_code)] +pub fn lit_bool(value: bool) -> Expr { + Expr { + inner: df_lit(value), + } +} + +/// Create a literal integer value expression. +#[napi] +#[allow(dead_code)] +pub fn lit_int(value: i64) -> Expr { + Expr { + inner: df_lit(value), + } +} + +/// Call an arbitrary registered SQL function by name. +#[napi] +#[allow(dead_code)] +pub fn func(name: String, args: Vec<&Expr>) -> napi::Result { + let df_args: Vec = args.into_iter().map(|e| e.inner.clone()).collect(); + lancedb::expr::func(&name, df_args) + .map(|inner| Expr { inner }) + .map_err(|e| napi::Error::from_reason(e.to_string())) +} diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index a92b8db98..cb40a51de 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -14,6 +14,7 @@ mod blob; mod catalog; mod connection; mod error; +mod expr; mod header; mod index; mod iterator; diff --git a/nodejs/typedoc.json b/nodejs/typedoc.json index c601ee61e..2c22b35ac 100644 --- a/nodejs/typedoc.json +++ b/nodejs/typedoc.json @@ -1,6 +1,7 @@ { "intentionallyNotExported": [ "lancedb/native.d.ts:Query", + "lancedb/native.d.ts:Expr", "lancedb/native.d.ts:VectorQuery", "lancedb/native.d.ts:TakeQuery", "lancedb/native.d.ts:RecordBatchIterator", diff --git a/rust/lancedb/src/expr.rs b/rust/lancedb/src/expr.rs index 1625d9632..04d1b0e9e 100644 --- a/rust/lancedb/src/expr.rs +++ b/rust/lancedb/src/expr.rs @@ -295,6 +295,49 @@ mod tests { ); } + #[test] + fn test_string_placeholder_does_not_rewrite_user_string() { + let marker = "__lancedb_string_placeholder_0__"; + let expr = col("value") + .eq(lit(marker)) + .and(col("other").eq(lit("ordinary"))); + assert_eq!( + expr_to_sql_string(&expr).unwrap(), + "((`value` = '__lancedb_string_placeholder_0__') AND (`other` = 'ordinary'))" + ); + } + + #[test] + fn test_string_literals_use_standard_quote_escaping() { + use datafusion_sql::sqlparser::{ + dialect::GenericDialect, tokenizer::Token, tokenizer::Tokenizer, + }; + + for value in [ + r"\", + "'", + r"\') OR true --", + "tenant -- 'not a comment", + "tenant /* 'not a comment */", + ] { + let expr = col("tenant").eq(lit(value)); + let sql = expr_to_sql_string(&expr).unwrap(); + assert_eq!(canonicalize_sql_predicate(&sql).unwrap(), sql); + + let string_literals = Tokenizer::new(&GenericDialect {}, &sql) + .with_unescape(false) + .tokenize() + .unwrap() + .into_iter() + .filter_map(|token| match token { + Token::SingleQuotedString(value) => Some(value), + _ => None, + }) + .collect::>(); + assert_eq!(string_literals, vec![value.replace('\'', "''")]); + } + } + #[test] fn test_binary_binding_skips_quoted_identifiers() { use datafusion_common::ScalarValue; diff --git a/rust/lancedb/src/expr/sql.rs b/rust/lancedb/src/expr/sql.rs index 2a1ca201d..96bb5b60c 100644 --- a/rust/lancedb/src/expr/sql.rs +++ b/rust/lancedb/src/expr/sql.rs @@ -104,9 +104,15 @@ pub fn canonicalize_sql_predicate(predicate: &str) -> crate::Result { .collect()) } -/// Prefix for placeholder strings inserted in place of binary literals. Chosen -/// to be extremely unlikely to occur in user data. +/// Prefixes for placeholder strings inserted in place of literals. Chosen to +/// be extremely unlikely to occur in user data. const BINARY_PLACEHOLDER_PREFIX: &str = "__lancedb_binary_placeholder_"; +const STRING_PLACEHOLDER_PREFIX: &str = "__lancedb_string_placeholder_"; + +enum LiteralBinding { + String(String), + Binary(Vec), +} fn bytes_to_hex_sql(bytes: &[u8]) -> String { let hex: String = bytes.iter().map(|b| format!("{b:02X}")).collect(); @@ -137,19 +143,28 @@ fn typed_string_literal(value: String, data_type: DataType) -> Expr { ) } -fn next_binary_placeholder(user_strings: &HashSet, next_id: &mut usize) -> String { +fn next_placeholder( + prefix: &str, + user_strings: &HashSet, + bindings: &HashMap, + next_id: &mut usize, +) -> String { loop { - let placeholder = format!("{BINARY_PLACEHOLDER_PREFIX}{}__", *next_id); + let placeholder = format!("{prefix}{}__", *next_id); *next_id += 1; - if !user_strings.contains(&placeholder) { + if !user_strings.contains(&placeholder) && !bindings.contains_key(&placeholder) { return placeholder; } } } -fn bind_binary_literals( +fn escape_string_literal(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +fn bind_literals( sql: &str, - mut bindings: HashMap>, + mut bindings: HashMap, ) -> crate::Result { let bytes = sql.as_bytes(); let mut output = Vec::with_capacity(bytes.len()); @@ -179,7 +194,7 @@ fn bind_binary_literals( let Some(identifier_end) = identifier_end else { return Err(crate::Error::InvalidInput { - message: "unterminated identifier while binding binary literal".to_string(), + message: "unterminated identifier while binding literals".to_string(), }); }; output.extend_from_slice(&bytes[identifier_start..identifier_end]); @@ -214,13 +229,17 @@ fn bind_binary_literals( let Some(content_end) = content_end else { return Err(crate::Error::InvalidInput { - message: "unterminated string while binding binary literal".to_string(), + message: "unterminated string while binding literals".to_string(), }); }; let placeholder = &sql[content_start..content_end]; if !escaped && let Some(value) = bindings.remove(placeholder) { - output.extend_from_slice(bytes_to_hex_sql(&value).as_bytes()); + let rendered = match value { + LiteralBinding::String(value) => escape_string_literal(&value), + LiteralBinding::Binary(value) => bytes_to_hex_sql(&value), + }; + output.extend_from_slice(rendered.as_bytes()); } else { output.extend_from_slice(&bytes[literal_start..index]); } @@ -228,12 +247,12 @@ fn bind_binary_literals( if !bindings.is_empty() { return Err(crate::Error::InvalidInput { - message: "failed to bind binary literal while serializing expression".to_string(), + message: "failed to bind literals while serializing expression".to_string(), }); } String::from_utf8(output).map_err(|e| crate::Error::InvalidInput { - message: format!("failed to bind binary literal: {e}"), + message: format!("failed to bind literals: {e}"), }) } @@ -253,7 +272,7 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result { // * decimal literals need an explicit cast to preserve precision and scale; // * casts need exact Arrow type names rather than SQL type aliases; // * an empty IN list is valid in DataFusion but invalid SQL; - // * binary literals are unsupported by the unparser and need placeholders. + // * literals are replaced with placeholders and restored after unparsing. // Eliminate empty membership expressions before visiting their children. // Otherwise a discarded binary child could leave behind a stale binding. let rewritten = expr @@ -270,14 +289,19 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result { .data; let user_strings = string_literals(&rewritten); - let mut next_placeholder_id = 0; - let mut binary_bindings = HashMap::new(); + let mut next_binary_placeholder_id = 0; + let mut bindings = HashMap::new(); let rewritten = rewritten .transform(|e: Expr| match e { Expr::Literal(ScalarValue::Binary(Some(bytes)), m) | Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), m) => { - let placeholder = next_binary_placeholder(&user_strings, &mut next_placeholder_id); - binary_bindings.insert(placeholder.clone(), bytes); + let placeholder = next_placeholder( + BINARY_PLACEHOLDER_PREFIX, + &user_strings, + &bindings, + &mut next_binary_placeholder_id, + ); + bindings.insert(placeholder.clone(), LiteralBinding::Binary(bytes)); Ok(Transformed::yes(Expr::Literal( ScalarValue::Utf8(Some(placeholder)), m, @@ -345,11 +369,44 @@ pub fn expr_to_sql_string(expr: &Expr) -> crate::Result { })? .data; + // DataFusion's SQLParser serializer treats a quote after a backslash as + // already escaped, but Lance's dialect does not support backslash escapes. + // Replace every string with an inert placeholder so the serializer cannot + // misinterpret its contents, then restore each value with standard SQL + // single-quote doubling below. + let mut next_string_placeholder_id = 0; + let rewritten = rewritten + .transform(|e: Expr| match e { + Expr::Literal( + ScalarValue::Utf8(Some(value)) + | ScalarValue::LargeUtf8(Some(value)) + | ScalarValue::Utf8View(Some(value)), + m, + ) if !bindings.contains_key(&value) => { + let placeholder = next_placeholder( + STRING_PLACEHOLDER_PREFIX, + &user_strings, + &bindings, + &mut next_string_placeholder_id, + ); + bindings.insert(placeholder.clone(), LiteralBinding::String(value)); + Ok(Transformed::yes(Expr::Literal( + ScalarValue::Utf8(Some(placeholder)), + m, + ))) + } + other => Ok(Transformed::no(other)), + }) + .map_err(|e| crate::Error::InvalidInput { + message: format!("failed to rewrite string literals: {}", e), + })? + .data; + let sql = run_unparser(&rewritten)?; - if binary_bindings.is_empty() { + if bindings.is_empty() { Ok(sql) } else { - bind_binary_literals(&sql, binary_bindings) + bind_literals(&sql, bindings) } }