From c68c236f17bfa5d6c5d0751ebd7d7b3d8ad23e8e Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Mon, 26 Jun 2023 18:38:20 -0700 Subject: [PATCH] [Js] Create index with replace flag (#229) --- node/README.md | 8 +----- node/package-lock.json | 45 +++++++++++++++++++++++++++++-- node/package.json | 4 ++- node/src/index.ts | 12 ++++----- node/src/test/test.ts | 26 ++++++++++++++++-- rust/ffi/node/src/index/vector.rs | 4 +++ 6 files changed, 80 insertions(+), 19 deletions(-) diff --git a/node/README.md b/node/README.md index 9302cf5b..6e629d6c 100644 --- a/node/README.md +++ b/node/README.md @@ -16,7 +16,7 @@ npm install vectordb const lancedb = require('vectordb'); const db = lancedb.connect(''); const table = await db.openTable('my_table'); -const query = await table.search([0.1, 0.3]).setLimit(20).execute(); +const query = await table.search([0.1, 0.3]).limit(20).execute(); console.log(results); ``` @@ -24,12 +24,6 @@ The [examples](./examples) folder contains complete examples. ## Development -The LanceDB javascript is built with npm: - -```bash -npm run tsc -``` - Run the tests with ```bash diff --git a/node/package-lock.json b/node/package-lock.json index 70203950..5eee35b1 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -1,12 +1,12 @@ { "name": "vectordb", - "version": "0.1.8", + "version": "0.1.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "vectordb", - "version": "0.1.8", + "version": "0.1.9", "license": "Apache-2.0", "dependencies": { "@apache-arrow/ts": "^12.0.0", @@ -14,6 +14,7 @@ }, "devDependencies": { "@types/chai": "^4.3.4", + "@types/chai-as-promised": "^7.1.5", "@types/mocha": "^10.0.1", "@types/node": "^18.16.2", "@types/sinon": "^10.0.15", @@ -21,6 +22,7 @@ "@typescript-eslint/eslint-plugin": "^5.59.1", "cargo-cp-artifact": "^0.1", "chai": "^4.3.7", + "chai-as-promised": "^7.1.1", "eslint": "^8.39.0", "eslint-config-standard-with-typescript": "^34.0.1", "eslint-plugin-import": "^2.26.0", @@ -311,6 +313,15 @@ "integrity": "sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==", "dev": true }, + "node_modules/@types/chai-as-promised": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.5.tgz", + "integrity": "sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ==", + "dev": true, + "dependencies": { + "@types/chai": "*" + } + }, "node_modules/@types/command-line-args": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz", @@ -942,6 +953,18 @@ "node": ">=4" } }, + "node_modules/chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "dependencies": { + "check-error": "^1.0.2" + }, + "peerDependencies": { + "chai": ">= 2.1.2 < 5" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -4680,6 +4703,15 @@ "integrity": "sha512-KnRanxnpfpjUTqTCXslZSEdLfXExwgNxYPdiO2WGUj8+HDjFi8R3k5RVKPeSCzLjCcshCAtVO2QBbVuAV4kTnw==", "dev": true }, + "@types/chai-as-promised": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.5.tgz", + "integrity": "sha512-jStwss93SITGBwt/niYrkf2C+/1KTeZCZl1LaeezTlqppAKeoQC7jxyqYuP72sxBGKCIbw7oHgbYssIRzT5FCQ==", + "dev": true, + "requires": { + "@types/chai": "*" + } + }, "@types/command-line-args": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz", @@ -5137,6 +5169,15 @@ "type-detect": "^4.0.5" } }, + "chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "requires": { + "check-error": "^1.0.2" + } + }, "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", diff --git a/node/package.json b/node/package.json index 585c185f..b4da793f 100644 --- a/node/package.json +++ b/node/package.json @@ -8,7 +8,7 @@ "tsc": "tsc -b", "build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics", "build-release": "npm run build -- --release", - "test": "mocha -recursive dist/test", + "test": "npm run tsc; mocha -recursive dist/test", "lint": "eslint src --ext .js,.ts", "clean": "rm -rf node_modules *.node dist/" }, @@ -26,6 +26,7 @@ "license": "Apache-2.0", "devDependencies": { "@types/chai": "^4.3.4", + "@types/chai-as-promised": "^7.1.5", "@types/mocha": "^10.0.1", "@types/node": "^18.16.2", "@types/sinon": "^10.0.15", @@ -33,6 +34,7 @@ "@typescript-eslint/eslint-plugin": "^5.59.1", "cargo-cp-artifact": "^0.1", "chai": "^4.3.7", + "chai-as-promised": "^7.1.1", "eslint": "^8.39.0", "eslint-config-standard-with-typescript": "^34.0.1", "eslint-plugin-import": "^2.26.0", diff --git a/node/src/index.ts b/node/src/index.ts index 4294b697..42e6b12e 100644 --- a/node/src/index.ts +++ b/node/src/index.ts @@ -180,13 +180,6 @@ export class Table { return tableCreateVectorIndex.call(this._tbl, indexParams) } - /** - * @deprecated Use [Table.createIndex] - */ - async create_index (indexParams: VectorIndexParams): Promise { - return await this.createIndex(indexParams) - } - /** * Returns the number of rows in this table. */ @@ -249,6 +242,11 @@ interface IvfPQIndexConfig { */ max_opq_iters?: number + /** + * Replace an existing index with the same name if it exists. + */ + replace?: boolean + type: 'ivf_pq' } diff --git a/node/src/test/test.ts b/node/src/test/test.ts index e5169e07..b10332b5 100644 --- a/node/src/test/test.ts +++ b/node/src/test/test.ts @@ -13,12 +13,17 @@ // limitations under the License. import { describe } from 'mocha' -import { assert } from 'chai' import { track } from 'temp' +import * as chai from 'chai' +import * as chaiAsPromised from 'chai-as-promised' import * as lancedb from '../index' import { type EmbeddingFunction, MetricType, Query } from '../index' +const expect = chai.expect +const assert = chai.assert +chai.use(chaiAsPromised) + describe('LanceDB client', function () { describe('when creating a connection to lancedb', function () { it('should have a valid url', async function () { @@ -165,8 +170,25 @@ describe('LanceDB client', function () { const uri = await createTestDB(32, 300) const con = await lancedb.connect(uri) const table = await con.openTable('vectors') - await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2 }) + await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 }) }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow + + it('replace an existing index', async function () { + const uri = await createTestDB(16, 300) + const con = await lancedb.connect(uri) + const table = await con.openTable('vectors') + + await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 }) + + // Replace should fail if the index already exists + await expect(table.createIndex({ + type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2, replace: false + }) + ).to.be.rejectedWith('LanceError(Index)') + + // Default replace = true + await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 }) + }).timeout(50_000) }) describe('when using a custom embedding function', function () { diff --git a/rust/ffi/node/src/index/vector.rs b/rust/ffi/node/src/index/vector.rs index 0b42c5bf..495441a8 100644 --- a/rust/ffi/node/src/index/vector.rs +++ b/rust/ffi/node/src/index/vector.rs @@ -122,6 +122,10 @@ fn get_index_params_builder( .map_err(|t| t.to_string())? .map(|s| pq_params.max_opq_iters = s.value(cx) as usize); + obj.get_opt::(cx, "replace") + .map_err(|t| t.to_string())? + .map(|s| index_builder.replace(s.value(cx))); + Ok(index_builder) } t => Err(format!("{} is not a valid index type", t).to_string()),