mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-03 10:22:56 +00:00
JavaScript / Node.js library for LanceDB
- Core rust library - ffi bridge that exposes rust functionality to javascript - npm package that provides a TypeScript / JavaScript library - limitations: it only supports reading for now
This commit is contained in:
10
.gitignore
vendored
10
.gitignore
vendored
@@ -18,3 +18,13 @@ python/dist
|
||||
notebooks/.ipynb_checkpoints
|
||||
|
||||
**/.hypothesis
|
||||
|
||||
## Javascript
|
||||
index.node
|
||||
**/node_modules
|
||||
**/.DS_Store
|
||||
node/dist
|
||||
|
||||
## Rust
|
||||
target
|
||||
|
||||
|
||||
3789
Cargo.lock
generated
Normal file
3789
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
5
Cargo.toml
Normal file
5
Cargo.toml
Normal file
@@ -0,0 +1,5 @@
|
||||
[workspace]
|
||||
members = [
|
||||
"rust/vectordb",
|
||||
"rust/ffi/node"
|
||||
]
|
||||
16
node/.eslintrc.js
Normal file
16
node/.eslintrc.js
Normal file
@@ -0,0 +1,16 @@
|
||||
module.exports = {
|
||||
env: {
|
||||
browser: true,
|
||||
es2021: true
|
||||
},
|
||||
extends: 'standard-with-typescript',
|
||||
overrides: [
|
||||
],
|
||||
parserOptions: {
|
||||
project: './tsconfig.json',
|
||||
ecmaVersion: 'latest',
|
||||
sourceType: 'module'
|
||||
},
|
||||
rules: {
|
||||
}
|
||||
}
|
||||
30
node/README.md
Normal file
30
node/README.md
Normal file
@@ -0,0 +1,30 @@
|
||||
|
||||
A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb).
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
npm i vectordb
|
||||
```
|
||||
|
||||
See the examples folder for usage.
|
||||
|
||||
## Development
|
||||
|
||||
The LanceDB javascript is built with npm:
|
||||
|
||||
```bash
|
||||
npm run tsc
|
||||
```
|
||||
|
||||
Run the tests with
|
||||
|
||||
```bash
|
||||
npm test
|
||||
```
|
||||
|
||||
To run the linter and have it automatically fix all errors
|
||||
|
||||
```bash
|
||||
npm run lint -- --fix
|
||||
```
|
||||
21
node/examples/simple/index.js
Normal file
21
node/examples/simple/index.js
Normal file
@@ -0,0 +1,21 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
'use strict'
|
||||
|
||||
const lancedb = require('@lancedb/vectordb')
|
||||
|
||||
const db = lancedb.connect('../../sample-lancedb')
|
||||
|
||||
console.log(db.tableNames())
|
||||
575
node/examples/simple/package-lock.json
generated
Normal file
575
node/examples/simple/package-lock.json
generated
Normal file
@@ -0,0 +1,575 @@
|
||||
{
|
||||
"name": "vectordb-example",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "vectordb-example",
|
||||
"version": "1.0.0",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@lancedb/vectordb": "^0.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@apache-arrow/ts": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.0.tgz",
|
||||
"integrity": "sha512-ArJ3Fw5W9RAeNWuyCU2CdjL/nEAZSVDG1p3jz/ZtLo/q3NTz2w7HUCOJeszejH/5alGX+QirYrJ5c6BW++/P7g==",
|
||||
"dependencies": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "18.14.5",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "6.1.3",
|
||||
"flatbuffers": "23.3.3",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@lancedb/vectordb": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb/0.0.1/3c0f01b97a74046f90807202605497536d982cd9",
|
||||
"integrity": "sha512-Bq6nuyCXOir7wnokhHt7GruydPFZxEQvTi7qxIBH28i8fjU2nVftElysKimm2TgaUnR756S15e4kL/Oma030MQ==",
|
||||
"hasInstallScript": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@apache-arrow/ts": "^12.0.0",
|
||||
"apache-arrow": "^12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/command-line-args": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
|
||||
"integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA=="
|
||||
},
|
||||
"node_modules/@types/command-line-usage": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz",
|
||||
"integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg=="
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "18.14.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
|
||||
"integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw=="
|
||||
},
|
||||
"node_modules/@types/pad-left": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz",
|
||||
"integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA=="
|
||||
},
|
||||
"node_modules/ansi-styles": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
|
||||
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
|
||||
"dependencies": {
|
||||
"color-convert": "^1.9.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/apache-arrow": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.0.tgz",
|
||||
"integrity": "sha512-uI+hnZZsGfNJiR/wG8j5yPQuDjmOHx4hZpkA743G4x3TlFrCpA3MMX7KUkIOIw0e/CwZ8NYuaMzaQsblA47qVA==",
|
||||
"dependencies": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "18.14.5",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "6.1.3",
|
||||
"flatbuffers": "23.3.3",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.0"
|
||||
},
|
||||
"bin": {
|
||||
"arrow2csv": "bin/arrow2csv.js"
|
||||
}
|
||||
},
|
||||
"node_modules/array-back": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
|
||||
"integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/chalk": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
|
||||
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
|
||||
"dependencies": {
|
||||
"ansi-styles": "^3.2.1",
|
||||
"escape-string-regexp": "^1.0.5",
|
||||
"supports-color": "^5.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/color-convert": {
|
||||
"version": "1.9.3",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
|
||||
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
|
||||
"dependencies": {
|
||||
"color-name": "1.1.3"
|
||||
}
|
||||
},
|
||||
"node_modules/color-name": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
|
||||
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
|
||||
},
|
||||
"node_modules/command-line-args": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
|
||||
"integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==",
|
||||
"dependencies": {
|
||||
"array-back": "^3.1.0",
|
||||
"find-replace": "^3.0.0",
|
||||
"lodash.camelcase": "^4.3.0",
|
||||
"typical": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz",
|
||||
"integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==",
|
||||
"dependencies": {
|
||||
"array-back": "^4.0.2",
|
||||
"chalk": "^2.4.2",
|
||||
"table-layout": "^1.0.2",
|
||||
"typical": "^5.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/array-back": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
|
||||
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage/node_modules/typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/deep-extend": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
|
||||
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/escape-string-regexp": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
|
||||
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
|
||||
"engines": {
|
||||
"node": ">=0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/find-replace": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz",
|
||||
"integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==",
|
||||
"dependencies": {
|
||||
"array-back": "^3.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/flatbuffers": {
|
||||
"version": "23.3.3",
|
||||
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz",
|
||||
"integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g=="
|
||||
},
|
||||
"node_modules/has-flag": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
|
||||
"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/json-bignum": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz",
|
||||
"integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash.camelcase": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
|
||||
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
|
||||
},
|
||||
"node_modules/pad-left": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz",
|
||||
"integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==",
|
||||
"dependencies": {
|
||||
"repeat-string": "^1.5.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/reduce-flatten": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz",
|
||||
"integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/repeat-string": {
|
||||
"version": "1.6.1",
|
||||
"resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
|
||||
"integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==",
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/supports-color": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
|
||||
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
|
||||
"dependencies": {
|
||||
"has-flag": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/table-layout": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz",
|
||||
"integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==",
|
||||
"dependencies": {
|
||||
"array-back": "^4.0.1",
|
||||
"deep-extend": "~0.6.0",
|
||||
"typical": "^5.2.0",
|
||||
"wordwrapjs": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/table-layout/node_modules/array-back": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
|
||||
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/table-layout/node_modules/typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
|
||||
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
|
||||
},
|
||||
"node_modules/typical": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz",
|
||||
"integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/wordwrapjs": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz",
|
||||
"integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==",
|
||||
"dependencies": {
|
||||
"reduce-flatten": "^2.0.0",
|
||||
"typical": "^5.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/wordwrapjs/node_modules/typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@apache-arrow/ts": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.0.tgz",
|
||||
"integrity": "sha512-ArJ3Fw5W9RAeNWuyCU2CdjL/nEAZSVDG1p3jz/ZtLo/q3NTz2w7HUCOJeszejH/5alGX+QirYrJ5c6BW++/P7g==",
|
||||
"requires": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "18.14.5",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "6.1.3",
|
||||
"flatbuffers": "23.3.3",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.0"
|
||||
}
|
||||
},
|
||||
"@lancedb/vectordb": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb/0.0.1/3c0f01b97a74046f90807202605497536d982cd9",
|
||||
"integrity": "sha512-Bq6nuyCXOir7wnokhHt7GruydPFZxEQvTi7qxIBH28i8fjU2nVftElysKimm2TgaUnR756S15e4kL/Oma030MQ==",
|
||||
"requires": {
|
||||
"@apache-arrow/ts": "^12.0.0",
|
||||
"apache-arrow": "^12.0.0"
|
||||
}
|
||||
},
|
||||
"@types/command-line-args": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
|
||||
"integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA=="
|
||||
},
|
||||
"@types/command-line-usage": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz",
|
||||
"integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg=="
|
||||
},
|
||||
"@types/node": {
|
||||
"version": "18.14.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
|
||||
"integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw=="
|
||||
},
|
||||
"@types/pad-left": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz",
|
||||
"integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA=="
|
||||
},
|
||||
"ansi-styles": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
|
||||
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
|
||||
"requires": {
|
||||
"color-convert": "^1.9.0"
|
||||
}
|
||||
},
|
||||
"apache-arrow": {
|
||||
"version": "12.0.0",
|
||||
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.0.tgz",
|
||||
"integrity": "sha512-uI+hnZZsGfNJiR/wG8j5yPQuDjmOHx4hZpkA743G4x3TlFrCpA3MMX7KUkIOIw0e/CwZ8NYuaMzaQsblA47qVA==",
|
||||
"requires": {
|
||||
"@types/command-line-args": "5.2.0",
|
||||
"@types/command-line-usage": "5.0.2",
|
||||
"@types/node": "18.14.5",
|
||||
"@types/pad-left": "2.1.1",
|
||||
"command-line-args": "5.2.1",
|
||||
"command-line-usage": "6.1.3",
|
||||
"flatbuffers": "23.3.3",
|
||||
"json-bignum": "^0.0.3",
|
||||
"pad-left": "^2.1.0",
|
||||
"tslib": "^2.5.0"
|
||||
}
|
||||
},
|
||||
"array-back": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
|
||||
"integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q=="
|
||||
},
|
||||
"chalk": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
|
||||
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
|
||||
"requires": {
|
||||
"ansi-styles": "^3.2.1",
|
||||
"escape-string-regexp": "^1.0.5",
|
||||
"supports-color": "^5.3.0"
|
||||
}
|
||||
},
|
||||
"color-convert": {
|
||||
"version": "1.9.3",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
|
||||
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
|
||||
"requires": {
|
||||
"color-name": "1.1.3"
|
||||
}
|
||||
},
|
||||
"color-name": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
|
||||
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
|
||||
},
|
||||
"command-line-args": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
|
||||
"integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==",
|
||||
"requires": {
|
||||
"array-back": "^3.1.0",
|
||||
"find-replace": "^3.0.0",
|
||||
"lodash.camelcase": "^4.3.0",
|
||||
"typical": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"command-line-usage": {
|
||||
"version": "6.1.3",
|
||||
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz",
|
||||
"integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==",
|
||||
"requires": {
|
||||
"array-back": "^4.0.2",
|
||||
"chalk": "^2.4.2",
|
||||
"table-layout": "^1.0.2",
|
||||
"typical": "^5.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"array-back": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
|
||||
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg=="
|
||||
},
|
||||
"typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"deep-extend": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
|
||||
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
|
||||
},
|
||||
"escape-string-regexp": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
|
||||
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg=="
|
||||
},
|
||||
"find-replace": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz",
|
||||
"integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==",
|
||||
"requires": {
|
||||
"array-back": "^3.0.1"
|
||||
}
|
||||
},
|
||||
"flatbuffers": {
|
||||
"version": "23.3.3",
|
||||
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz",
|
||||
"integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g=="
|
||||
},
|
||||
"has-flag": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
|
||||
"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw=="
|
||||
},
|
||||
"json-bignum": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz",
|
||||
"integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg=="
|
||||
},
|
||||
"lodash.camelcase": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
|
||||
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
|
||||
},
|
||||
"pad-left": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz",
|
||||
"integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==",
|
||||
"requires": {
|
||||
"repeat-string": "^1.5.4"
|
||||
}
|
||||
},
|
||||
"reduce-flatten": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz",
|
||||
"integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w=="
|
||||
},
|
||||
"repeat-string": {
|
||||
"version": "1.6.1",
|
||||
"resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
|
||||
"integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w=="
|
||||
},
|
||||
"supports-color": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
|
||||
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
|
||||
"requires": {
|
||||
"has-flag": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"table-layout": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz",
|
||||
"integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==",
|
||||
"requires": {
|
||||
"array-back": "^4.0.1",
|
||||
"deep-extend": "~0.6.0",
|
||||
"typical": "^5.2.0",
|
||||
"wordwrapjs": "^4.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"array-back": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
|
||||
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg=="
|
||||
},
|
||||
"typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"tslib": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
|
||||
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
|
||||
},
|
||||
"typical": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz",
|
||||
"integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw=="
|
||||
},
|
||||
"wordwrapjs": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz",
|
||||
"integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==",
|
||||
"requires": {
|
||||
"reduce-flatten": "^2.0.0",
|
||||
"typical": "^5.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"typical": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
|
||||
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
14
node/examples/simple/package.json
Normal file
14
node/examples/simple/package.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "vectordb-example",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"vectordb": "^0.0.1"
|
||||
}
|
||||
}
|
||||
8
node/gen_test_data.py
Normal file
8
node/gen_test_data.py
Normal file
@@ -0,0 +1,8 @@
|
||||
import lancedb
|
||||
|
||||
uri = "sample-lancedb"
|
||||
db = lancedb.connect(uri)
|
||||
table = db.create_table("my_table",
|
||||
data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
|
||||
{"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
|
||||
|
||||
6870
node/package-lock.json
generated
Normal file
6870
node/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
48
node/package.json
Normal file
48
node/package.json
Normal file
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"name": "vectordb",
|
||||
"version": "0.0.1",
|
||||
"description": " Serverless, low-latency vector database for AI applications",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"tsc": "tsc -b",
|
||||
"build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
|
||||
"build-release": "npm run build -- --release",
|
||||
"install": "npm run build-release",
|
||||
"test": "mocha -recursive dist/test",
|
||||
"lint": "eslint src --ext .js,.ts"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/lancedb/lancedb/node"
|
||||
},
|
||||
"keywords": [
|
||||
"data-format",
|
||||
"data-science",
|
||||
"machine-learning",
|
||||
"data-analytics"
|
||||
],
|
||||
"author": "Lance Devs",
|
||||
"license": "Apache-2.0",
|
||||
"devDependencies": {
|
||||
"@types/chai": "^4.3.4",
|
||||
"@types/mocha": "^10.0.1",
|
||||
"@types/node": "^18.16.2",
|
||||
"@typescript-eslint/eslint-plugin": "^5.59.1",
|
||||
"cargo-cp-artifact": "^0.1",
|
||||
"chai": "^4.3.7",
|
||||
"eslint": "^8.39.0",
|
||||
"eslint-config-standard-with-typescript": "^34.0.1",
|
||||
"eslint-plugin-import": "^2.27.5",
|
||||
"eslint-plugin-n": "^15.7.0",
|
||||
"eslint-plugin-promise": "^6.1.1",
|
||||
"mocha": "^10.2.0",
|
||||
"ts-node": "^10.9.1",
|
||||
"ts-node-dev": "^2.0.0",
|
||||
"typescript": "*"
|
||||
},
|
||||
"dependencies": {
|
||||
"@apache-arrow/ts": "^12.0.0",
|
||||
"apache-arrow": "^12.0.0"
|
||||
}
|
||||
}
|
||||
BIN
node/sample-lancedb/my_table.lance/_latest.manifest
Normal file
BIN
node/sample-lancedb/my_table.lance/_latest.manifest
Normal file
Binary file not shown.
BIN
node/sample-lancedb/my_table.lance/_versions/1.manifest
Normal file
BIN
node/sample-lancedb/my_table.lance/_versions/1.manifest
Normal file
Binary file not shown.
Binary file not shown.
134
node/src/index.ts
Normal file
134
node/src/index.ts
Normal file
@@ -0,0 +1,134 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import { tableFromIPC, Vector } from 'apache-arrow'
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { databaseNew, databaseTableNames, databaseOpenTable, tableSearch } = require('../index.node')
|
||||
|
||||
/**
|
||||
* Connect to a LanceDB instance at the given URI
|
||||
* @param uri The uri of the database.
|
||||
*/
|
||||
export function connect (uri: string): Connection {
|
||||
return new Connection(uri)
|
||||
}
|
||||
|
||||
/**
|
||||
* A connection to a LanceDB database.
|
||||
*/
|
||||
export class Connection {
|
||||
private readonly _uri: string
|
||||
private readonly _db: any
|
||||
|
||||
constructor (uri: string) {
|
||||
this._uri = uri
|
||||
this._db = databaseNew(uri)
|
||||
}
|
||||
|
||||
get uri (): string {
|
||||
return this._uri
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the names of all tables in the database.
|
||||
*/
|
||||
tableNames (): string[] {
|
||||
return databaseTableNames.call(this._db)
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a table in the database.
|
||||
* @param name The name of the table.
|
||||
*/
|
||||
async openTable (name: string): Promise<Table> {
|
||||
const tbl = await databaseOpenTable.call(this._db, name)
|
||||
return new Table(tbl, name)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A table in a LanceDB database.
|
||||
*/
|
||||
export class Table {
|
||||
private readonly _tbl: any
|
||||
private readonly _name: string
|
||||
|
||||
constructor (tbl: any, name: string) {
|
||||
this._tbl = tbl
|
||||
this._name = name
|
||||
}
|
||||
|
||||
get name (): string {
|
||||
return this._name
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a search query to find the nearest neighbors of the given query vector.
|
||||
* @param queryVector The query vector.
|
||||
*/
|
||||
search(queryVector: number[]): Query {
|
||||
return new Query(this._tbl, queryVector)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A builder for nearest neighbor queries for LanceDB.
|
||||
*/
|
||||
export class Query {
|
||||
private readonly _tbl: any
|
||||
private readonly _query_vector: number[]
|
||||
private readonly _limit: number
|
||||
private readonly _refine_factor?: number
|
||||
private readonly _nprobes: number
|
||||
private readonly _columns?: string[]
|
||||
private readonly _where?: string
|
||||
private readonly _metric = 'L2'
|
||||
|
||||
constructor (tbl: any, queryVector: number[]) {
|
||||
this._tbl = tbl
|
||||
this._query_vector = queryVector
|
||||
this._limit = 10
|
||||
this._nprobes = 20
|
||||
this._refine_factor = undefined
|
||||
this._columns = undefined
|
||||
this._where = undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the query and return the results as an Array of Objects
|
||||
*/
|
||||
async execute (): Promise<unknown[]> {
|
||||
const buffer = await tableSearch.call(this._tbl, this._query_vector, this._limit)
|
||||
const data = tableFromIPC(buffer)
|
||||
return data.toArray().map((entry: Record<string, unknown>) => {
|
||||
const newObject: Record<string, unknown> = {}
|
||||
Object.keys(entry).forEach((key: string) => {
|
||||
if (entry[key] instanceof Vector) {
|
||||
newObject[key] = (entry[key] as Vector).toArray()
|
||||
} else {
|
||||
newObject[key] = entry[key]
|
||||
}
|
||||
})
|
||||
return newObject
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the query and return the results as an Array of the generic type provided
|
||||
*/
|
||||
async execute_cast<T>(): Promise<T[]> {
|
||||
return await this.execute() as T[]
|
||||
}
|
||||
}
|
||||
71
node/src/test/test.ts
Normal file
71
node/src/test/test.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import { describe } from 'mocha'
|
||||
import { assert } from 'chai'
|
||||
|
||||
import * as lancedb from '../index'
|
||||
|
||||
describe('LanceDB client', function () {
|
||||
describe('open a connection to lancedb', function () {
|
||||
const con = lancedb.connect('.../../sample-lancedb')
|
||||
|
||||
it('should have a valid url', function () {
|
||||
assert.equal(con.uri, '.../../sample-lancedb')
|
||||
})
|
||||
|
||||
it('should return the existing table names', function () {
|
||||
assert.deepEqual(con.tableNames(), ['my_table'])
|
||||
})
|
||||
|
||||
describe('open a table from a connection', function () {
|
||||
const tablePromise = con.openTable('my_table')
|
||||
|
||||
it('should have a valid name', async function () {
|
||||
const table = await tablePromise
|
||||
assert.equal(table.name, 'my_table')
|
||||
})
|
||||
|
||||
class MyResult {
|
||||
vector: Float32Array = new Float32Array(0)
|
||||
price: number = 0
|
||||
item: string = ''
|
||||
}
|
||||
|
||||
it('execute a query', async function () {
|
||||
const table = await tablePromise
|
||||
const builder = table.search([0.1, 0.3])
|
||||
const results = await builder.execute() as MyResult[]
|
||||
|
||||
assert.equal(results.length, 2)
|
||||
assert.equal(results[0].item, 'foo')
|
||||
assert.equal(results[0].price, 10)
|
||||
assert.approximately(results[0].vector[0], 3.1, 0.1)
|
||||
assert.approximately(results[0].vector[1], 4.1, 0.1)
|
||||
})
|
||||
|
||||
it('execute a query and type cast the result', async function () {
|
||||
const table = await tablePromise
|
||||
|
||||
const builder = table.search([0.1, 0.3])
|
||||
const results = await builder.execute_cast<MyResult>()
|
||||
assert.equal(results.length, 2)
|
||||
assert.equal(results[0].item, 'foo')
|
||||
assert.equal(results[0].price, 10)
|
||||
assert.approximately(results[0].vector[0], 3.1, 0.1)
|
||||
assert.approximately(results[0].vector[1], 4.1, 0.1)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
10
node/tsconfig.json
Normal file
10
node/tsconfig.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"include": ["src/**/*.ts"],
|
||||
"compilerOptions": {
|
||||
"target": "es2016",
|
||||
"module": "commonjs",
|
||||
"declaration": true,
|
||||
"outDir": "./dist",
|
||||
"strict": true
|
||||
}
|
||||
}
|
||||
19
rust/ffi/node/Cargo.toml
Normal file
19
rust/ffi/node/Cargo.toml
Normal file
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "vectordb-node"
|
||||
version = "0.1.0"
|
||||
description = "Serverless, low-latency vector database for AI applications"
|
||||
license = "Apache-2.0"
|
||||
edition = "2018"
|
||||
exclude = ["index.node"]
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
arrow-array = "37.0"
|
||||
arrow-ipc = "37.0"
|
||||
once_cell = "1"
|
||||
futures = "0.3"
|
||||
vectordb = { path = "../../vectordb" }
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread"] }
|
||||
neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }
|
||||
3
rust/ffi/node/README.md
Normal file
3
rust/ffi/node/README.md
Normal file
@@ -0,0 +1,3 @@
|
||||
The LanceDB node bridge (vectordb-node) allows javascript applications to access LanceDB datasets.
|
||||
|
||||
It is build using [Neon](https://neon-bindings.com). See the node project for an example of how it is used / tests
|
||||
36
rust/ffi/node/src/convert.rs
Normal file
36
rust/ffi/node/src/convert.rs
Normal file
@@ -0,0 +1,36 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use neon::prelude::*;
|
||||
|
||||
pub(crate) fn vec_str_to_array<'a, C: Context<'a>>(
|
||||
vec: &Vec<String>,
|
||||
cx: &mut C,
|
||||
) -> JsResult<'a, JsArray> {
|
||||
let a = JsArray::new(cx, vec.len() as u32);
|
||||
for (i, s) in vec.iter().enumerate() {
|
||||
let v = cx.string(s);
|
||||
a.set(cx, i as u32, v)?;
|
||||
}
|
||||
Ok(a)
|
||||
}
|
||||
|
||||
pub(crate) fn js_array_to_vec(array: &JsArray, cx: &mut FunctionContext) -> Vec<f32> {
|
||||
let mut query_vec: Vec<f32> = Vec::new();
|
||||
for i in 0..array.len(cx) {
|
||||
let entry: Handle<JsNumber> = array.get(cx, i).unwrap();
|
||||
query_vec.push(entry.value(cx) as f32);
|
||||
}
|
||||
query_vec
|
||||
}
|
||||
145
rust/ffi/node/src/lib.rs
Normal file
145
rust/ffi/node/src/lib.rs
Normal file
@@ -0,0 +1,145 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod convert;
|
||||
|
||||
use std::ops::Deref;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::Float32Array;
|
||||
use arrow_ipc::writer::FileWriter;
|
||||
use futures::TryStreamExt;
|
||||
use neon::prelude::*;
|
||||
use once_cell::sync::OnceCell;
|
||||
use tokio::runtime::Runtime;
|
||||
|
||||
use vectordb::database::Database;
|
||||
use vectordb::table::Table;
|
||||
|
||||
struct JsDatabase {
|
||||
database: Arc<Database>,
|
||||
}
|
||||
|
||||
struct JsTable {
|
||||
table: Arc<Table>,
|
||||
}
|
||||
|
||||
impl Finalize for JsDatabase {}
|
||||
|
||||
impl Finalize for JsTable {}
|
||||
|
||||
fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
|
||||
static RUNTIME: OnceCell<Runtime> = OnceCell::new();
|
||||
|
||||
RUNTIME.get_or_try_init(|| Runtime::new().or_else(|err| cx.throw_error(err.to_string())))
|
||||
}
|
||||
|
||||
fn database_new(mut cx: FunctionContext) -> JsResult<JsBox<JsDatabase>> {
|
||||
let path = cx.argument::<JsString>(0)?.value(&mut cx);
|
||||
let db = JsDatabase {
|
||||
database: Arc::new(Database::connect(path).or_else(|err| cx.throw_error(err.to_string()))?),
|
||||
};
|
||||
Ok(cx.boxed(db))
|
||||
}
|
||||
|
||||
fn database_table_names(mut cx: FunctionContext) -> JsResult<JsArray> {
|
||||
let db = cx
|
||||
.this()
|
||||
.downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
|
||||
let tables = db
|
||||
.database
|
||||
.table_names()
|
||||
.or_else(|err| cx.throw_error(err.to_string()))?;
|
||||
convert::vec_str_to_array(&tables, &mut cx)
|
||||
}
|
||||
|
||||
fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
|
||||
let db = cx
|
||||
.this()
|
||||
.downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
|
||||
let table_name = cx.argument::<JsString>(0)?.value(&mut cx);
|
||||
|
||||
let rt = runtime(&mut cx)?;
|
||||
let channel = cx.channel();
|
||||
let database = db.database.clone();
|
||||
|
||||
let (deferred, promise) = cx.promise();
|
||||
rt.spawn(async move {
|
||||
let table_rst = database.open_table(table_name).await;
|
||||
|
||||
deferred.settle_with(&channel, move |mut cx| {
|
||||
let table = Arc::new(table_rst.or_else(|err| cx.throw_error(err.to_string()))?);
|
||||
Ok(cx.boxed(JsTable { table }))
|
||||
});
|
||||
});
|
||||
Ok(promise)
|
||||
}
|
||||
|
||||
fn table_search(mut cx: FunctionContext) -> JsResult<JsPromise> {
|
||||
let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
|
||||
let query_vector = cx.argument::<JsArray>(0)?; //. .as_value(&mut cx);
|
||||
let limit = cx.argument::<JsNumber>(1)?.value(&mut cx);
|
||||
|
||||
let rt = runtime(&mut cx)?;
|
||||
let channel = cx.channel();
|
||||
|
||||
let (deferred, promise) = cx.promise();
|
||||
let table = js_table.table.clone();
|
||||
let query = convert::js_array_to_vec(query_vector.deref(), &mut cx);
|
||||
|
||||
rt.spawn(async move {
|
||||
let builder = table
|
||||
.search(Float32Array::from(query))
|
||||
.limit(limit as usize);
|
||||
let results = builder
|
||||
.execute()
|
||||
.await
|
||||
.unwrap() // FIXME unwrap
|
||||
.try_collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
deferred.settle_with(&channel, move |mut cx| {
|
||||
let results = results.or_else(|err| cx.throw_error(err.to_string()))?;
|
||||
let vector: Vec<u8> = Vec::new();
|
||||
|
||||
if results.is_empty() {
|
||||
return cx.buffer(0);
|
||||
}
|
||||
|
||||
let schema = results.get(0).unwrap().schema();
|
||||
let mut fr = FileWriter::try_new(vector, schema.deref())
|
||||
.or_else(|err| cx.throw_error(err.to_string()))?;
|
||||
|
||||
for batch in results.iter() {
|
||||
fr.write(batch)
|
||||
.or_else(|err| cx.throw_error(err.to_string()))?;
|
||||
}
|
||||
fr.finish().or_else(|err| cx.throw_error(err.to_string()))?;
|
||||
let buf = fr
|
||||
.into_inner()
|
||||
.or_else(|err| cx.throw_error(err.to_string()))?;
|
||||
Ok(JsBuffer::external(&mut cx, buf))
|
||||
});
|
||||
});
|
||||
Ok(promise)
|
||||
}
|
||||
|
||||
#[neon::main]
|
||||
fn main(mut cx: ModuleContext) -> NeonResult<()> {
|
||||
cx.export_function("databaseNew", database_new)?;
|
||||
cx.export_function("databaseTableNames", database_table_names)?;
|
||||
cx.export_function("databaseOpenTable", database_open_table)?;
|
||||
cx.export_function("tableSearch", table_search)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
pub fn add(left: usize, right: usize) -> usize {
|
||||
left + right
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn it_works() {
|
||||
let result = add(2, 2);
|
||||
assert_eq!(result, 4);
|
||||
}
|
||||
}
|
||||
@@ -9,4 +9,10 @@ repository = "https://github.com/lancedb/lancedb"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
arrow-array = "37.0"
|
||||
arrow-schema = "37.0"
|
||||
lance = "0.4.3"
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.5.0"
|
||||
118
rust/vectordb/src/database.rs
Normal file
118
rust/vectordb/src/database.rs
Normal file
@@ -0,0 +1,118 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fs::create_dir_all;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::table::Table;
|
||||
|
||||
pub struct Database {
|
||||
pub(crate) path: Arc<PathBuf>,
|
||||
}
|
||||
|
||||
const LANCE_EXTENSION: &str = "lance";
|
||||
|
||||
/// A connection to LanceDB
|
||||
impl Database {
|
||||
/// Connects to LanceDB
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `path` - URI where the database is located, can be a local file or a supported remote cloud storage
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * A [Database] object.
|
||||
pub fn connect<P: AsRef<Path>>(path: P) -> Result<Database> {
|
||||
if !path.as_ref().try_exists()? {
|
||||
create_dir_all(&path)?;
|
||||
}
|
||||
Ok(Database {
|
||||
path: Arc::new(path.as_ref().to_path_buf()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the names of all tables in the database.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * A [Vec<String>] with all table names.
|
||||
pub fn table_names(&self) -> Result<Vec<String>> {
|
||||
let f = self
|
||||
.path
|
||||
.read_dir()?
|
||||
.flatten()
|
||||
.map(|dir_entry| dir_entry.path())
|
||||
.filter(|path| {
|
||||
let is_lance = path
|
||||
.extension()
|
||||
.map(|e| e.to_str().map(|e| e == LANCE_EXTENSION))
|
||||
.flatten();
|
||||
is_lance.unwrap_or(false)
|
||||
})
|
||||
.map(|p| {
|
||||
p.file_stem()
|
||||
.map(|s| s.to_str().map(|s| String::from(s)))
|
||||
.flatten()
|
||||
})
|
||||
.flatten()
|
||||
.collect();
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
/// Open a table in the database.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `name` - The name of the table.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * A [Table] object.
|
||||
pub async fn open_table(&self, name: String) -> Result<Table> {
|
||||
Table::new(self.path.clone(), name).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs::create_dir_all;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use crate::database::Database;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_connect() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let path_buf = tmp_dir.into_path();
|
||||
let db = Database::connect(&path_buf);
|
||||
|
||||
assert_eq!(db.unwrap().path.as_path(), path_buf.as_path())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_table_names() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
create_dir_all(tmp_dir.path().join("table1.lance")).unwrap();
|
||||
create_dir_all(tmp_dir.path().join("table2.lance")).unwrap();
|
||||
create_dir_all(tmp_dir.path().join("invalidlance")).unwrap();
|
||||
|
||||
let db = Database::connect(&tmp_dir.into_path()).unwrap();
|
||||
let tables = db.table_names().unwrap();
|
||||
assert_eq!(tables.len(), 2);
|
||||
assert!(tables.contains(&String::from("table1")));
|
||||
assert!(tables.contains(&String::from("table2")));
|
||||
}
|
||||
}
|
||||
43
rust/vectordb/src/error.rs
Normal file
43
rust/vectordb/src/error.rs
Normal file
@@ -0,0 +1,43 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
IO(String),
|
||||
Lance(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let (catalog, message) = match self {
|
||||
Self::IO(s) => ("I/O", s.as_str()),
|
||||
Self::Lance(s) => ("Lance", s.as_str()),
|
||||
};
|
||||
write!(f, "LanceDBError({catalog}): {message}")
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl From<std::io::Error> for Error {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
Self::IO(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lance::Error> for Error {
|
||||
fn from(e: lance::Error) -> Self {
|
||||
Self::Lance(e.to_string())
|
||||
}
|
||||
}
|
||||
18
rust/vectordb/src/lib.rs
Normal file
18
rust/vectordb/src/lib.rs
Normal file
@@ -0,0 +1,18 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod database;
|
||||
pub mod error;
|
||||
pub mod query;
|
||||
pub mod table;
|
||||
210
rust/vectordb/src/query.rs
Normal file
210
rust/vectordb/src/query.rs
Normal file
@@ -0,0 +1,210 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::Float32Array;
|
||||
use lance::dataset::scanner::{DatasetRecordBatchStream, Scanner};
|
||||
use lance::dataset::Dataset;
|
||||
use lance::index::vector::MetricType;
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
/// A builder for nearest neighbor queries for LanceDB.
|
||||
pub struct Query {
|
||||
pub dataset: Arc<Dataset>,
|
||||
pub query_vector: Float32Array,
|
||||
pub limit: usize,
|
||||
pub nprobes: usize,
|
||||
pub refine_factor: Option<u32>,
|
||||
pub metric_type: MetricType,
|
||||
pub use_index: bool,
|
||||
}
|
||||
|
||||
impl Query {
|
||||
/// Creates a new Query object
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `dataset` - The table / dataset the query will be run against.
|
||||
/// * `vector` The vector used for this query.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * A [Query] object.
|
||||
pub(crate) fn new(dataset: Arc<Dataset>, vector: Float32Array) -> Self {
|
||||
Query {
|
||||
dataset,
|
||||
query_vector: vector,
|
||||
limit: 10,
|
||||
nprobes: 20,
|
||||
refine_factor: None,
|
||||
metric_type: MetricType::L2,
|
||||
use_index: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute the queries and return its results.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * A [DatasetRecordBatchStream] with the query's results.
|
||||
pub async fn execute(&self) -> Result<DatasetRecordBatchStream> {
|
||||
let mut scanner: Scanner = self.dataset.scan();
|
||||
|
||||
scanner.nearest(
|
||||
crate::table::VECTOR_COLUMN_NAME,
|
||||
&self.query_vector,
|
||||
self.limit,
|
||||
)?;
|
||||
scanner.nprobs(self.nprobes);
|
||||
scanner.distance_metric(self.metric_type);
|
||||
scanner.use_index(self.use_index);
|
||||
self.refine_factor.map(|rf| scanner.refine(rf));
|
||||
Ok(scanner.try_into_stream().await?)
|
||||
}
|
||||
|
||||
/// Set the maximum number of results to return.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `limit` - The maximum number of results to return.
|
||||
pub fn limit(mut self, limit: usize) -> Query {
|
||||
self.limit = limit;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the vector used for this query.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `vector` - The vector that will be used for search.
|
||||
pub fn query_vector(mut self, query_vector: Float32Array) -> Query {
|
||||
self.query_vector = query_vector;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the number of probes to use.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `nprobes` - The number of probes to use.
|
||||
pub fn nprobes(mut self, nprobes: usize) -> Query {
|
||||
self.nprobes = nprobes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the refine factor to use.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `refine_factor` - The refine factor to use.
|
||||
pub fn refine_factor(mut self, refine_factor: Option<u32>) -> Query {
|
||||
self.refine_factor = refine_factor;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the distance metric to use.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `metric_type` - The distance metric to use. By default [MetricType::L2] is used.
|
||||
pub fn metric_type(mut self, metric_type: MetricType) -> Query {
|
||||
self.metric_type = metric_type;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether to use an ANN index if available
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `use_index` - Sets Whether to use an ANN index if available
|
||||
pub fn use_index(mut self, use_index: bool) -> Query {
|
||||
self.use_index = use_index;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{Float32Array, RecordBatch, RecordBatchReader};
|
||||
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
|
||||
use lance::arrow::RecordBatchBuffer;
|
||||
use lance::dataset::Dataset;
|
||||
use lance::index::vector::MetricType;
|
||||
|
||||
use crate::query::Query;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_setters_getters() {
|
||||
let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
|
||||
let ds = Dataset::write(&mut batches, ":memory:", None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let vector = Float32Array::from_iter_values([0.1, 0.2]);
|
||||
let query = Query::new(Arc::new(ds), vector.clone());
|
||||
assert_eq!(query.query_vector, vector);
|
||||
|
||||
let new_vector = Float32Array::from_iter_values([9.8, 8.7]);
|
||||
|
||||
let query = query
|
||||
.query_vector(new_vector.clone())
|
||||
.limit(100)
|
||||
.nprobes(1000)
|
||||
.use_index(true)
|
||||
.metric_type(MetricType::Cosine)
|
||||
.refine_factor(Some(999));
|
||||
|
||||
assert_eq!(query.query_vector, new_vector);
|
||||
assert_eq!(query.limit, 100);
|
||||
assert_eq!(query.nprobes, 1000);
|
||||
assert_eq!(query.use_index, true);
|
||||
assert_eq!(query.metric_type, MetricType::Cosine);
|
||||
assert_eq!(query.refine_factor, Some(999));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute() {
|
||||
let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
|
||||
let ds = Dataset::write(&mut batches, ":memory:", None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let vector = Float32Array::from_iter_values([0.1; 128]);
|
||||
let query = Query::new(Arc::new(ds), vector.clone());
|
||||
let result = query.execute().await;
|
||||
assert_eq!(result.is_ok(), true);
|
||||
}
|
||||
|
||||
fn make_test_batches() -> RecordBatchBuffer {
|
||||
let dim: usize = 128;
|
||||
let schema = Arc::new(ArrowSchema::new(vec![
|
||||
ArrowField::new("key", DataType::Int32, false),
|
||||
ArrowField::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(
|
||||
Arc::new(ArrowField::new("item", DataType::Float32, true)),
|
||||
dim as i32,
|
||||
),
|
||||
true,
|
||||
),
|
||||
ArrowField::new("uri", DataType::Utf8, true),
|
||||
]));
|
||||
|
||||
RecordBatchBuffer::new(vec![RecordBatch::new_empty(schema.clone())])
|
||||
}
|
||||
}
|
||||
144
rust/vectordb/src/table.rs
Normal file
144
rust/vectordb/src/table.rs
Normal file
@@ -0,0 +1,144 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::Float32Array;
|
||||
use lance::dataset::Dataset;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::query::Query;
|
||||
|
||||
pub const VECTOR_COLUMN_NAME: &str = "vector";
|
||||
|
||||
pub const LANCE_FILE_EXTENSION: &str = "lance";
|
||||
|
||||
/// A table in a LanceDB database.
|
||||
pub struct Table {
|
||||
name: String,
|
||||
dataset: Arc<Dataset>,
|
||||
}
|
||||
|
||||
impl Table {
|
||||
/// Creates a new Table object
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `base_path` - The base path where the table is located
|
||||
/// * `name` The Table name
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * A [Table] object.
|
||||
pub async fn new(base_path: Arc<PathBuf>, name: String) -> Result<Self> {
|
||||
let ds_path = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
|
||||
let ds_uri = ds_path
|
||||
.to_str()
|
||||
.ok_or(Error::IO(format!("Unable to find table {}", name)))?;
|
||||
let dataset = Dataset::open(ds_uri).await?;
|
||||
let table = Table {
|
||||
name,
|
||||
dataset: Arc::new(dataset),
|
||||
};
|
||||
Ok(table)
|
||||
}
|
||||
|
||||
/// Creates a new Query object that can be executed.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `vector` The vector used for this query.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * A [Query] object.
|
||||
pub fn search(&self, query_vector: Float32Array) -> Query {
|
||||
Query::new(self.dataset.clone(), query_vector)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use arrow_array::{Float32Array, Int32Array, RecordBatch, RecordBatchReader};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use lance::arrow::RecordBatchBuffer;
|
||||
use lance::dataset::Dataset;
|
||||
use std::sync::Arc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
use crate::table::Table;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_table_not_exists() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let path_buf = tmp_dir.into_path();
|
||||
|
||||
let table = Table::new(Arc::new(path_buf), "test".to_string()).await;
|
||||
assert!(table.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let path_buf = tmp_dir.into_path();
|
||||
|
||||
let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
|
||||
Dataset::write(
|
||||
&mut batches,
|
||||
path_buf.join("test.lance").to_str().unwrap(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let table = Table::new(Arc::new(path_buf), "test".to_string())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(table.name, "test")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_search() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let path_buf = tmp_dir.into_path();
|
||||
|
||||
let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
|
||||
Dataset::write(
|
||||
&mut batches,
|
||||
path_buf.join("test.lance").to_str().unwrap(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let table = Table::new(Arc::new(path_buf), "test".to_string())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let vector = Float32Array::from_iter_values([0.1, 0.2]);
|
||||
let query = table.search(vector.clone());
|
||||
assert_eq!(vector, query.query_vector);
|
||||
}
|
||||
|
||||
fn make_test_batches() -> RecordBatchBuffer {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)]));
|
||||
RecordBatchBuffer::new(vec![RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(Int32Array::from_iter_values(0..20))],
|
||||
)
|
||||
.unwrap()])
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user