JavaScript / Node.js library for LanceDB

- Core rust library
- ffi bridge that exposes rust functionality to javascript
- npm package that provides a TypeScript / JavaScript library
- limitations: it only supports reading for now
This commit is contained in:
gsilvestrin
2023-05-10 12:51:49 -07:00
parent 636a6d3761
commit 38e6efc185
28 changed files with 12343 additions and 14 deletions

10
.gitignore vendored
View File

@@ -18,3 +18,13 @@ python/dist
notebooks/.ipynb_checkpoints
**/.hypothesis
## Javascript
index.node
**/node_modules
**/.DS_Store
node/dist
## Rust
target

3789
Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

5
Cargo.toml Normal file
View File

@@ -0,0 +1,5 @@
[workspace]
members = [
"rust/vectordb",
"rust/ffi/node"
]

16
node/.eslintrc.js Normal file
View File

@@ -0,0 +1,16 @@
module.exports = {
env: {
browser: true,
es2021: true
},
extends: 'standard-with-typescript',
overrides: [
],
parserOptions: {
project: './tsconfig.json',
ecmaVersion: 'latest',
sourceType: 'module'
},
rules: {
}
}

30
node/README.md Normal file
View File

@@ -0,0 +1,30 @@
A JavaScript / Node.js library for [LanceDB](https://github.com/lancedb/lancedb).
## Quick Start
```bash
npm i vectordb
```
See the examples folder for usage.
## Development
The LanceDB javascript is built with npm:
```bash
npm run tsc
```
Run the tests with
```bash
npm test
```
To run the linter and have it automatically fix all errors
```bash
npm run lint -- --fix
```

View File

@@ -0,0 +1,21 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
'use strict'
const lancedb = require('@lancedb/vectordb')
const db = lancedb.connect('../../sample-lancedb')
console.log(db.tableNames())

575
node/examples/simple/package-lock.json generated Normal file
View File

@@ -0,0 +1,575 @@
{
"name": "vectordb-example",
"version": "1.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "vectordb-example",
"version": "1.0.0",
"license": "Apache-2.0",
"dependencies": {
"@lancedb/vectordb": "^0.0.1"
}
},
"node_modules/@apache-arrow/ts": {
"version": "12.0.0",
"resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.0.tgz",
"integrity": "sha512-ArJ3Fw5W9RAeNWuyCU2CdjL/nEAZSVDG1p3jz/ZtLo/q3NTz2w7HUCOJeszejH/5alGX+QirYrJ5c6BW++/P7g==",
"dependencies": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "18.14.5",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "6.1.3",
"flatbuffers": "23.3.3",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.0"
}
},
"node_modules/@lancedb/vectordb": {
"version": "0.0.1",
"resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb/0.0.1/3c0f01b97a74046f90807202605497536d982cd9",
"integrity": "sha512-Bq6nuyCXOir7wnokhHt7GruydPFZxEQvTi7qxIBH28i8fjU2nVftElysKimm2TgaUnR756S15e4kL/Oma030MQ==",
"hasInstallScript": true,
"license": "Apache-2.0",
"dependencies": {
"@apache-arrow/ts": "^12.0.0",
"apache-arrow": "^12.0.0"
}
},
"node_modules/@types/command-line-args": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
"integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA=="
},
"node_modules/@types/command-line-usage": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz",
"integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg=="
},
"node_modules/@types/node": {
"version": "18.14.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
"integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw=="
},
"node_modules/@types/pad-left": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz",
"integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA=="
},
"node_modules/ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"dependencies": {
"color-convert": "^1.9.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/apache-arrow": {
"version": "12.0.0",
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.0.tgz",
"integrity": "sha512-uI+hnZZsGfNJiR/wG8j5yPQuDjmOHx4hZpkA743G4x3TlFrCpA3MMX7KUkIOIw0e/CwZ8NYuaMzaQsblA47qVA==",
"dependencies": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "18.14.5",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "6.1.3",
"flatbuffers": "23.3.3",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.0"
},
"bin": {
"arrow2csv": "bin/arrow2csv.js"
}
},
"node_modules/array-back": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
"integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==",
"engines": {
"node": ">=6"
}
},
"node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
"dependencies": {
"ansi-styles": "^3.2.1",
"escape-string-regexp": "^1.0.5",
"supports-color": "^5.3.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"dependencies": {
"color-name": "1.1.3"
}
},
"node_modules/color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
},
"node_modules/command-line-args": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
"integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==",
"dependencies": {
"array-back": "^3.1.0",
"find-replace": "^3.0.0",
"lodash.camelcase": "^4.3.0",
"typical": "^4.0.0"
},
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/command-line-usage": {
"version": "6.1.3",
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz",
"integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==",
"dependencies": {
"array-back": "^4.0.2",
"chalk": "^2.4.2",
"table-layout": "^1.0.2",
"typical": "^5.2.0"
},
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/command-line-usage/node_modules/array-back": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==",
"engines": {
"node": ">=8"
}
},
"node_modules/command-line-usage/node_modules/typical": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
"engines": {
"node": ">=8"
}
},
"node_modules/deep-extend": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
"engines": {
"node": ">=0.8.0"
}
},
"node_modules/find-replace": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz",
"integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==",
"dependencies": {
"array-back": "^3.0.1"
},
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/flatbuffers": {
"version": "23.3.3",
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz",
"integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g=="
},
"node_modules/has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
"engines": {
"node": ">=4"
}
},
"node_modules/json-bignum": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz",
"integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==",
"engines": {
"node": ">=0.8"
}
},
"node_modules/lodash.camelcase": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
},
"node_modules/pad-left": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz",
"integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==",
"dependencies": {
"repeat-string": "^1.5.4"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/reduce-flatten": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz",
"integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==",
"engines": {
"node": ">=6"
}
},
"node_modules/repeat-string": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
"integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==",
"engines": {
"node": ">=0.10"
}
},
"node_modules/supports-color": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
"dependencies": {
"has-flag": "^3.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/table-layout": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz",
"integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==",
"dependencies": {
"array-back": "^4.0.1",
"deep-extend": "~0.6.0",
"typical": "^5.2.0",
"wordwrapjs": "^4.0.0"
},
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/table-layout/node_modules/array-back": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==",
"engines": {
"node": ">=8"
}
},
"node_modules/table-layout/node_modules/typical": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
"engines": {
"node": ">=8"
}
},
"node_modules/tslib": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
},
"node_modules/typical": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz",
"integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==",
"engines": {
"node": ">=8"
}
},
"node_modules/wordwrapjs": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz",
"integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==",
"dependencies": {
"reduce-flatten": "^2.0.0",
"typical": "^5.2.0"
},
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/wordwrapjs/node_modules/typical": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==",
"engines": {
"node": ">=8"
}
}
},
"dependencies": {
"@apache-arrow/ts": {
"version": "12.0.0",
"resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-12.0.0.tgz",
"integrity": "sha512-ArJ3Fw5W9RAeNWuyCU2CdjL/nEAZSVDG1p3jz/ZtLo/q3NTz2w7HUCOJeszejH/5alGX+QirYrJ5c6BW++/P7g==",
"requires": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "18.14.5",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "6.1.3",
"flatbuffers": "23.3.3",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.0"
}
},
"@lancedb/vectordb": {
"version": "0.0.1",
"resolved": "https://npm.pkg.github.com/download/@lancedb/vectordb/0.0.1/3c0f01b97a74046f90807202605497536d982cd9",
"integrity": "sha512-Bq6nuyCXOir7wnokhHt7GruydPFZxEQvTi7qxIBH28i8fjU2nVftElysKimm2TgaUnR756S15e4kL/Oma030MQ==",
"requires": {
"@apache-arrow/ts": "^12.0.0",
"apache-arrow": "^12.0.0"
}
},
"@types/command-line-args": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
"integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA=="
},
"@types/command-line-usage": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz",
"integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg=="
},
"@types/node": {
"version": "18.14.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.14.5.tgz",
"integrity": "sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw=="
},
"@types/pad-left": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz",
"integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA=="
},
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"requires": {
"color-convert": "^1.9.0"
}
},
"apache-arrow": {
"version": "12.0.0",
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-12.0.0.tgz",
"integrity": "sha512-uI+hnZZsGfNJiR/wG8j5yPQuDjmOHx4hZpkA743G4x3TlFrCpA3MMX7KUkIOIw0e/CwZ8NYuaMzaQsblA47qVA==",
"requires": {
"@types/command-line-args": "5.2.0",
"@types/command-line-usage": "5.0.2",
"@types/node": "18.14.5",
"@types/pad-left": "2.1.1",
"command-line-args": "5.2.1",
"command-line-usage": "6.1.3",
"flatbuffers": "23.3.3",
"json-bignum": "^0.0.3",
"pad-left": "^2.1.0",
"tslib": "^2.5.0"
}
},
"array-back": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
"integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q=="
},
"chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
"requires": {
"ansi-styles": "^3.2.1",
"escape-string-regexp": "^1.0.5",
"supports-color": "^5.3.0"
}
},
"color-convert": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
"requires": {
"color-name": "1.1.3"
}
},
"color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="
},
"command-line-args": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
"integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==",
"requires": {
"array-back": "^3.1.0",
"find-replace": "^3.0.0",
"lodash.camelcase": "^4.3.0",
"typical": "^4.0.0"
}
},
"command-line-usage": {
"version": "6.1.3",
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-6.1.3.tgz",
"integrity": "sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==",
"requires": {
"array-back": "^4.0.2",
"chalk": "^2.4.2",
"table-layout": "^1.0.2",
"typical": "^5.2.0"
},
"dependencies": {
"array-back": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg=="
},
"typical": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
}
}
},
"deep-extend": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
},
"escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg=="
},
"find-replace": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz",
"integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==",
"requires": {
"array-back": "^3.0.1"
}
},
"flatbuffers": {
"version": "23.3.3",
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.3.3.tgz",
"integrity": "sha512-jmreOaAT1t55keaf+Z259Tvh8tR/Srry9K8dgCgvizhKSEr6gLGgaOJI2WFL5fkOpGOGRZwxUrlFn0GCmXUy6g=="
},
"has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw=="
},
"json-bignum": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz",
"integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg=="
},
"lodash.camelcase": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
},
"pad-left": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz",
"integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==",
"requires": {
"repeat-string": "^1.5.4"
}
},
"reduce-flatten": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz",
"integrity": "sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w=="
},
"repeat-string": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
"integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w=="
},
"supports-color": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
"requires": {
"has-flag": "^3.0.0"
}
},
"table-layout": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-1.0.2.tgz",
"integrity": "sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==",
"requires": {
"array-back": "^4.0.1",
"deep-extend": "~0.6.0",
"typical": "^5.2.0",
"wordwrapjs": "^4.0.0"
},
"dependencies": {
"array-back": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-4.0.2.tgz",
"integrity": "sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg=="
},
"typical": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
}
}
},
"tslib": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
},
"typical": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz",
"integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw=="
},
"wordwrapjs": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-4.0.1.tgz",
"integrity": "sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==",
"requires": {
"reduce-flatten": "^2.0.0",
"typical": "^5.2.0"
},
"dependencies": {
"typical": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-5.2.0.tgz",
"integrity": "sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg=="
}
}
}
}
}

View File

@@ -0,0 +1,14 @@
{
"name": "vectordb-example",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "Apache-2.0",
"dependencies": {
"vectordb": "^0.0.1"
}
}

8
node/gen_test_data.py Normal file
View File

@@ -0,0 +1,8 @@
import lancedb
uri = "sample-lancedb"
db = lancedb.connect(uri)
table = db.create_table("my_table",
data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
{"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])

6870
node/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

48
node/package.json Normal file
View File

@@ -0,0 +1,48 @@
{
"name": "vectordb",
"version": "0.0.1",
"description": " Serverless, low-latency vector database for AI applications",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"tsc": "tsc -b",
"build": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cargo build --message-format=json-render-diagnostics",
"build-release": "npm run build -- --release",
"install": "npm run build-release",
"test": "mocha -recursive dist/test",
"lint": "eslint src --ext .js,.ts"
},
"repository": {
"type": "git",
"url": "https://github.com/lancedb/lancedb/node"
},
"keywords": [
"data-format",
"data-science",
"machine-learning",
"data-analytics"
],
"author": "Lance Devs",
"license": "Apache-2.0",
"devDependencies": {
"@types/chai": "^4.3.4",
"@types/mocha": "^10.0.1",
"@types/node": "^18.16.2",
"@typescript-eslint/eslint-plugin": "^5.59.1",
"cargo-cp-artifact": "^0.1",
"chai": "^4.3.7",
"eslint": "^8.39.0",
"eslint-config-standard-with-typescript": "^34.0.1",
"eslint-plugin-import": "^2.27.5",
"eslint-plugin-n": "^15.7.0",
"eslint-plugin-promise": "^6.1.1",
"mocha": "^10.2.0",
"ts-node": "^10.9.1",
"ts-node-dev": "^2.0.0",
"typescript": "*"
},
"dependencies": {
"@apache-arrow/ts": "^12.0.0",
"apache-arrow": "^12.0.0"
}
}

Binary file not shown.

134
node/src/index.ts Normal file
View File

@@ -0,0 +1,134 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { tableFromIPC, Vector } from 'apache-arrow'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { databaseNew, databaseTableNames, databaseOpenTable, tableSearch } = require('../index.node')
/**
* Connect to a LanceDB instance at the given URI
* @param uri The uri of the database.
*/
export function connect (uri: string): Connection {
return new Connection(uri)
}
/**
* A connection to a LanceDB database.
*/
export class Connection {
private readonly _uri: string
private readonly _db: any
constructor (uri: string) {
this._uri = uri
this._db = databaseNew(uri)
}
get uri (): string {
return this._uri
}
/**
* Get the names of all tables in the database.
*/
tableNames (): string[] {
return databaseTableNames.call(this._db)
}
/**
* Open a table in the database.
* @param name The name of the table.
*/
async openTable (name: string): Promise<Table> {
const tbl = await databaseOpenTable.call(this._db, name)
return new Table(tbl, name)
}
}
/**
* A table in a LanceDB database.
*/
export class Table {
private readonly _tbl: any
private readonly _name: string
constructor (tbl: any, name: string) {
this._tbl = tbl
this._name = name
}
get name (): string {
return this._name
}
/**
* Create a search query to find the nearest neighbors of the given query vector.
* @param queryVector The query vector.
*/
search(queryVector: number[]): Query {
return new Query(this._tbl, queryVector)
}
}
/**
* A builder for nearest neighbor queries for LanceDB.
*/
export class Query {
private readonly _tbl: any
private readonly _query_vector: number[]
private readonly _limit: number
private readonly _refine_factor?: number
private readonly _nprobes: number
private readonly _columns?: string[]
private readonly _where?: string
private readonly _metric = 'L2'
constructor (tbl: any, queryVector: number[]) {
this._tbl = tbl
this._query_vector = queryVector
this._limit = 10
this._nprobes = 20
this._refine_factor = undefined
this._columns = undefined
this._where = undefined
}
/**
* Execute the query and return the results as an Array of Objects
*/
async execute (): Promise<unknown[]> {
const buffer = await tableSearch.call(this._tbl, this._query_vector, this._limit)
const data = tableFromIPC(buffer)
return data.toArray().map((entry: Record<string, unknown>) => {
const newObject: Record<string, unknown> = {}
Object.keys(entry).forEach((key: string) => {
if (entry[key] instanceof Vector) {
newObject[key] = (entry[key] as Vector).toArray()
} else {
newObject[key] = entry[key]
}
})
return newObject
})
}
/**
* Execute the query and return the results as an Array of the generic type provided
*/
async execute_cast<T>(): Promise<T[]> {
return await this.execute() as T[]
}
}

71
node/src/test/test.ts Normal file
View File

@@ -0,0 +1,71 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import { describe } from 'mocha'
import { assert } from 'chai'
import * as lancedb from '../index'
describe('LanceDB client', function () {
describe('open a connection to lancedb', function () {
const con = lancedb.connect('.../../sample-lancedb')
it('should have a valid url', function () {
assert.equal(con.uri, '.../../sample-lancedb')
})
it('should return the existing table names', function () {
assert.deepEqual(con.tableNames(), ['my_table'])
})
describe('open a table from a connection', function () {
const tablePromise = con.openTable('my_table')
it('should have a valid name', async function () {
const table = await tablePromise
assert.equal(table.name, 'my_table')
})
class MyResult {
vector: Float32Array = new Float32Array(0)
price: number = 0
item: string = ''
}
it('execute a query', async function () {
const table = await tablePromise
const builder = table.search([0.1, 0.3])
const results = await builder.execute() as MyResult[]
assert.equal(results.length, 2)
assert.equal(results[0].item, 'foo')
assert.equal(results[0].price, 10)
assert.approximately(results[0].vector[0], 3.1, 0.1)
assert.approximately(results[0].vector[1], 4.1, 0.1)
})
it('execute a query and type cast the result', async function () {
const table = await tablePromise
const builder = table.search([0.1, 0.3])
const results = await builder.execute_cast<MyResult>()
assert.equal(results.length, 2)
assert.equal(results[0].item, 'foo')
assert.equal(results[0].price, 10)
assert.approximately(results[0].vector[0], 3.1, 0.1)
assert.approximately(results[0].vector[1], 4.1, 0.1)
})
})
})
})

10
node/tsconfig.json Normal file
View File

@@ -0,0 +1,10 @@
{
"include": ["src/**/*.ts"],
"compilerOptions": {
"target": "es2016",
"module": "commonjs",
"declaration": true,
"outDir": "./dist",
"strict": true
}
}

19
rust/ffi/node/Cargo.toml Normal file
View File

@@ -0,0 +1,19 @@
[package]
name = "vectordb-node"
version = "0.1.0"
description = "Serverless, low-latency vector database for AI applications"
license = "Apache-2.0"
edition = "2018"
exclude = ["index.node"]
[lib]
crate-type = ["cdylib"]
[dependencies]
arrow-array = "37.0"
arrow-ipc = "37.0"
once_cell = "1"
futures = "0.3"
vectordb = { path = "../../vectordb" }
tokio = { version = "1.23", features = ["rt-multi-thread"] }
neon = {version = "0.10.1", default-features = false, features = ["channel-api", "napi-6", "promise-api", "task-api"] }

3
rust/ffi/node/README.md Normal file
View File

@@ -0,0 +1,3 @@
The LanceDB node bridge (vectordb-node) allows javascript applications to access LanceDB datasets.
It is build using [Neon](https://neon-bindings.com). See the node project for an example of how it is used / tests

View File

@@ -0,0 +1,36 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use neon::prelude::*;
pub(crate) fn vec_str_to_array<'a, C: Context<'a>>(
vec: &Vec<String>,
cx: &mut C,
) -> JsResult<'a, JsArray> {
let a = JsArray::new(cx, vec.len() as u32);
for (i, s) in vec.iter().enumerate() {
let v = cx.string(s);
a.set(cx, i as u32, v)?;
}
Ok(a)
}
pub(crate) fn js_array_to_vec(array: &JsArray, cx: &mut FunctionContext) -> Vec<f32> {
let mut query_vec: Vec<f32> = Vec::new();
for i in 0..array.len(cx) {
let entry: Handle<JsNumber> = array.get(cx, i).unwrap();
query_vec.push(entry.value(cx) as f32);
}
query_vec
}

145
rust/ffi/node/src/lib.rs Normal file
View File

@@ -0,0 +1,145 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
mod convert;
use std::ops::Deref;
use std::sync::Arc;
use arrow_array::Float32Array;
use arrow_ipc::writer::FileWriter;
use futures::TryStreamExt;
use neon::prelude::*;
use once_cell::sync::OnceCell;
use tokio::runtime::Runtime;
use vectordb::database::Database;
use vectordb::table::Table;
struct JsDatabase {
database: Arc<Database>,
}
struct JsTable {
table: Arc<Table>,
}
impl Finalize for JsDatabase {}
impl Finalize for JsTable {}
fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
static RUNTIME: OnceCell<Runtime> = OnceCell::new();
RUNTIME.get_or_try_init(|| Runtime::new().or_else(|err| cx.throw_error(err.to_string())))
}
fn database_new(mut cx: FunctionContext) -> JsResult<JsBox<JsDatabase>> {
let path = cx.argument::<JsString>(0)?.value(&mut cx);
let db = JsDatabase {
database: Arc::new(Database::connect(path).or_else(|err| cx.throw_error(err.to_string()))?),
};
Ok(cx.boxed(db))
}
fn database_table_names(mut cx: FunctionContext) -> JsResult<JsArray> {
let db = cx
.this()
.downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
let tables = db
.database
.table_names()
.or_else(|err| cx.throw_error(err.to_string()))?;
convert::vec_str_to_array(&tables, &mut cx)
}
fn database_open_table(mut cx: FunctionContext) -> JsResult<JsPromise> {
let db = cx
.this()
.downcast_or_throw::<JsBox<JsDatabase>, _>(&mut cx)?;
let table_name = cx.argument::<JsString>(0)?.value(&mut cx);
let rt = runtime(&mut cx)?;
let channel = cx.channel();
let database = db.database.clone();
let (deferred, promise) = cx.promise();
rt.spawn(async move {
let table_rst = database.open_table(table_name).await;
deferred.settle_with(&channel, move |mut cx| {
let table = Arc::new(table_rst.or_else(|err| cx.throw_error(err.to_string()))?);
Ok(cx.boxed(JsTable { table }))
});
});
Ok(promise)
}
fn table_search(mut cx: FunctionContext) -> JsResult<JsPromise> {
let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
let query_vector = cx.argument::<JsArray>(0)?; //. .as_value(&mut cx);
let limit = cx.argument::<JsNumber>(1)?.value(&mut cx);
let rt = runtime(&mut cx)?;
let channel = cx.channel();
let (deferred, promise) = cx.promise();
let table = js_table.table.clone();
let query = convert::js_array_to_vec(query_vector.deref(), &mut cx);
rt.spawn(async move {
let builder = table
.search(Float32Array::from(query))
.limit(limit as usize);
let results = builder
.execute()
.await
.unwrap() // FIXME unwrap
.try_collect::<Vec<_>>()
.await;
deferred.settle_with(&channel, move |mut cx| {
let results = results.or_else(|err| cx.throw_error(err.to_string()))?;
let vector: Vec<u8> = Vec::new();
if results.is_empty() {
return cx.buffer(0);
}
let schema = results.get(0).unwrap().schema();
let mut fr = FileWriter::try_new(vector, schema.deref())
.or_else(|err| cx.throw_error(err.to_string()))?;
for batch in results.iter() {
fr.write(batch)
.or_else(|err| cx.throw_error(err.to_string()))?;
}
fr.finish().or_else(|err| cx.throw_error(err.to_string()))?;
let buf = fr
.into_inner()
.or_else(|err| cx.throw_error(err.to_string()))?;
Ok(JsBuffer::external(&mut cx, buf))
});
});
Ok(promise)
}
#[neon::main]
fn main(mut cx: ModuleContext) -> NeonResult<()> {
cx.export_function("databaseNew", database_new)?;
cx.export_function("databaseTableNames", database_table_names)?;
cx.export_function("databaseOpenTable", database_open_table)?;
cx.export_function("tableSearch", table_search)?;
Ok(())
}

View File

@@ -1,14 +0,0 @@
pub fn add(left: usize, right: usize) -> usize {
left + right
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn it_works() {
let result = add(2, 2);
assert_eq!(result, 4);
}
}

View File

@@ -9,4 +9,10 @@ repository = "https://github.com/lancedb/lancedb"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
arrow-array = "37.0"
arrow-schema = "37.0"
lance = "0.4.3"
tokio = { version = "1.23", features = ["rt-multi-thread"] }
[dev-dependencies]
tempfile = "3.5.0"

View File

@@ -0,0 +1,118 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fs::create_dir_all;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::error::Result;
use crate::table::Table;
pub struct Database {
pub(crate) path: Arc<PathBuf>,
}
const LANCE_EXTENSION: &str = "lance";
/// A connection to LanceDB
impl Database {
/// Connects to LanceDB
///
/// # Arguments
///
/// * `path` - URI where the database is located, can be a local file or a supported remote cloud storage
///
/// # Returns
///
/// * A [Database] object.
pub fn connect<P: AsRef<Path>>(path: P) -> Result<Database> {
if !path.as_ref().try_exists()? {
create_dir_all(&path)?;
}
Ok(Database {
path: Arc::new(path.as_ref().to_path_buf()),
})
}
/// Get the names of all tables in the database.
///
/// # Returns
///
/// * A [Vec<String>] with all table names.
pub fn table_names(&self) -> Result<Vec<String>> {
let f = self
.path
.read_dir()?
.flatten()
.map(|dir_entry| dir_entry.path())
.filter(|path| {
let is_lance = path
.extension()
.map(|e| e.to_str().map(|e| e == LANCE_EXTENSION))
.flatten();
is_lance.unwrap_or(false)
})
.map(|p| {
p.file_stem()
.map(|s| s.to_str().map(|s| String::from(s)))
.flatten()
})
.flatten()
.collect();
Ok(f)
}
/// Open a table in the database.
///
/// # Arguments
/// * `name` - The name of the table.
///
/// # Returns
///
/// * A [Table] object.
pub async fn open_table(&self, name: String) -> Result<Table> {
Table::new(self.path.clone(), name).await
}
}
#[cfg(test)]
mod tests {
use std::fs::create_dir_all;
use tempfile::tempdir;
use crate::database::Database;
#[tokio::test]
async fn test_connect() {
let tmp_dir = tempdir().unwrap();
let path_buf = tmp_dir.into_path();
let db = Database::connect(&path_buf);
assert_eq!(db.unwrap().path.as_path(), path_buf.as_path())
}
#[tokio::test]
async fn test_table_names() {
let tmp_dir = tempdir().unwrap();
create_dir_all(tmp_dir.path().join("table1.lance")).unwrap();
create_dir_all(tmp_dir.path().join("table2.lance")).unwrap();
create_dir_all(tmp_dir.path().join("invalidlance")).unwrap();
let db = Database::connect(&tmp_dir.into_path()).unwrap();
let tables = db.table_names().unwrap();
assert_eq!(tables.len(), 2);
assert!(tables.contains(&String::from("table1")));
assert!(tables.contains(&String::from("table2")));
}
}

View File

@@ -0,0 +1,43 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[derive(Debug)]
pub enum Error {
IO(String),
Lance(String),
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let (catalog, message) = match self {
Self::IO(s) => ("I/O", s.as_str()),
Self::Lance(s) => ("Lance", s.as_str()),
};
write!(f, "LanceDBError({catalog}): {message}")
}
}
pub type Result<T> = std::result::Result<T, Error>;
impl From<std::io::Error> for Error {
fn from(e: std::io::Error) -> Self {
Self::IO(e.to_string())
}
}
impl From<lance::Error> for Error {
fn from(e: lance::Error) -> Self {
Self::Lance(e.to_string())
}
}

18
rust/vectordb/src/lib.rs Normal file
View File

@@ -0,0 +1,18 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod database;
pub mod error;
pub mod query;
pub mod table;

210
rust/vectordb/src/query.rs Normal file
View File

@@ -0,0 +1,210 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use arrow_array::Float32Array;
use lance::dataset::scanner::{DatasetRecordBatchStream, Scanner};
use lance::dataset::Dataset;
use lance::index::vector::MetricType;
use crate::error::Result;
/// A builder for nearest neighbor queries for LanceDB.
pub struct Query {
pub dataset: Arc<Dataset>,
pub query_vector: Float32Array,
pub limit: usize,
pub nprobes: usize,
pub refine_factor: Option<u32>,
pub metric_type: MetricType,
pub use_index: bool,
}
impl Query {
/// Creates a new Query object
///
/// # Arguments
///
/// * `dataset` - The table / dataset the query will be run against.
/// * `vector` The vector used for this query.
///
/// # Returns
///
/// * A [Query] object.
pub(crate) fn new(dataset: Arc<Dataset>, vector: Float32Array) -> Self {
Query {
dataset,
query_vector: vector,
limit: 10,
nprobes: 20,
refine_factor: None,
metric_type: MetricType::L2,
use_index: false,
}
}
/// Execute the queries and return its results.
///
/// # Returns
///
/// * A [DatasetRecordBatchStream] with the query's results.
pub async fn execute(&self) -> Result<DatasetRecordBatchStream> {
let mut scanner: Scanner = self.dataset.scan();
scanner.nearest(
crate::table::VECTOR_COLUMN_NAME,
&self.query_vector,
self.limit,
)?;
scanner.nprobs(self.nprobes);
scanner.distance_metric(self.metric_type);
scanner.use_index(self.use_index);
self.refine_factor.map(|rf| scanner.refine(rf));
Ok(scanner.try_into_stream().await?)
}
/// Set the maximum number of results to return.
///
/// # Arguments
///
/// * `limit` - The maximum number of results to return.
pub fn limit(mut self, limit: usize) -> Query {
self.limit = limit;
self
}
/// Set the vector used for this query.
///
/// # Arguments
///
/// * `vector` - The vector that will be used for search.
pub fn query_vector(mut self, query_vector: Float32Array) -> Query {
self.query_vector = query_vector;
self
}
/// Set the number of probes to use.
///
/// # Arguments
///
/// * `nprobes` - The number of probes to use.
pub fn nprobes(mut self, nprobes: usize) -> Query {
self.nprobes = nprobes;
self
}
/// Set the refine factor to use.
///
/// # Arguments
///
/// * `refine_factor` - The refine factor to use.
pub fn refine_factor(mut self, refine_factor: Option<u32>) -> Query {
self.refine_factor = refine_factor;
self
}
/// Set the distance metric to use.
///
/// # Arguments
///
/// * `metric_type` - The distance metric to use. By default [MetricType::L2] is used.
pub fn metric_type(mut self, metric_type: MetricType) -> Query {
self.metric_type = metric_type;
self
}
/// Whether to use an ANN index if available
///
/// # Arguments
///
/// * `use_index` - Sets Whether to use an ANN index if available
pub fn use_index(mut self, use_index: bool) -> Query {
self.use_index = use_index;
self
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow_array::{Float32Array, RecordBatch, RecordBatchReader};
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
use lance::arrow::RecordBatchBuffer;
use lance::dataset::Dataset;
use lance::index::vector::MetricType;
use crate::query::Query;
#[tokio::test]
async fn test_setters_getters() {
let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
let ds = Dataset::write(&mut batches, ":memory:", None)
.await
.unwrap();
let vector = Float32Array::from_iter_values([0.1, 0.2]);
let query = Query::new(Arc::new(ds), vector.clone());
assert_eq!(query.query_vector, vector);
let new_vector = Float32Array::from_iter_values([9.8, 8.7]);
let query = query
.query_vector(new_vector.clone())
.limit(100)
.nprobes(1000)
.use_index(true)
.metric_type(MetricType::Cosine)
.refine_factor(Some(999));
assert_eq!(query.query_vector, new_vector);
assert_eq!(query.limit, 100);
assert_eq!(query.nprobes, 1000);
assert_eq!(query.use_index, true);
assert_eq!(query.metric_type, MetricType::Cosine);
assert_eq!(query.refine_factor, Some(999));
}
#[tokio::test]
async fn test_execute() {
let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
let ds = Dataset::write(&mut batches, ":memory:", None)
.await
.unwrap();
let vector = Float32Array::from_iter_values([0.1; 128]);
let query = Query::new(Arc::new(ds), vector.clone());
let result = query.execute().await;
assert_eq!(result.is_ok(), true);
}
fn make_test_batches() -> RecordBatchBuffer {
let dim: usize = 128;
let schema = Arc::new(ArrowSchema::new(vec![
ArrowField::new("key", DataType::Int32, false),
ArrowField::new(
"vector",
DataType::FixedSizeList(
Arc::new(ArrowField::new("item", DataType::Float32, true)),
dim as i32,
),
true,
),
ArrowField::new("uri", DataType::Utf8, true),
]));
RecordBatchBuffer::new(vec![RecordBatch::new_empty(schema.clone())])
}
}

144
rust/vectordb/src/table.rs Normal file
View File

@@ -0,0 +1,144 @@
// Copyright 2023 Lance Developers.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::path::PathBuf;
use std::sync::Arc;
use arrow_array::Float32Array;
use lance::dataset::Dataset;
use crate::error::{Error, Result};
use crate::query::Query;
pub const VECTOR_COLUMN_NAME: &str = "vector";
pub const LANCE_FILE_EXTENSION: &str = "lance";
/// A table in a LanceDB database.
pub struct Table {
name: String,
dataset: Arc<Dataset>,
}
impl Table {
/// Creates a new Table object
///
/// # Arguments
///
/// * `base_path` - The base path where the table is located
/// * `name` The Table name
///
/// # Returns
///
/// * A [Table] object.
pub async fn new(base_path: Arc<PathBuf>, name: String) -> Result<Self> {
let ds_path = base_path.join(format!("{}.{}", name, LANCE_FILE_EXTENSION));
let ds_uri = ds_path
.to_str()
.ok_or(Error::IO(format!("Unable to find table {}", name)))?;
let dataset = Dataset::open(ds_uri).await?;
let table = Table {
name,
dataset: Arc::new(dataset),
};
Ok(table)
}
/// Creates a new Query object that can be executed.
///
/// # Arguments
///
/// * `vector` The vector used for this query.
///
/// # Returns
///
/// * A [Query] object.
pub fn search(&self, query_vector: Float32Array) -> Query {
Query::new(self.dataset.clone(), query_vector)
}
}
#[cfg(test)]
mod tests {
use arrow_array::{Float32Array, Int32Array, RecordBatch, RecordBatchReader};
use arrow_schema::{DataType, Field, Schema};
use lance::arrow::RecordBatchBuffer;
use lance::dataset::Dataset;
use std::sync::Arc;
use tempfile::tempdir;
use crate::table::Table;
#[tokio::test]
async fn test_new_table_not_exists() {
let tmp_dir = tempdir().unwrap();
let path_buf = tmp_dir.into_path();
let table = Table::new(Arc::new(path_buf), "test".to_string()).await;
assert!(table.is_err());
}
#[tokio::test]
async fn test_new() {
let tmp_dir = tempdir().unwrap();
let path_buf = tmp_dir.into_path();
let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
Dataset::write(
&mut batches,
path_buf.join("test.lance").to_str().unwrap(),
None,
)
.await
.unwrap();
let table = Table::new(Arc::new(path_buf), "test".to_string())
.await
.unwrap();
assert_eq!(table.name, "test")
}
#[tokio::test]
async fn test_search() {
let tmp_dir = tempdir().unwrap();
let path_buf = tmp_dir.into_path();
let mut batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
Dataset::write(
&mut batches,
path_buf.join("test.lance").to_str().unwrap(),
None,
)
.await
.unwrap();
let table = Table::new(Arc::new(path_buf), "test".to_string())
.await
.unwrap();
let vector = Float32Array::from_iter_values([0.1, 0.2]);
let query = table.search(vector.clone());
assert_eq!(vector, query.query_vector);
}
fn make_test_batches() -> RecordBatchBuffer {
let schema = Arc::new(Schema::new(vec![Field::new("i", DataType::Int32, false)]));
RecordBatchBuffer::new(vec![RecordBatch::try_new(
schema.clone(),
vec![Arc::new(Int32Array::from_iter_values(0..20))],
)
.unwrap()])
}
}