mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-23 21:39:57 +00:00
Compare commits
12 Commits
v0.1.4-pyt
...
v0.1.5-pyt
| Author | SHA1 | Date | |
|---|---|---|---|
|
fbd0bc7740 | ||
|
f765a453cf | ||
|
|
45b3a14f26 | ||
|
9965b4564d | ||
|
|
0719e4b3fb | ||
|
091fb9b665 | ||
|
03013a4434 | ||
|
|
3e14b357e7 | ||
|
|
99cbda8b07 | ||
|
e50b642d80 | ||
|
|
6d8cf52e01 | ||
|
53f3882d6e |
1
Cargo.lock
generated
1
Cargo.lock
generated
@@ -3366,6 +3366,7 @@ dependencies = [
|
||||
"lance",
|
||||
"object_store",
|
||||
"rand",
|
||||
"snafu",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
@@ -38,7 +38,7 @@ result = table.search([100, 100]).limit(2).to_df()
|
||||
## Complete Demos
|
||||
|
||||
We will be adding completed demo apps built using LanceDB.
|
||||
- [YouTube Transcript Search](../notebooks/youtube_transcript_search.ipynb)
|
||||
- [YouTube Transcript Search](../../notebooks/youtube_transcript_search.ipynb)
|
||||
|
||||
|
||||
## Documentation Quick Links
|
||||
|
||||
41
node/examples/js-openai/index.js
Normal file
41
node/examples/js-openai/index.js
Normal file
@@ -0,0 +1,41 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
'use strict'
|
||||
|
||||
async function example () {
|
||||
const lancedb = require('vectordb')
|
||||
// You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
|
||||
const apiKey = process.env.OPENAI_API_KEY
|
||||
// The embedding function will create embeddings for the 'text' column(text in this case)
|
||||
const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
|
||||
|
||||
const db = await lancedb.connect('data/sample-lancedb')
|
||||
|
||||
const data = [
|
||||
{ id: 1, text: 'Black T-Shirt', price: 10 },
|
||||
{ id: 2, text: 'Leather Jacket', price: 50 }
|
||||
]
|
||||
|
||||
const table = await db.createTable('vectors', data, embedding)
|
||||
console.log(await db.tableNames())
|
||||
|
||||
const results = await table
|
||||
.search('keeps me warm')
|
||||
.limit(1)
|
||||
.execute()
|
||||
console.log(results[0].text)
|
||||
}
|
||||
|
||||
example().then(_ => { console.log('All done!') })
|
||||
15
node/examples/js-openai/package.json
Normal file
15
node/examples/js-openai/package.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "vectordb-example-js-openai",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "Lance Devs",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"vectordb": "file:../..",
|
||||
"openai": "^3.2.1"
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,6 @@
|
||||
"author": "Lance Devs",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"vectordb": "^0.1.0"
|
||||
"vectordb": "file:../.."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,6 @@
|
||||
"typescript": "*"
|
||||
},
|
||||
"dependencies": {
|
||||
"vectordb": "^0.1.0"
|
||||
"vectordb": "file:../.."
|
||||
}
|
||||
}
|
||||
|
||||
470
node/package-lock.json
generated
470
node/package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "vectordb",
|
||||
"version": "0.1.1",
|
||||
"version": "0.1.3",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "vectordb",
|
||||
"version": "0.1.1",
|
||||
"version": "0.1.3",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@apache-arrow/ts": "^12.0.0",
|
||||
@@ -16,6 +16,7 @@
|
||||
"@types/chai": "^4.3.4",
|
||||
"@types/mocha": "^10.0.1",
|
||||
"@types/node": "^18.16.2",
|
||||
"@types/sinon": "^10.0.15",
|
||||
"@types/temp": "^0.9.1",
|
||||
"@typescript-eslint/eslint-plugin": "^5.59.1",
|
||||
"cargo-cp-artifact": "^0.1",
|
||||
@@ -26,6 +27,8 @@
|
||||
"eslint-plugin-n": "^15.7.0",
|
||||
"eslint-plugin-promise": "^6.1.1",
|
||||
"mocha": "^10.2.0",
|
||||
"openai": "^3.2.1",
|
||||
"sinon": "^15.1.0",
|
||||
"temp": "^0.9.4",
|
||||
"ts-node": "^10.9.1",
|
||||
"ts-node-dev": "^2.0.0",
|
||||
@@ -232,6 +235,50 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@sinonjs/commons": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz",
|
||||
"integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"type-detect": "4.0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@sinonjs/fake-timers": {
|
||||
"version": "10.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.2.0.tgz",
|
||||
"integrity": "sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@sinonjs/commons": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@sinonjs/samsam": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz",
|
||||
"integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@sinonjs/commons": "^2.0.0",
|
||||
"lodash.get": "^4.4.2",
|
||||
"type-detect": "^4.0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@sinonjs/samsam/node_modules/@sinonjs/commons": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz",
|
||||
"integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"type-detect": "4.0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@sinonjs/text-encoding": {
|
||||
"version": "0.7.2",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz",
|
||||
"integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@tsconfig/node10": {
|
||||
"version": "1.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz",
|
||||
@@ -307,6 +354,21 @@
|
||||
"integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/sinon": {
|
||||
"version": "10.0.15",
|
||||
"resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.15.tgz",
|
||||
"integrity": "sha512-3lrFNQG0Kr2LDzvjyjB6AMJk4ge+8iYhQfdnSwIwlG88FUOV43kPcQqDZkDa/h3WSZy6i8Fr0BSjfQtB1B3xuQ==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/sinonjs__fake-timers": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/sinonjs__fake-timers": {
|
||||
"version": "8.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.2.tgz",
|
||||
"integrity": "sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/strip-bom": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz",
|
||||
@@ -744,6 +806,12 @@
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/asynckit": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/available-typed-arrays": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
|
||||
@@ -756,6 +824,15 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "0.26.1",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
|
||||
"integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.14.8"
|
||||
}
|
||||
},
|
||||
"node_modules/balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
@@ -968,6 +1045,18 @@
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/combined-stream": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-args": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
|
||||
@@ -1179,6 +1268,15 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/diff": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
|
||||
@@ -1942,6 +2040,26 @@
|
||||
"integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/follow-redirects": {
|
||||
"version": "1.15.2",
|
||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
|
||||
"integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "individual",
|
||||
"url": "https://github.com/sponsors/RubenVerborgh"
|
||||
}
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=4.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"debug": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/for-each": {
|
||||
"version": "0.3.3",
|
||||
"resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
|
||||
@@ -1951,6 +2069,20 @@
|
||||
"is-callable": "^1.1.3"
|
||||
}
|
||||
},
|
||||
"node_modules/form-data": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
|
||||
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
@@ -2584,6 +2716,12 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/isarray": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
|
||||
"integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
@@ -2644,6 +2782,12 @@
|
||||
"json5": "lib/cli.js"
|
||||
}
|
||||
},
|
||||
"node_modules/just-extend": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz",
|
||||
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/levn": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
|
||||
@@ -2677,6 +2821,12 @@
|
||||
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
|
||||
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
|
||||
},
|
||||
"node_modules/lodash.get": {
|
||||
"version": "4.4.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
|
||||
"integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/lodash.merge": {
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
|
||||
@@ -2748,6 +2898,27 @@
|
||||
"node": ">=8.6"
|
||||
}
|
||||
},
|
||||
"node_modules/mime-db": {
|
||||
"version": "1.52.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
|
||||
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/mime-types": {
|
||||
"version": "2.1.35",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
|
||||
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"mime-db": "1.52.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
@@ -2925,6 +3096,28 @@
|
||||
"integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/nise": {
|
||||
"version": "5.1.4",
|
||||
"resolved": "https://registry.npmjs.org/nise/-/nise-5.1.4.tgz",
|
||||
"integrity": "sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@sinonjs/commons": "^2.0.0",
|
||||
"@sinonjs/fake-timers": "^10.0.2",
|
||||
"@sinonjs/text-encoding": "^0.7.1",
|
||||
"just-extend": "^4.0.2",
|
||||
"path-to-regexp": "^1.7.0"
|
||||
}
|
||||
},
|
||||
"node_modules/nise/node_modules/@sinonjs/commons": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz",
|
||||
"integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"type-detect": "4.0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/normalize-path": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
|
||||
@@ -2996,6 +3189,16 @@
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/openai": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/openai/-/openai-3.2.1.tgz",
|
||||
"integrity": "sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"axios": "^0.26.0",
|
||||
"form-data": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/optionator": {
|
||||
"version": "0.9.1",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
|
||||
@@ -3099,6 +3302,15 @@
|
||||
"integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/path-to-regexp": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz",
|
||||
"integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"isarray": "0.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/path-type": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
|
||||
@@ -3406,6 +3618,45 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/sinon": {
|
||||
"version": "15.1.0",
|
||||
"resolved": "https://registry.npmjs.org/sinon/-/sinon-15.1.0.tgz",
|
||||
"integrity": "sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@sinonjs/commons": "^3.0.0",
|
||||
"@sinonjs/fake-timers": "^10.2.0",
|
||||
"@sinonjs/samsam": "^8.0.0",
|
||||
"diff": "^5.1.0",
|
||||
"nise": "^5.1.4",
|
||||
"supports-color": "^7.2.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/sinon"
|
||||
}
|
||||
},
|
||||
"node_modules/sinon/node_modules/diff": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz",
|
||||
"integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=0.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/sinon/node_modules/supports-color": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"has-flag": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/slash": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
|
||||
@@ -4217,6 +4468,52 @@
|
||||
"fastq": "^1.6.0"
|
||||
}
|
||||
},
|
||||
"@sinonjs/commons": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.0.tgz",
|
||||
"integrity": "sha512-jXBtWAF4vmdNmZgD5FoKsVLv3rPgDnLgPbU84LIJ3otV44vJlDRokVng5v8NFJdCf/da9legHcKaRuZs4L7faA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"type-detect": "4.0.8"
|
||||
}
|
||||
},
|
||||
"@sinonjs/fake-timers": {
|
||||
"version": "10.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.2.0.tgz",
|
||||
"integrity": "sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"@sinonjs/samsam": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz",
|
||||
"integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^2.0.0",
|
||||
"lodash.get": "^4.4.2",
|
||||
"type-detect": "^4.0.8"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sinonjs/commons": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz",
|
||||
"integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"type-detect": "4.0.8"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"@sinonjs/text-encoding": {
|
||||
"version": "0.7.2",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz",
|
||||
"integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==",
|
||||
"dev": true
|
||||
},
|
||||
"@tsconfig/node10": {
|
||||
"version": "1.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz",
|
||||
@@ -4292,6 +4589,21 @@
|
||||
"integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/sinon": {
|
||||
"version": "10.0.15",
|
||||
"resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.15.tgz",
|
||||
"integrity": "sha512-3lrFNQG0Kr2LDzvjyjB6AMJk4ge+8iYhQfdnSwIwlG88FUOV43kPcQqDZkDa/h3WSZy6i8Fr0BSjfQtB1B3xuQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/sinonjs__fake-timers": "*"
|
||||
}
|
||||
},
|
||||
"@types/sinonjs__fake-timers": {
|
||||
"version": "8.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.2.tgz",
|
||||
"integrity": "sha512-9GcLXF0/v3t80caGs5p2rRfkB+a8VBGLJZVih6CNFkx8IZ994wiKKLSRs9nuFwk1HevWs/1mnUmkApGrSGsShA==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/strip-bom": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz",
|
||||
@@ -4579,12 +4891,27 @@
|
||||
"integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
|
||||
"dev": true
|
||||
},
|
||||
"asynckit": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
|
||||
"dev": true
|
||||
},
|
||||
"available-typed-arrays": {
|
||||
"version": "1.0.5",
|
||||
"resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz",
|
||||
"integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==",
|
||||
"dev": true
|
||||
},
|
||||
"axios": {
|
||||
"version": "0.26.1",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
|
||||
"integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"follow-redirects": "^1.14.8"
|
||||
}
|
||||
},
|
||||
"balanced-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
|
||||
@@ -4749,6 +5076,15 @@
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"dev": true
|
||||
},
|
||||
"combined-stream": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"command-line-args": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz",
|
||||
@@ -4908,6 +5244,12 @@
|
||||
"object-keys": "^1.1.1"
|
||||
}
|
||||
},
|
||||
"delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
|
||||
"dev": true
|
||||
},
|
||||
"diff": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
|
||||
@@ -5475,6 +5817,12 @@
|
||||
"integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
|
||||
"dev": true
|
||||
},
|
||||
"follow-redirects": {
|
||||
"version": "1.15.2",
|
||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
|
||||
"integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
|
||||
"dev": true
|
||||
},
|
||||
"for-each": {
|
||||
"version": "0.3.3",
|
||||
"resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
|
||||
@@ -5484,6 +5832,17 @@
|
||||
"is-callable": "^1.1.3"
|
||||
}
|
||||
},
|
||||
"form-data": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
|
||||
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
"fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
@@ -5912,6 +6271,12 @@
|
||||
"call-bind": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"isarray": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
|
||||
"integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==",
|
||||
"dev": true
|
||||
},
|
||||
"isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
@@ -5959,6 +6324,12 @@
|
||||
"minimist": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"just-extend": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz",
|
||||
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
|
||||
"dev": true
|
||||
},
|
||||
"levn": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
|
||||
@@ -5983,6 +6354,12 @@
|
||||
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
|
||||
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
|
||||
},
|
||||
"lodash.get": {
|
||||
"version": "4.4.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
|
||||
"integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==",
|
||||
"dev": true
|
||||
},
|
||||
"lodash.merge": {
|
||||
"version": "4.6.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
|
||||
@@ -6039,6 +6416,21 @@
|
||||
"picomatch": "^2.3.1"
|
||||
}
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.52.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
|
||||
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
|
||||
"dev": true
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.35",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
|
||||
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"mime-db": "1.52.0"
|
||||
}
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
|
||||
@@ -6172,6 +6564,30 @@
|
||||
"integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==",
|
||||
"dev": true
|
||||
},
|
||||
"nise": {
|
||||
"version": "5.1.4",
|
||||
"resolved": "https://registry.npmjs.org/nise/-/nise-5.1.4.tgz",
|
||||
"integrity": "sha512-8+Ib8rRJ4L0o3kfmyVCL7gzrohyDe0cMFTBa2d364yIrEGMEoetznKJx899YxjybU6bL9SQkYPSBBs1gyYs8Xg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^2.0.0",
|
||||
"@sinonjs/fake-timers": "^10.0.2",
|
||||
"@sinonjs/text-encoding": "^0.7.1",
|
||||
"just-extend": "^4.0.2",
|
||||
"path-to-regexp": "^1.7.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sinonjs/commons": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz",
|
||||
"integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"type-detect": "4.0.8"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"normalize-path": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
|
||||
@@ -6222,6 +6638,16 @@
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"openai": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/openai/-/openai-3.2.1.tgz",
|
||||
"integrity": "sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"axios": "^0.26.0",
|
||||
"form-data": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"optionator": {
|
||||
"version": "0.9.1",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
|
||||
@@ -6295,6 +6721,15 @@
|
||||
"integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==",
|
||||
"dev": true
|
||||
},
|
||||
"path-to-regexp": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz",
|
||||
"integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"isarray": "0.0.1"
|
||||
}
|
||||
},
|
||||
"path-type": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
|
||||
@@ -6484,6 +6919,37 @@
|
||||
"object-inspect": "^1.9.0"
|
||||
}
|
||||
},
|
||||
"sinon": {
|
||||
"version": "15.1.0",
|
||||
"resolved": "https://registry.npmjs.org/sinon/-/sinon-15.1.0.tgz",
|
||||
"integrity": "sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^3.0.0",
|
||||
"@sinonjs/fake-timers": "^10.2.0",
|
||||
"@sinonjs/samsam": "^8.0.0",
|
||||
"diff": "^5.1.0",
|
||||
"nise": "^5.1.4",
|
||||
"supports-color": "^7.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"diff": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz",
|
||||
"integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==",
|
||||
"dev": true
|
||||
},
|
||||
"supports-color": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"has-flag": "^4.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"slash": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "vectordb",
|
||||
"version": "0.1.1",
|
||||
"version": "0.1.3",
|
||||
"description": " Serverless, low-latency vector database for AI applications",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
@@ -27,6 +27,7 @@
|
||||
"@types/chai": "^4.3.4",
|
||||
"@types/mocha": "^10.0.1",
|
||||
"@types/node": "^18.16.2",
|
||||
"@types/sinon": "^10.0.15",
|
||||
"@types/temp": "^0.9.1",
|
||||
"@typescript-eslint/eslint-plugin": "^5.59.1",
|
||||
"cargo-cp-artifact": "^0.1",
|
||||
@@ -37,6 +38,8 @@
|
||||
"eslint-plugin-n": "^15.7.0",
|
||||
"eslint-plugin-promise": "^6.1.1",
|
||||
"mocha": "^10.2.0",
|
||||
"sinon": "^15.1.0",
|
||||
"openai": "^3.2.1",
|
||||
"temp": "^0.9.4",
|
||||
"ts-node": "^10.9.1",
|
||||
"ts-node-dev": "^2.0.0",
|
||||
|
||||
@@ -24,7 +24,7 @@ import {
|
||||
} from 'apache-arrow'
|
||||
import { type EmbeddingFunction } from './index'
|
||||
|
||||
export function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Table {
|
||||
export async function convertToTable<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Table> {
|
||||
if (data.length === 0) {
|
||||
throw new Error('At least one record needs to be provided')
|
||||
}
|
||||
@@ -51,7 +51,7 @@ export function convertToTable<T> (data: Array<Record<string, unknown>>, embeddi
|
||||
}
|
||||
|
||||
if (columnsKey === embeddings?.sourceColumn) {
|
||||
const vectors = embeddings.embed(values as T[])
|
||||
const vectors = await embeddings.embed(values as T[])
|
||||
const listBuilder = newVectorListBuilder()
|
||||
vectors.map(v => listBuilder.append(v))
|
||||
records.vector = listBuilder.finish().toVector()
|
||||
@@ -79,7 +79,7 @@ function newVectorListBuilder (): ListBuilder<Float32, any> {
|
||||
}
|
||||
|
||||
export async function fromRecordsToBuffer<T> (data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>): Promise<Buffer> {
|
||||
const table = convertToTable(data, embeddings)
|
||||
const table = await convertToTable(data, embeddings)
|
||||
const writer = RecordBatchFileWriter.writeAll(table)
|
||||
return Buffer.from(await writer.toUint8Array())
|
||||
}
|
||||
|
||||
28
node/src/embedding/embedding_function.ts
Normal file
28
node/src/embedding/embedding_function.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/**
|
||||
* An embedding function that automatically creates vector representation for a given column.
|
||||
*/
|
||||
export interface EmbeddingFunction<T> {
|
||||
/**
|
||||
* The name of the column that will be used as input for the Embedding Function.
|
||||
*/
|
||||
sourceColumn: string
|
||||
|
||||
/**
|
||||
* Creates a vector representation for the given values.
|
||||
*/
|
||||
embed: (data: T[]) => Promise<number[][]>
|
||||
}
|
||||
51
node/src/embedding/openai.ts
Normal file
51
node/src/embedding/openai.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import { type EmbeddingFunction } from '../index'
|
||||
|
||||
export class OpenAIEmbeddingFunction implements EmbeddingFunction<string> {
|
||||
private readonly _openai: any
|
||||
private readonly _modelName: string
|
||||
|
||||
constructor (sourceColumn: string, openAIKey: string, modelName: string = 'text-embedding-ada-002') {
|
||||
let openai
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
openai = require('openai')
|
||||
} catch {
|
||||
throw new Error('please install openai using npm install openai')
|
||||
}
|
||||
|
||||
this.sourceColumn = sourceColumn
|
||||
const configuration = new openai.Configuration({
|
||||
apiKey: openAIKey
|
||||
})
|
||||
this._openai = new openai.OpenAIApi(configuration)
|
||||
this._modelName = modelName
|
||||
}
|
||||
|
||||
async embed (data: string[]): Promise<number[][]> {
|
||||
const response = await this._openai.createEmbedding({
|
||||
model: this._modelName,
|
||||
input: data
|
||||
})
|
||||
const embeddings: number[][] = []
|
||||
for (let i = 0; i < response.data.data.length; i++) {
|
||||
embeddings.push(response.data.data[i].embedding as number[])
|
||||
}
|
||||
return embeddings
|
||||
}
|
||||
|
||||
sourceColumn: string
|
||||
}
|
||||
@@ -19,10 +19,14 @@ import {
|
||||
Vector
|
||||
} from 'apache-arrow'
|
||||
import { fromRecordsToBuffer } from './arrow'
|
||||
import type { EmbeddingFunction } from './embedding/embedding_function'
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { databaseNew, databaseTableNames, databaseOpenTable, tableCreate, tableSearch, tableAdd, tableCreateVectorIndex } = require('../native.js')
|
||||
|
||||
export type { EmbeddingFunction }
|
||||
export { OpenAIEmbeddingFunction } from './embedding/openai'
|
||||
|
||||
/**
|
||||
* Connect to a LanceDB instance at the given URI
|
||||
* @param uri The uri of the database.
|
||||
@@ -135,14 +139,8 @@ export class Table<T = number[]> {
|
||||
* Creates a search query to find the nearest neighbors of the given search term
|
||||
* @param query The query search term
|
||||
*/
|
||||
search (query: T): Query {
|
||||
let queryVector: number[]
|
||||
if (this._embeddings !== undefined) {
|
||||
queryVector = this._embeddings.embed([query])[0]
|
||||
} else {
|
||||
queryVector = query as number[]
|
||||
}
|
||||
return new Query(this._tbl, queryVector)
|
||||
search (query: T): Query<T> {
|
||||
return new Query(this._tbl, query, this._embeddings)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -228,32 +226,35 @@ export type VectorIndexParams = IvfPQIndexConfig
|
||||
/**
|
||||
* A builder for nearest neighbor queries for LanceDB.
|
||||
*/
|
||||
export class Query {
|
||||
export class Query<T = number[]> {
|
||||
private readonly _tbl: any
|
||||
private readonly _queryVector: number[]
|
||||
private readonly _query: T
|
||||
private _queryVector?: number[]
|
||||
private _limit: number
|
||||
private _refineFactor?: number
|
||||
private _nprobes: number
|
||||
private readonly _columns?: string[]
|
||||
private _filter?: string
|
||||
private _metricType?: MetricType
|
||||
private readonly _embeddings?: EmbeddingFunction<T>
|
||||
|
||||
constructor (tbl: any, queryVector: number[]) {
|
||||
constructor (tbl: any, query: T, embeddings?: EmbeddingFunction<T>) {
|
||||
this._tbl = tbl
|
||||
this._queryVector = queryVector
|
||||
this._query = query
|
||||
this._limit = 10
|
||||
this._nprobes = 20
|
||||
this._refineFactor = undefined
|
||||
this._columns = undefined
|
||||
this._filter = undefined
|
||||
this._metricType = undefined
|
||||
this._embeddings = embeddings
|
||||
}
|
||||
|
||||
/***
|
||||
* Sets the number of results that will be returned
|
||||
* @param value number of results
|
||||
*/
|
||||
limit (value: number): Query {
|
||||
limit (value: number): Query<T> {
|
||||
this._limit = value
|
||||
return this
|
||||
}
|
||||
@@ -262,7 +263,7 @@ export class Query {
|
||||
* Refine the results by reading extra elements and re-ranking them in memory.
|
||||
* @param value refine factor to use in this query.
|
||||
*/
|
||||
refineFactor (value: number): Query {
|
||||
refineFactor (value: number): Query<T> {
|
||||
this._refineFactor = value
|
||||
return this
|
||||
}
|
||||
@@ -271,7 +272,7 @@ export class Query {
|
||||
* The number of probes used. A higher number makes search more accurate but also slower.
|
||||
* @param value The number of probes used.
|
||||
*/
|
||||
nprobes (value: number): Query {
|
||||
nprobes (value: number): Query<T> {
|
||||
this._nprobes = value
|
||||
return this
|
||||
}
|
||||
@@ -280,7 +281,7 @@ export class Query {
|
||||
* A filter statement to be applied to this query.
|
||||
* @param value A filter in the same format used by a sql WHERE clause.
|
||||
*/
|
||||
filter (value: string): Query {
|
||||
filter (value: string): Query<T> {
|
||||
this._filter = value
|
||||
return this
|
||||
}
|
||||
@@ -289,7 +290,7 @@ export class Query {
|
||||
* The MetricType used for this Query.
|
||||
* @param value The metric to the. @see MetricType for the different options
|
||||
*/
|
||||
metricType (value: MetricType): Query {
|
||||
metricType (value: MetricType): Query<T> {
|
||||
this._metricType = value
|
||||
return this
|
||||
}
|
||||
@@ -298,6 +299,12 @@ export class Query {
|
||||
* Execute the query and return the results as an Array of Objects
|
||||
*/
|
||||
async execute<T = Record<string, unknown>> (): Promise<T[]> {
|
||||
if (this._embeddings !== undefined) {
|
||||
this._queryVector = (await this._embeddings.embed([this._query]))[0]
|
||||
} else {
|
||||
this._queryVector = this._query as number[]
|
||||
}
|
||||
|
||||
const buffer = await tableSearch.call(this._tbl, this)
|
||||
const data = tableFromIPC(buffer)
|
||||
return data.toArray().map((entry: Record<string, unknown>) => {
|
||||
@@ -319,21 +326,6 @@ export enum WriteMode {
|
||||
Append = 'append'
|
||||
}
|
||||
|
||||
/**
|
||||
* An embedding function that automatically creates vector representation for a given column.
|
||||
*/
|
||||
export interface EmbeddingFunction<T> {
|
||||
/**
|
||||
* The name of the column that will be used as input for the Embedding Function.
|
||||
*/
|
||||
sourceColumn: string
|
||||
|
||||
/**
|
||||
* Creates a vector representation for the given values.
|
||||
*/
|
||||
embed: (data: T[]) => number[][]
|
||||
}
|
||||
|
||||
/**
|
||||
* Distance metrics type.
|
||||
*/
|
||||
|
||||
50
node/src/test/embedding/openai.ts
Normal file
50
node/src/test/embedding/openai.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
// Copyright 2023 Lance Developers.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
import { describe } from 'mocha'
|
||||
import { assert } from 'chai'
|
||||
|
||||
import { OpenAIEmbeddingFunction } from '../../embedding/openai'
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { OpenAIApi } = require('openai')
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const { stub } = require('sinon')
|
||||
|
||||
describe('OpenAPIEmbeddings', function () {
|
||||
const stubValue = {
|
||||
data: {
|
||||
data: [
|
||||
{
|
||||
embedding: Array(1536).fill(1.0)
|
||||
},
|
||||
{
|
||||
embedding: Array(1536).fill(2.0)
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
describe('#embed', function () {
|
||||
it('should create vector embeddings', async function () {
|
||||
const openAIStub = stub(OpenAIApi.prototype, 'createEmbedding').returns(stubValue)
|
||||
const f = new OpenAIEmbeddingFunction('text', 'sk-key')
|
||||
const vectors = await f.embed(['abc', 'def'])
|
||||
assert.isTrue(openAIStub.calledOnce)
|
||||
assert.equal(vectors.length, 2)
|
||||
assert.deepEqual(vectors[0], stubValue.data.data[0].embedding)
|
||||
assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -154,7 +154,7 @@ describe('LanceDB client', function () {
|
||||
['bar', [3.1, 3.2]]
|
||||
])
|
||||
|
||||
embed (data: string[]): number[][] {
|
||||
async embed (data: string[]): Promise<number[][]> {
|
||||
return data.map(datum => this._embedding_map.get(datum) ?? [0.0, 0.0])
|
||||
}
|
||||
}
|
||||
|
||||
108
notebooks/diffusiondb/datagen.py
Executable file
108
notebooks/diffusiondb/datagen.py
Executable file
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2023 LanceDB Developers
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Dataset hf://poloclub/diffusiondb
|
||||
"""
|
||||
|
||||
import io
|
||||
from argparse import ArgumentParser
|
||||
from multiprocessing import Pool
|
||||
|
||||
import lance
|
||||
import lancedb
|
||||
import pyarrow as pa
|
||||
from datasets import load_dataset
|
||||
from PIL import Image
|
||||
from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
|
||||
|
||||
MODEL_ID = "openai/clip-vit-base-patch32"
|
||||
|
||||
device = "cuda"
|
||||
|
||||
tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)
|
||||
model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
|
||||
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
|
||||
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("prompt", pa.string()),
|
||||
pa.field("seed", pa.uint32()),
|
||||
pa.field("step", pa.uint16()),
|
||||
pa.field("cfg", pa.float32()),
|
||||
pa.field("sampler", pa.string()),
|
||||
pa.field("width", pa.uint16()),
|
||||
pa.field("height", pa.uint16()),
|
||||
pa.field("timestamp", pa.timestamp("s")),
|
||||
pa.field("image_nsfw", pa.float32()),
|
||||
pa.field("prompt_nsfw", pa.float32()),
|
||||
pa.field("vector", pa.list_(pa.float32(), 512)),
|
||||
pa.field("image", pa.binary()),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def pil_to_bytes(img) -> list[bytes]:
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def generate_clip_embeddings(batch) -> pa.RecordBatch:
|
||||
image = processor(text=None, images=batch["image"], return_tensors="pt")[
|
||||
"pixel_values"
|
||||
].to(device)
|
||||
img_emb = model.get_image_features(image)
|
||||
batch["vector"] = img_emb.cpu().tolist()
|
||||
|
||||
with Pool() as p:
|
||||
batch["image_bytes"] = p.map(pil_to_bytes, batch["image"])
|
||||
return batch
|
||||
|
||||
|
||||
def datagen(args):
|
||||
"""Generate DiffusionDB dataset, and use CLIP model to generate image embeddings."""
|
||||
dataset = load_dataset("poloclub/diffusiondb", args.subset)
|
||||
data = []
|
||||
for b in dataset.map(
|
||||
generate_clip_embeddings, batched=True, batch_size=256, remove_columns=["image"]
|
||||
)["train"]:
|
||||
b["image"] = b["image_bytes"]
|
||||
del b["image_bytes"]
|
||||
data.append(b)
|
||||
tbl = pa.Table.from_pylist(data, schema=schema)
|
||||
return tbl
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-o", "--output", metavar="DIR", help="Output lance directory", required=True
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s",
|
||||
"--subset",
|
||||
choices=["2m_all", "2m_first_10k", "2m_first_100k"],
|
||||
default="2m_first_10k",
|
||||
help="subset of the hg dataset",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
batches = datagen(args)
|
||||
lance.write_dataset(batches, args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
9
notebooks/diffusiondb/requirements.txt
Normal file
9
notebooks/diffusiondb/requirements.txt
Normal file
@@ -0,0 +1,9 @@
|
||||
datasets
|
||||
Pillow
|
||||
lancedb
|
||||
isort
|
||||
black
|
||||
transformers
|
||||
--index-url https://download.pytorch.org/whl/cu118
|
||||
torch
|
||||
torchvision
|
||||
240
notebooks/multimodal_search.ipynb
Normal file
240
notebooks/multimodal_search.ipynb
Normal file
@@ -0,0 +1,240 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
|
||||
"\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!pip install --quiet -U lancedb\n",
|
||||
"!pip install --quiet gradio transformers torch torchvision"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 60,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import io\n",
|
||||
"import PIL\n",
|
||||
"import duckdb\n",
|
||||
"import lancedb"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## First run setup: Download data and pre-process"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<lance.dataset.LanceDataset at 0x3045db590>"
|
||||
]
|
||||
},
|
||||
"execution_count": 30,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# remove null prompts\n",
|
||||
"import lance\n",
|
||||
"import pyarrow.compute as pc\n",
|
||||
"\n",
|
||||
"# download s3://eto-public/datasets/diffusiondb/small_10k.lance to this uri\n",
|
||||
"data = lance.dataset(\"~/datasets/rawdata.lance\").to_table()\n",
|
||||
"\n",
|
||||
"# First data processing and full-text-search index\n",
|
||||
"db = lancedb.connect(\"~/datasets/demo\")\n",
|
||||
"tbl = db.create_table(\"diffusiondb\", data.filter(~pc.field(\"prompt\").is_null()))\n",
|
||||
"tbl = tbl.create_fts_index([\"prompt\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Create / Open LanceDB Table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 62,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"db = lancedb.connect(\"~/datasets/demo\")\n",
|
||||
"tbl = db.open_table(\"diffusiondb\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Create CLIP embedding function for the text"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 63,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"openai/clip-vit-base-patch32\"\n",
|
||||
"\n",
|
||||
"tokenizer = CLIPTokenizerFast.from_pretrained(MODEL_ID)\n",
|
||||
"model = CLIPModel.from_pretrained(MODEL_ID)\n",
|
||||
"processor = CLIPProcessor.from_pretrained(MODEL_ID)\n",
|
||||
"\n",
|
||||
"def embed_func(query):\n",
|
||||
" inputs = tokenizer([query], padding=True, return_tensors=\"pt\")\n",
|
||||
" text_features = model.get_text_features(**inputs)\n",
|
||||
" return text_features.detach().numpy()[0]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Search functions for Gradio"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 64,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def find_image_vectors(query):\n",
|
||||
" emb = embed_func(query)\n",
|
||||
" return _extract(tbl.search(emb).limit(9).to_df())\n",
|
||||
"\n",
|
||||
"def find_image_keywords(query):\n",
|
||||
" return _extract(tbl.search(query).limit(9).to_df())\n",
|
||||
"\n",
|
||||
"def find_image_sql(query):\n",
|
||||
" diffusiondb = tbl.to_lance()\n",
|
||||
" return _extract(duckdb.query(query).to_df())\n",
|
||||
"\n",
|
||||
"def _extract(df):\n",
|
||||
" image_col = \"image\"\n",
|
||||
" return [(PIL.Image.open(io.BytesIO(row[image_col])), row[\"prompt\"]) for _, row in df.iterrows()]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup Gradio interface"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 65,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Running on local URL: http://127.0.0.1:7867\n",
|
||||
"\n",
|
||||
"To create a public link, set `share=True` in `launch()`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div><iframe src=\"http://127.0.0.1:7867/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": []
|
||||
},
|
||||
"execution_count": 65,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import gradio as gr\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"with gr.Blocks() as demo:\n",
|
||||
"\n",
|
||||
" with gr.Row():\n",
|
||||
" with gr.Tab(\"Embeddings\"):\n",
|
||||
" vector_query = gr.Textbox(value=\"portraits of a person\", show_label=False)\n",
|
||||
" b1 = gr.Button(\"Submit\")\n",
|
||||
" with gr.Tab(\"Keywords\"):\n",
|
||||
" keyword_query = gr.Textbox(value=\"ninja turtle\", show_label=False)\n",
|
||||
" b2 = gr.Button(\"Submit\")\n",
|
||||
" with gr.Tab(\"SQL\"):\n",
|
||||
" sql_query = gr.Textbox(value=\"SELECT * from diffusiondb WHERE image_nsfw >= 2 LIMIT 9\", show_label=False)\n",
|
||||
" b3 = gr.Button(\"Submit\")\n",
|
||||
" with gr.Row():\n",
|
||||
" gallery = gr.Gallery(\n",
|
||||
" label=\"Found images\", show_label=False, elem_id=\"gallery\"\n",
|
||||
" ).style(columns=[3], rows=[3], object_fit=\"contain\", height=\"auto\") \n",
|
||||
" \n",
|
||||
" b1.click(find_image_vectors, inputs=vector_query, outputs=gallery)\n",
|
||||
" b2.click(find_image_keywords, inputs=keyword_query, outputs=gallery)\n",
|
||||
" b3.click(find_image_sql, inputs=sql_query, outputs=gallery)\n",
|
||||
" \n",
|
||||
"demo.launch()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
@@ -13,13 +13,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
import pyarrow as pa
|
||||
from pyarrow import fs
|
||||
|
||||
from .common import DATA, URI
|
||||
from .table import LanceTable
|
||||
from .util import get_uri_scheme
|
||||
from .util import get_uri_scheme, get_uri_location
|
||||
|
||||
|
||||
class LanceDBConnection:
|
||||
@@ -47,12 +50,21 @@ class LanceDBConnection:
|
||||
-------
|
||||
A list of table names.
|
||||
"""
|
||||
if get_uri_scheme(self.uri) == "file":
|
||||
return [p.stem for p in Path(self.uri).glob("*.lance")]
|
||||
try:
|
||||
filesystem, path = fs.FileSystem.from_uri(self.uri)
|
||||
except pa.ArrowInvalid:
|
||||
raise NotImplementedError(
|
||||
"List table_names is only supported for local filesystem for now"
|
||||
"Unsupported scheme: " + self.uri
|
||||
)
|
||||
|
||||
try:
|
||||
paths = filesystem.get_file_info(fs.FileSelector(get_uri_location(self.uri)))
|
||||
except FileNotFoundError:
|
||||
# It is ok if the file does not exist since it will be created
|
||||
paths = []
|
||||
tables = [os.path.splitext(file_info.base_name)[0] for file_info in paths if file_info.extension == 'lance']
|
||||
return tables
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.table_names())
|
||||
|
||||
@@ -112,3 +124,15 @@ class LanceDBConnection:
|
||||
A LanceTable object representing the table.
|
||||
"""
|
||||
return LanceTable(self, name)
|
||||
|
||||
def drop_table(self, name: str):
|
||||
"""Drop a table from the database.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name: str
|
||||
The name of the table.
|
||||
"""
|
||||
filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
|
||||
table_path = os.path.join(path, name + ".lance")
|
||||
filesystem.delete_dir(table_path)
|
||||
|
||||
@@ -253,8 +253,7 @@ def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table
|
||||
vector_column_name: str
|
||||
The name of the vector column.
|
||||
"""
|
||||
i = data.column_names.index(vector_column_name)
|
||||
if i < 0:
|
||||
if vector_column_name not in data.column_names:
|
||||
raise ValueError(f"Missing vector column: {vector_column_name}")
|
||||
vec_arr = data[vector_column_name].combine_chunks()
|
||||
if pa.types.is_fixed_size_list(vec_arr.type):
|
||||
@@ -266,4 +265,4 @@ def _sanitize_vector_column(data: pa.Table, vector_column_name: str) -> pa.Table
|
||||
values = values.cast(pa.float32())
|
||||
list_size = len(values) / len(data)
|
||||
vec_arr = pa.FixedSizeListArray.from_arrays(values, list_size)
|
||||
return data.set_column(i, vector_column_name, vec_arr)
|
||||
return data.set_column(data.column_names.index(vector_column_name), vector_column_name, vec_arr)
|
||||
|
||||
@@ -41,3 +41,23 @@ def get_uri_scheme(uri: str) -> str:
|
||||
# So we add special handling here for schemes that are a single character
|
||||
scheme = "file"
|
||||
return scheme
|
||||
|
||||
|
||||
def get_uri_location(uri: str) -> str:
|
||||
"""
|
||||
Get the location of a URI. If the parameter is not a url, assumes it is just a path
|
||||
|
||||
Parameters
|
||||
----------
|
||||
uri : str
|
||||
The URI to parse.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str: Location part of the URL, without scheme
|
||||
"""
|
||||
parsed = urlparse(uri)
|
||||
if not parsed.netloc:
|
||||
return parsed.path
|
||||
else:
|
||||
return parsed.netloc + parsed.path
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "lancedb"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
dependencies = ["pylance>=0.4.17", "ratelimiter", "retry", "tqdm"]
|
||||
description = "lancedb"
|
||||
authors = [
|
||||
|
||||
@@ -97,3 +97,26 @@ def test_create_mode(tmp_path):
|
||||
)
|
||||
tbl = db.create_table("test", data=new_data, mode="overwrite")
|
||||
assert tbl.to_pandas().item.tolist() == ["fizz", "buzz"]
|
||||
|
||||
|
||||
def test_delete_table(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
data = pd.DataFrame(
|
||||
{
|
||||
"vector": [[3.1, 4.1], [5.9, 26.5]],
|
||||
"item": ["foo", "bar"],
|
||||
"price": [10.0, 20.0],
|
||||
}
|
||||
)
|
||||
db.create_table("test", data=data)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
db.create_table("test", data=data)
|
||||
|
||||
assert db.table_names() == ["test"]
|
||||
|
||||
db.drop_table("test")
|
||||
assert db.table_names() == []
|
||||
|
||||
db.create_table("test", data=data)
|
||||
assert db.table_names() == ["test"]
|
||||
49
python/tests/test_io.py
Normal file
49
python/tests/test_io.py
Normal file
@@ -0,0 +1,49 @@
|
||||
# Copyright 2023 LanceDB Developers
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
|
||||
# You need to setup AWS credentials an a base path to run this test. Example
|
||||
# AWS_PROFILE=default TEST_S3_BASE_URL=s3://my_bucket/dataset pytest tests/test_io.py
|
||||
|
||||
@pytest.mark.skipif(
|
||||
(os.environ.get("TEST_S3_BASE_URL") is None),
|
||||
reason="please setup s3 base url",
|
||||
)
|
||||
def test_s3_io():
|
||||
db = lancedb.connect(os.environ.get("TEST_S3_BASE_URL"))
|
||||
assert db.table_names() == []
|
||||
|
||||
table = db.create_table(
|
||||
"test",
|
||||
data=[
|
||||
{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
|
||||
{"vector": [5.9, 26.5], "item": "bar", "price": 20.0},
|
||||
],
|
||||
)
|
||||
rs = table.search([100, 100]).limit(1).to_df()
|
||||
assert len(rs) == 1
|
||||
assert rs["item"].iloc[0] == "bar"
|
||||
|
||||
rs = table.search([100, 100]).where("price < 15").limit(2).to_df()
|
||||
assert len(rs) == 1
|
||||
assert rs["item"].iloc[0] == "foo"
|
||||
|
||||
assert db.table_names() == ["test"]
|
||||
assert "test" in db
|
||||
assert len(db) == 1
|
||||
|
||||
assert db.open_table("test").name == db["test"].name
|
||||
@@ -13,7 +13,7 @@ arrow-array = "37.0"
|
||||
arrow-data = "37.0"
|
||||
arrow-schema = "37.0"
|
||||
object_store = "0.5.6"
|
||||
|
||||
snafu = "0.7.4"
|
||||
lance = "0.4.17"
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread"] }
|
||||
|
||||
|
||||
@@ -17,8 +17,9 @@ use std::path::Path;
|
||||
|
||||
use arrow_array::RecordBatchReader;
|
||||
use lance::io::object_store::ObjectStore;
|
||||
use snafu::prelude::*;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::error::{CreateDirSnafu, Result};
|
||||
use crate::table::Table;
|
||||
|
||||
pub struct Database {
|
||||
@@ -43,10 +44,7 @@ impl Database {
|
||||
pub async fn connect(uri: &str) -> Result<Database> {
|
||||
let object_store = ObjectStore::new(uri).await?;
|
||||
if object_store.is_local() {
|
||||
let path = Path::new(uri);
|
||||
if !path.try_exists()? {
|
||||
create_dir_all(&path)?;
|
||||
}
|
||||
Self::try_create_dir(uri).context(CreateDirSnafu { path: uri })?;
|
||||
}
|
||||
Ok(Database {
|
||||
uri: uri.to_string(),
|
||||
@@ -54,6 +52,15 @@ impl Database {
|
||||
})
|
||||
}
|
||||
|
||||
/// Try to create a local directory to store the lancedb dataset
|
||||
fn try_create_dir(path: &str) -> core::result::Result<(), std::io::Error> {
|
||||
let path = Path::new(path);
|
||||
if !path.try_exists()? {
|
||||
create_dir_all(&path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the names of all tables in the database.
|
||||
///
|
||||
/// # Returns
|
||||
|
||||
@@ -12,44 +12,50 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
IO(String),
|
||||
Lance(String),
|
||||
}
|
||||
use snafu::Snafu;
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let (catalog, message) = match self {
|
||||
Self::IO(s) => ("I/O", s.as_str()),
|
||||
Self::Lance(s) => ("Lance", s.as_str()),
|
||||
};
|
||||
write!(f, "LanceDBError({catalog}): {message}")
|
||||
}
|
||||
#[derive(Debug, Snafu)]
|
||||
#[snafu(visibility(pub(crate)))]
|
||||
pub enum Error {
|
||||
#[snafu(display("LanceDBError: Invalid table name: {name}"))]
|
||||
InvalidTableName { name: String },
|
||||
#[snafu(display("LanceDBError: Table '{name}' was not found"))]
|
||||
TableNotFound { name: String },
|
||||
#[snafu(display("LanceDBError: Table '{name}' already exists"))]
|
||||
TableAlreadyExists { name: String },
|
||||
#[snafu(display("LanceDBError: Unable to created lance dataset at {path}: {source}"))]
|
||||
CreateDir {
|
||||
path: String,
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[snafu(display("LanceDBError: {message}"))]
|
||||
Store { message: String },
|
||||
#[snafu(display("LanceDBError: {message}"))]
|
||||
Lance { message: String },
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl From<std::io::Error> for Error {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
Self::IO(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lance::Error> for Error {
|
||||
fn from(e: lance::Error) -> Self {
|
||||
Self::Lance(e.to_string())
|
||||
Self::Lance {
|
||||
message: e.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<object_store::Error> for Error {
|
||||
fn from(e: object_store::Error) -> Self {
|
||||
Self::IO(e.to_string())
|
||||
Self::Store {
|
||||
message: e.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<object_store::path::Error> for Error {
|
||||
fn from(e: object_store::path::Error) -> Self {
|
||||
Self::IO(e.to_string())
|
||||
Self::Store {
|
||||
message: e.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,8 +18,9 @@ use std::sync::Arc;
|
||||
use arrow_array::{Float32Array, RecordBatchReader};
|
||||
use lance::dataset::{Dataset, WriteMode, WriteParams};
|
||||
use lance::index::IndexType;
|
||||
use snafu::prelude::*;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::error::{Error, InvalidTableNameSnafu, Result};
|
||||
use crate::index::vector::VectorIndexBuilder;
|
||||
use crate::query::Query;
|
||||
|
||||
@@ -27,6 +28,7 @@ pub const VECTOR_COLUMN_NAME: &str = "vector";
|
||||
pub const LANCE_FILE_EXTENSION: &str = "lance";
|
||||
|
||||
/// A table in a LanceDB database.
|
||||
#[derive(Debug)]
|
||||
pub struct Table {
|
||||
name: String,
|
||||
uri: String,
|
||||
@@ -57,9 +59,16 @@ impl Table {
|
||||
let uri = table_uri
|
||||
.as_path()
|
||||
.to_str()
|
||||
.ok_or(Error::IO(format!("Invalid table name: {}", name)))?;
|
||||
.context(InvalidTableNameSnafu { name })?;
|
||||
|
||||
let dataset = Dataset::open(&uri).await?;
|
||||
let dataset = Dataset::open(&uri).await.map_err(|e| match e {
|
||||
lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
|
||||
name: name.to_string(),
|
||||
},
|
||||
e => Error::Lance {
|
||||
message: e.to_string(),
|
||||
},
|
||||
})?;
|
||||
Ok(Table {
|
||||
name: name.to_string(),
|
||||
uri: uri.to_string(),
|
||||
@@ -88,14 +97,22 @@ impl Table {
|
||||
let uri = table_uri
|
||||
.as_path()
|
||||
.to_str()
|
||||
.ok_or(Error::IO(format!("Invalid table name: {}", name)))?
|
||||
.context(InvalidTableNameSnafu { name })?
|
||||
.to_string();
|
||||
let dataset =
|
||||
Arc::new(Dataset::write(&mut batches, &uri, Some(WriteParams::default())).await?);
|
||||
let dataset = Dataset::write(&mut batches, &uri, Some(WriteParams::default()))
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
lance::Error::DatasetAlreadyExists { .. } => Error::TableAlreadyExists {
|
||||
name: name.to_string(),
|
||||
},
|
||||
e => Error::Lance {
|
||||
message: e.to_string(),
|
||||
},
|
||||
})?;
|
||||
Ok(Table {
|
||||
name: name.to_string(),
|
||||
uri,
|
||||
dataset,
|
||||
dataset: Arc::new(dataset),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -178,15 +195,6 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::index::vector::IvfPQIndexBuilder;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_table_not_exists() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let uri = tmp_dir.path().to_str().unwrap();
|
||||
|
||||
let table = Table::open(&uri, "test").await;
|
||||
assert!(table.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_open() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
@@ -203,6 +211,14 @@ mod tests {
|
||||
assert_eq!(table.name, "test")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_open_not_found() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let uri = tmp_dir.path().to_str().unwrap();
|
||||
let table = Table::open(uri, "test").await;
|
||||
assert!(matches!(table.unwrap_err(), Error::TableNotFound { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_object_store_path() {
|
||||
use std::path::Path as StdPath;
|
||||
@@ -211,6 +227,23 @@ mod tests {
|
||||
assert_eq!(c.to_str().unwrap(), "s3://bucket/path/to/file/subfile");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_already_exists() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let uri = tmp_dir.path().to_str().unwrap();
|
||||
|
||||
let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
|
||||
let schema = batches.schema().clone();
|
||||
Table::create(&uri, "test", batches).await.unwrap();
|
||||
|
||||
let batches: Box<dyn RecordBatchReader> = Box::new(make_test_batches());
|
||||
let result = Table::create(&uri, "test", batches).await;
|
||||
assert!(matches!(
|
||||
result.unwrap_err(),
|
||||
Error::TableAlreadyExists { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_add() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user