mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-23 13:29:57 +00:00
Compare commits
7 Commits
python-v0.
...
python-v0.
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2c36767f20 | ||
|
|
1fa7e96aa1 | ||
|
|
7ae327242b | ||
|
|
1f4a051070 | ||
|
|
92c93b08bf | ||
|
|
a363b02ca7 | ||
|
|
ff8eaab894 |
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.6.0"
|
current_version = "0.7.0"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
@@ -109,7 +109,7 @@ nav:
|
|||||||
- Filtering: sql.md
|
- Filtering: sql.md
|
||||||
- Versioning & Reproducibility: notebooks/reproducibility.ipynb
|
- Versioning & Reproducibility: notebooks/reproducibility.ipynb
|
||||||
- Configuring Storage: guides/storage.md
|
- Configuring Storage: guides/storage.md
|
||||||
- Sync -> Async Migration Guide: migration.md
|
- Migration Guide: migration.md
|
||||||
- Tuning retrieval performance:
|
- Tuning retrieval performance:
|
||||||
- Choosing right query type: guides/tuning_retrievers/1_query_types.md
|
- Choosing right query type: guides/tuning_retrievers/1_query_types.md
|
||||||
- Reranking: guides/tuning_retrievers/2_reranking.md
|
- Reranking: guides/tuning_retrievers/2_reranking.md
|
||||||
@@ -194,7 +194,7 @@ nav:
|
|||||||
- Filtering: sql.md
|
- Filtering: sql.md
|
||||||
- Versioning & Reproducibility: notebooks/reproducibility.ipynb
|
- Versioning & Reproducibility: notebooks/reproducibility.ipynb
|
||||||
- Configuring Storage: guides/storage.md
|
- Configuring Storage: guides/storage.md
|
||||||
- Sync -> Async Migration Guide: migration.md
|
- Migration Guide: migration.md
|
||||||
- Tuning retrieval performance:
|
- Tuning retrieval performance:
|
||||||
- Choosing right query type: guides/tuning_retrievers/1_query_types.md
|
- Choosing right query type: guides/tuning_retrievers/1_query_types.md
|
||||||
- Reranking: guides/tuning_retrievers/2_reranking.md
|
- Reranking: guides/tuning_retrievers/2_reranking.md
|
||||||
|
|||||||
@@ -9,7 +9,8 @@ around the asynchronous client.
|
|||||||
This guide describes the differences between the two APIs and will hopefully assist users
|
This guide describes the differences between the two APIs and will hopefully assist users
|
||||||
that would like to migrate to the new API.
|
that would like to migrate to the new API.
|
||||||
|
|
||||||
## Closeable Connections
|
## Python
|
||||||
|
### Closeable Connections
|
||||||
|
|
||||||
The Connection now has a `close` method. You can call this when
|
The Connection now has a `close` method. You can call this when
|
||||||
you are done with the connection to eagerly free resources. Currently
|
you are done with the connection to eagerly free resources. Currently
|
||||||
@@ -32,20 +33,20 @@ async def my_async_fn():
|
|||||||
It is not mandatory to call the `close` method. If you do not call it
|
It is not mandatory to call the `close` method. If you do not call it
|
||||||
then the connection will be closed when the object is garbage collected.
|
then the connection will be closed when the object is garbage collected.
|
||||||
|
|
||||||
## Closeable Table
|
### Closeable Table
|
||||||
|
|
||||||
The Table now also has a `close` method, similar to the connection. This
|
The Table now also has a `close` method, similar to the connection. This
|
||||||
can be used to eagerly free the cache used by a Table object. Similar to
|
can be used to eagerly free the cache used by a Table object. Similar to
|
||||||
the connection, it can be used as a context manager and it is not mandatory
|
the connection, it can be used as a context manager and it is not mandatory
|
||||||
to call the `close` method.
|
to call the `close` method.
|
||||||
|
|
||||||
### Changes to Table APIs
|
#### Changes to Table APIs
|
||||||
|
|
||||||
- Previously `Table.schema` was a property. Now it is an async method.
|
- Previously `Table.schema` was a property. Now it is an async method.
|
||||||
- The method `Table.__len__` was removed and `len(table)` will no longer
|
- The method `Table.__len__` was removed and `len(table)` will no longer
|
||||||
work. Use `Table.count_rows` instead.
|
work. Use `Table.count_rows` instead.
|
||||||
|
|
||||||
### Creating Indices
|
#### Creating Indices
|
||||||
|
|
||||||
The `Table.create_index` method is now used for creating both vector indices
|
The `Table.create_index` method is now used for creating both vector indices
|
||||||
and scalar indices. It currently requires a column name to be specified (the
|
and scalar indices. It currently requires a column name to be specified (the
|
||||||
@@ -55,12 +56,12 @@ the size of the data.
|
|||||||
To specify index configuration details you will need to specify which kind of
|
To specify index configuration details you will need to specify which kind of
|
||||||
index you are using.
|
index you are using.
|
||||||
|
|
||||||
### Querying
|
#### Querying
|
||||||
|
|
||||||
The `Table.search` method has been renamed to `AsyncTable.vector_search` for
|
The `Table.search` method has been renamed to `AsyncTable.vector_search` for
|
||||||
clarity.
|
clarity.
|
||||||
|
|
||||||
## Features not yet supported
|
### Features not yet supported
|
||||||
|
|
||||||
The following features are not yet supported by the asynchronous API. However,
|
The following features are not yet supported by the asynchronous API. However,
|
||||||
we plan to support them soon.
|
we plan to support them soon.
|
||||||
@@ -74,3 +75,22 @@ we plan to support them soon.
|
|||||||
search
|
search
|
||||||
- Remote connections to LanceDb Cloud are not yet supported.
|
- Remote connections to LanceDb Cloud are not yet supported.
|
||||||
- The method Table.head is not yet supported.
|
- The method Table.head is not yet supported.
|
||||||
|
|
||||||
|
## TypeScript/JavaScript
|
||||||
|
|
||||||
|
For JS/TS users, we offer a brand new SDK [@lancedb/lancedb](https://www.npmjs.com/package/@lancedb/lancedb)
|
||||||
|
|
||||||
|
### Changes to Table APIs
|
||||||
|
|
||||||
|
Previously `Table.schema` was a property. Now it is an async method.
|
||||||
|
|
||||||
|
|
||||||
|
#### Creating Indices
|
||||||
|
|
||||||
|
The `Table.createIndex` method is now used for creating both vector indices
|
||||||
|
and scalar indices. It currently requires a column name to be specified (the
|
||||||
|
column to index). Vector index defaults are now smarter and scale better with
|
||||||
|
the size of the data.
|
||||||
|
|
||||||
|
To specify index configuration details you will need to specify which kind of
|
||||||
|
index you are using.
|
||||||
|
|||||||
4
node/package-lock.json
generated
4
node/package-lock.json
generated
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "vectordb",
|
"name": "vectordb",
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "vectordb",
|
"name": "vectordb",
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64",
|
"x64",
|
||||||
"arm64"
|
"arm64"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "vectordb",
|
"name": "vectordb",
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"description": " Serverless, low-latency vector database for AI applications",
|
"description": " Serverless, low-latency vector database for AI applications",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"types": "dist/index.d.ts",
|
"types": "dist/index.d.ts",
|
||||||
|
|||||||
@@ -62,6 +62,8 @@ export {
|
|||||||
|
|
||||||
const defaultAwsRegion = "us-west-2";
|
const defaultAwsRegion = "us-west-2";
|
||||||
|
|
||||||
|
const defaultRequestTimeout = 10_000
|
||||||
|
|
||||||
export interface AwsCredentials {
|
export interface AwsCredentials {
|
||||||
accessKeyId: string
|
accessKeyId: string
|
||||||
|
|
||||||
@@ -119,6 +121,11 @@ export interface ConnectionOptions {
|
|||||||
*/
|
*/
|
||||||
hostOverride?: string
|
hostOverride?: string
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Duration in milliseconds for request timeout. Default = 10,000 (10 seconds)
|
||||||
|
*/
|
||||||
|
timeout?: number
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* (For LanceDB OSS only): The interval, in seconds, at which to check for
|
* (For LanceDB OSS only): The interval, in seconds, at which to check for
|
||||||
* updates to the table from other processes. If None, then consistency is not
|
* updates to the table from other processes. If None, then consistency is not
|
||||||
@@ -204,7 +211,8 @@ export async function connect(
|
|||||||
awsCredentials: undefined,
|
awsCredentials: undefined,
|
||||||
awsRegion: defaultAwsRegion,
|
awsRegion: defaultAwsRegion,
|
||||||
apiKey: undefined,
|
apiKey: undefined,
|
||||||
region: defaultAwsRegion
|
region: defaultAwsRegion,
|
||||||
|
timeout: defaultRequestTimeout
|
||||||
},
|
},
|
||||||
arg
|
arg
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ async function callWithMiddlewares (
|
|||||||
if (i > middlewares.length) {
|
if (i > middlewares.length) {
|
||||||
const headers = Object.fromEntries(req.headers.entries())
|
const headers = Object.fromEntries(req.headers.entries())
|
||||||
const params = Object.fromEntries(req.params?.entries() ?? [])
|
const params = Object.fromEntries(req.params?.entries() ?? [])
|
||||||
const timeout = 10000
|
const timeout = opts?.timeout
|
||||||
let res
|
let res
|
||||||
if (req.method === Method.POST) {
|
if (req.method === Method.POST) {
|
||||||
res = await axios.post(
|
res = await axios.post(
|
||||||
@@ -82,6 +82,7 @@ async function callWithMiddlewares (
|
|||||||
|
|
||||||
interface MiddlewareInvocationOptions {
|
interface MiddlewareInvocationOptions {
|
||||||
responseType?: ResponseType
|
responseType?: ResponseType
|
||||||
|
timeout?: number,
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -123,15 +124,19 @@ export class HttpLancedbClient {
|
|||||||
private readonly _url: string
|
private readonly _url: string
|
||||||
private readonly _apiKey: () => string
|
private readonly _apiKey: () => string
|
||||||
private readonly _middlewares: HttpLancedbClientMiddleware[]
|
private readonly _middlewares: HttpLancedbClientMiddleware[]
|
||||||
|
private readonly _timeout: number | undefined
|
||||||
|
|
||||||
public constructor (
|
public constructor (
|
||||||
url: string,
|
url: string,
|
||||||
apiKey: string,
|
apiKey: string,
|
||||||
private readonly _dbName?: string
|
timeout?: number,
|
||||||
|
private readonly _dbName?: string,
|
||||||
|
|
||||||
) {
|
) {
|
||||||
this._url = url
|
this._url = url
|
||||||
this._apiKey = () => apiKey
|
this._apiKey = () => apiKey
|
||||||
this._middlewares = []
|
this._middlewares = []
|
||||||
|
this._timeout = timeout
|
||||||
}
|
}
|
||||||
|
|
||||||
get uri (): string {
|
get uri (): string {
|
||||||
@@ -230,7 +235,10 @@ export class HttpLancedbClient {
|
|||||||
|
|
||||||
let response
|
let response
|
||||||
try {
|
try {
|
||||||
response = await callWithMiddlewares(req, this._middlewares, { responseType })
|
response = await callWithMiddlewares(req, this._middlewares, {
|
||||||
|
responseType,
|
||||||
|
timeout: this._timeout,
|
||||||
|
})
|
||||||
|
|
||||||
// return response
|
// return response
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
@@ -267,7 +275,7 @@ export class HttpLancedbClient {
|
|||||||
* Make a clone of this client
|
* Make a clone of this client
|
||||||
*/
|
*/
|
||||||
private clone (): HttpLancedbClient {
|
private clone (): HttpLancedbClient {
|
||||||
const clone = new HttpLancedbClient(this._url, this._apiKey(), this._dbName)
|
const clone = new HttpLancedbClient(this._url, this._apiKey(), this._timeout, this._dbName)
|
||||||
for (const mw of this._middlewares) {
|
for (const mw of this._middlewares) {
|
||||||
clone._middlewares.push(mw)
|
clone._middlewares.push(mw)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,6 +72,7 @@ export class RemoteConnection implements Connection {
|
|||||||
this._client = new HttpLancedbClient(
|
this._client = new HttpLancedbClient(
|
||||||
server,
|
server,
|
||||||
opts.apiKey,
|
opts.apiKey,
|
||||||
|
opts.timeout,
|
||||||
opts.hostOverride === undefined ? undefined : this._dbName
|
opts.hostOverride === undefined ? undefined : this._dbName
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-arm64",
|
"name": "@lancedb/lancedb-darwin-arm64",
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.darwin-arm64.node",
|
"main": "lancedb.darwin-arm64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-x64",
|
"name": "@lancedb/lancedb-darwin-x64",
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.darwin-x64.node",
|
"main": "lancedb.darwin-x64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-gnu.node",
|
"main": "lancedb.linux-arm64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-gnu.node",
|
"main": "lancedb.linux-x64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"os": ["win32"],
|
"os": ["win32"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.win32-x64-msvc.node",
|
"main": "lancedb.win32-x64-msvc.node",
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
"vector database",
|
"vector database",
|
||||||
"ann"
|
"ann"
|
||||||
],
|
],
|
||||||
"version": "0.6.0",
|
"version": "0.7.0",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./dist/index.js",
|
".": "./dist/index.js",
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.10.0"
|
current_version = "0.10.1"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.10.0"
|
version = "0.10.1"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "Python bindings for LanceDB"
|
description = "Python bindings for LanceDB"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-node"
|
name = "lancedb-node"
|
||||||
version = "0.6.0"
|
version = "0.7.0"
|
||||||
description = "Serverless, low-latency vector database for AI applications"
|
description = "Serverless, low-latency vector database for AI applications"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.6.0"
|
version = "0.7.0"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
Reference in New Issue
Block a user