diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 9c04c1ed..39ff39ee 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -12,6 +12,15 @@ theme:
- content.code.copy
- content.tabs.link
- content.action.edit
+ - toc.follow
+ - toc.integrate
+ - navigation.top
+ - navigation.tabs
+ - navigation.tabs.sticky
+ - navigation.footer
+ - navigation.tracking
+ - navigation.instant
+ - navigation.indexes
icon:
repo: fontawesome/brands/github
custom_dir: overrides
@@ -55,11 +64,43 @@ markdown_extensions:
- md_in_html
nav:
-- Home: index.md
+- Home:
+ - 🏢 Home: index.md
+ - 💡 Basics: basic.md
+ - 🧬 Embeddings: embedding.md
+ - 🔍 Python full-text search: fts.md
+ - 🔌 Integrations:
+ - integrations/index.md
+ - Pandas and PyArrow: python/arrow.md
+ - DuckDB: python/duckdb.md
+ - LangChain 🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
+ - LangChain JS/TS 🔗: https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb
+ - LlamaIndex 🦙: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+ - Pydantic: python/pydantic.md
+ - Voxel51: integrations/voxel51.md
+ - PromptTools: integrations/prompttools.md
+ - 🐍 Python examples:
+ - examples/index.md
+ - YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
+ - Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
+ - Multimodal search using CLIP: notebooks/multimodal_search.ipynb
+ - Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
+ - Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
+ - 🌐 Javascript examples:
+ - Examples: examples/index_js.md
+ - Serverless Website Chatbot: examples/serverless_website_chatbot.md
+ - YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+ - TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
+ - 📚 Guides:
+ - Tables: guides/tables.md
+ - Vector Search: search.md
+ - SQL filters: sql.md
+ - Indexing: ann_indexes.md
- Basics: basic.md
- Embeddings: embedding.md
- Python full-text search: fts.md
- Integrations:
+ - integrations/index.md
- Pandas and PyArrow: python/arrow.md
- DuckDB: python/duckdb.md
- LangChain 🦜️🔗: https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html
@@ -69,13 +110,16 @@ nav:
- Voxel51: integrations/voxel51.md
- PromptTools: integrations/prompttools.md
- Python examples:
+ - examples/index.md
- YouTube Transcript Search: notebooks/youtube_transcript_search.ipynb
- Documentation QA Bot using LangChain: notebooks/code_qa_bot.ipynb
- Multimodal search using CLIP: notebooks/multimodal_search.ipynb
- Serverless QA Bot with S3 and Lambda: examples/serverless_lancedb_with_s3_and_lambda.md
- Serverless QA Bot with Modal: examples/serverless_qa_bot_with_modal_and_langchain.md
- Javascript examples:
+ - examples/index_js.md
- YouTube Transcript Search: examples/youtube_transcript_bot_with_nodejs.md
+ - Serverless Chatbot from any website: examples/serverless_website_chatbot.md
- TransformersJS Embedding Search: examples/transformerjs_embedding_search_nodejs.md
- Guides:
@@ -86,6 +130,7 @@ nav:
- API references:
- Python API: python/python.md
- Javascript API: javascript/modules.md
+- LanceDB Cloud↗: https://noteforms.com/forms/lancedb-mailing-list-cloud-kty1o5?notionforms=1&utm_source=notionforms
extra_css:
- styles/global.css
diff --git a/docs/src/assets/ecosystem-illustration.png b/docs/src/assets/ecosystem-illustration.png
new file mode 100644
index 00000000..553d6b11
Binary files /dev/null and b/docs/src/assets/ecosystem-illustration.png differ
diff --git a/docs/src/assets/langchain.png b/docs/src/assets/langchain.png
new file mode 100644
index 00000000..d7be35f7
Binary files /dev/null and b/docs/src/assets/langchain.png differ
diff --git a/docs/src/assets/llama-index.jpg b/docs/src/assets/llama-index.jpg
new file mode 100644
index 00000000..0e738331
Binary files /dev/null and b/docs/src/assets/llama-index.jpg differ
diff --git a/docs/src/assets/prompttools.jpeg b/docs/src/assets/prompttools.jpeg
new file mode 100644
index 00000000..ff783f1b
Binary files /dev/null and b/docs/src/assets/prompttools.jpeg differ
diff --git a/docs/src/assets/vercel-template.gif b/docs/src/assets/vercel-template.gif
new file mode 100644
index 00000000..6c16d7de
Binary files /dev/null and b/docs/src/assets/vercel-template.gif differ
diff --git a/docs/src/basic.md b/docs/src/basic.md
index 328c2dfb..53c235ae 100644
--- a/docs/src/basic.md
+++ b/docs/src/basic.md
@@ -198,3 +198,15 @@ you can pass in `ignore_missing=True`.
This section covered the very basics of the LanceDB API.
LanceDB supports many additional features when creating indices to speed up search and options for search.
These are contained in the next section of the documentation.
+
+## Note: Bundling vectorDB apps with webpack
+Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
+```javascript
+/** @type {import('next').NextConfig} */
+module.exports = ({
+ webpack(config) {
+ config.externals.push({ vectordb: 'vectordb' })
+ return config;
+ }
+})
+```
\ No newline at end of file
diff --git a/docs/src/embedding.md b/docs/src/embedding.md
index 0657f9bd..e8090578 100644
--- a/docs/src/embedding.md
+++ b/docs/src/embedding.md
@@ -66,7 +66,7 @@ You can also use an external API like OpenAI to generate embeddings
to generate embeddings for each row.
Say if you have a pandas DataFrame with a `text` column that you want to be embedded,
- you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/#lancedb.embeddings.with_embeddings)
+ you can use the [with_embeddings](https://lancedb.github.io/lancedb/python/python/#lancedb.embeddings.with_embeddings)
function to generate embeddings and add create a combined pyarrow table:
diff --git a/docs/src/examples/index.md b/docs/src/examples/index.md
new file mode 100644
index 00000000..76f4b035
--- /dev/null
+++ b/docs/src/examples/index.md
@@ -0,0 +1,23 @@
+# Examples
+
+Here are some of the examples, projects and applications using LanceDB python library. Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
+
+| Example | Interactive Envs | Scripts |
+|-------- | ---------------- | ------ |
+| | | |
+| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | 
| [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/main.py)|
+| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/main.py) |
+| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/main.py)|
+| [Multimodal CLIP: DiffusionDB](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/) | | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_clip/main.py) |
+| [Multimodal CLIP: Youtube videos](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/) | | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_video_search/main.py) |
+| [Movie Recommender](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/) | | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/movie-recommender/main.py) |
+| [Audio Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/) | | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/audio_search/main.py) |
+| [Multimodal Image + Text Search](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/) | | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/multimodal_search/main.py) |
+| [Evaluating Prompts with Prompttools](https://github.com/lancedb/vectordb-recipes/tree/main/examples/prompttools-eval-prompts/) | | |
+
+## Projects & Applications powered by LanceDB
+
+| Project Name | Description | Screenshot |
+|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
+| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds |  |
+| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. |  |
diff --git a/docs/src/examples/index_js.md b/docs/src/examples/index_js.md
new file mode 100644
index 00000000..375c4a1c
--- /dev/null
+++ b/docs/src/examples/index_js.md
@@ -0,0 +1,19 @@
+# Examples
+
+Here are some of the examples, projects and applications using vectordb native javascript library.
+Some examples are covered in detail in the next sections. You can find more on [VectorDB Recipes](https://github.com/lancedb/vectordb-recipes)
+
+| Example | Scripts |
+|-------- | ------ |
+| | |
+| [Youtube transcript search bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/) | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/youtube_bot/index.js)|
+| [Langchain: Code Docs QA bot](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/) | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/Code-Documentation-QA-Bot/index.js)|
+| [AI Agents: Reducing Hallucination](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/) | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/reducing_hallucinations_ai_agents/index.js)|
+| [TransformersJS Embedding example](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/) | [](https://github.com/lancedb/vectordb-recipes/tree/main/examples/js-transformers/index.js) |
+
+## Projects & Applications
+
+| Project Name | Description | Screenshot |
+|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------|
+| [YOLOExplorer](https://github.com/lancedb/yoloexplorer) | Iterate on your YOLO / CV datasets using SQL, Vector semantic search, and more within seconds |  |
+| [Website Chatbot (Deployable Vercel Template)](https://github.com/lancedb/lancedb-vercel-chatbot) | Create a chatbot from the sitemap of any website/docs of your choice. Built using vectorDB serverless native javascript package. |  |
diff --git a/docs/src/examples/serverless_website_chatbot.md b/docs/src/examples/serverless_website_chatbot.md
new file mode 100644
index 00000000..7e763818
--- /dev/null
+++ b/docs/src/examples/serverless_website_chatbot.md
@@ -0,0 +1,61 @@
+# LanceDB Chatbot - Vercel Next.js Template
+Use an AI chatbot with website context retrieved from a vector store like LanceDB. LanceDB is lightweight and can be embedded directly into Next.js, with data stored on-prem.
+
+## One click deploy on Vercel
+[](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Flancedb%2Flancedb-vercel-chatbot&env=OPENAI_API_KEY&envDescription=OpenAI%20API%20Key%20for%20chat%20completion.&project-name=lancedb-vercel-chatbot&repository-name=lancedb-vercel-chatbot&demo-title=LanceDB%20Chatbot%20Demo&demo-description=Demo%20website%20chatbot%20with%20LanceDB.&demo-url=https%3A%2F%2Flancedb.vercel.app&demo-image=https%3A%2F%2Fi.imgur.com%2FazVJtvr.png)
+
+
+
+## Development
+
+First, rename `.env.example` to `.env.local`, and fill out `OPENAI_API_KEY` with your OpenAI API key. You can get one [here](https://openai.com/blog/openai-api).
+
+Run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about LanceDB or Next.js, take a look at the following resources:
+
+- [LanceDB Documentation](https://lancedb.github.io/lancedb/) - learn about LanceDB, the developer-friendly serverless vector database.
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+## LanceDB on Next.js and Vercel
+
+FYI: these configurations have been pre-implemented in this template.
+
+Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying on Vercel.
+```js
+/** @type {import('next').NextConfig} */
+module.exports = ({
+ webpack(config) {
+ config.externals.push({ vectordb: 'vectordb' })
+ return config;
+ }
+})
+```
+
+To deploy on Vercel, we need to make sure that the NodeJS runtime static file analysis for Vercel can find the binary, since LanceDB uses dynamic imports by default. We can do this by modifying `package.json` in the `scripts` section.
+```json
+{
+ ...
+ "scripts": {
+ ...
+ "vercel-build": "sed -i 's/nativeLib = require(`@lancedb\\/vectordb-\\${currentTarget()}`);/nativeLib = require(`@lancedb\\/vectordb-linux-x64-gnu`);/' node_modules/vectordb/native.js && next build",
+ ...
+ },
+ ...
+}
+```
diff --git a/docs/src/examples/youtube_transcript_bot.md b/docs/src/examples/youtube_transcript_bot.md
index 790e09d0..f000359a 100644
--- a/docs/src/examples/youtube_transcript_bot.md
+++ b/docs/src/examples/youtube_transcript_bot.md
@@ -7,7 +7,7 @@
-Scripts - [](./examples/youtube_bot/main.py) [](./examples/youtube_bot/index.js)
+Scripts - [](https://github.com/lancedb/vectordb-recipesexamples/youtube_bot/main.py) [](https://github.com/lancedb/vectordb-recipes/examples/youtube_bot/index.js)
This example is in a [notebook](https://github.com/lancedb/lancedb/blob/main/docs/src/notebooks/youtube_transcript_search.ipynb)
diff --git a/docs/src/guides/tables.md b/docs/src/guides/tables.md
index 08ffdb07..74b28177 100644
--- a/docs/src/guides/tables.md
+++ b/docs/src/guides/tables.md
@@ -1,4 +1,5 @@
-A Table is a collection of Records in a LanceDB Database.
+
+A Table is a collection of Records in a LanceDB Database. You can follow along on colab!
## Creating a LanceDB Table
@@ -171,7 +172,6 @@ A Table is a collection of Records in a LanceDB Database.
tbl = db.create_table("table5", schema=Model.to_arrow_schema())
```
-
=== "Javascript/Typescript"
@@ -246,23 +246,21 @@ After a table has been created, you can always add more data to it using
tbl.add(df)
```
- You can also add a large dataset batch in one go using pyArrow RecordBatch Iterator.
+ You can also add a large dataset batch in one go using Iterator of any supported data types.
- ### Adding RecordBatch Iterator
+ ### Adding to table using Iterator
```python
- import pyarrow as pa
+ import pandas as pd
def make_batches():
for i in range(5):
- yield pa.RecordBatch.from_arrays(
- [
- pa.array([[3.1, 4.1], [5.9, 26.5]]),
- pa.array(["foo", "bar"]),
- pa.array([10.0, 20.0]),
- ],
- ["vector", "item", "price"],
- )
+ yield pd.DataFrame(
+ {
+ "vector": [[3.1, 4.1], [1, 1]],
+ "item": ["foo", "bar"],
+ "price": [10.0, 20.0],
+ })
tbl.add(make_batches())
```
@@ -294,8 +292,6 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
tbl.delete('item = "fizz"')
```
- ## Examples
-
### Deleting row with specific column value
```python
diff --git a/docs/src/index.md b/docs/src/index.md
index 8472c03f..b964e19a 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -1,20 +1,23 @@
-# Welcome to LanceDB's Documentation
+# LanceDB
LanceDB is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings.
-The key features of LanceDB include:
+
-* Production-scale vector search with no servers to manage.
+The key features of LanceDB include:
* Store, query and filter vectors, metadata and multi-modal data (text, images, videos, point clouds, and more).
-* Support for vector similarity search, full-text search and SQL.
+* Support for production-scale vector similarity search, full-text search and SQL, with no servers to manage.
* Native Python and Javascript/Typescript support.
* Zero-copy, automatic versioning, manage versions of your data without needing extra infrastructure.
-* Ecosystem integrations with [LangChain 🦜️🔗](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html), [LlamaIndex 🦙](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html), Apache-Arrow, Pandas, Polars, DuckDB and more on the way.
+* Persisted on HDD, allowing scalability without breaking the bank.
+
+* Ingest your favorite data formats directly, like pandas DataFrames, Pydantic objects and more.
+
LanceDB's core is written in Rust 🦀 and is built using Lance, an open-source columnar format designed for performant ML workloads.
diff --git a/docs/src/integrations/index.md b/docs/src/integrations/index.md
new file mode 100644
index 00000000..7b696410
--- /dev/null
+++ b/docs/src/integrations/index.md
@@ -0,0 +1,21 @@
+# Integrations
+
+## Data Formats
+
+LanceDB supports ingesting from your favorite data tools.
+
+
+
+
+## Tools
+
+LanceDB is integrated with most of the popular AI tools, with more coming soon.
+Get started using these examples and quick links.
+
+| Integrations | |
+|---|---:|
+| LlamaIndex
LlamaIndex is a simple, flexible data framework for connecting custom data sources to large language models. Llama index integrates with LanceDB as the serverless VectorDB. [Lean More](https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html)
|
|
+| Langchain
Langchain allows building applications with LLMs through composability [Lean More](https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/lancedb.html) | 
|
+| Langchain TS
Javascript bindings for Langchain. It integrates with LanceDB's serverless vectordb allowing you to build powerful AI applications through composibility using only serverless functions. [Learn More]( https://js.langchain.com/docs/modules/data_connection/vectorstores/integrations/lancedb) | |
+| Voxel51
It is an open source toolkit that enables you to build better computer vision workflows by improving the quality of your datasets and delivering insights about your models.[Learn More](./voxel51.md) | 
|
+| PromptTools
Offers a set of free, open-source tools for testing and experimenting with models, prompts, and configurations. The core idea is to enable developers to evaluate prompts using familiar interfaces like code and notebooks. You can use it to experiment with different configurations of LanceDB, and test how LanceDB integrates with the LLM of your choice.[Learn More](./prompttools.md) | 
|
diff --git a/docs/src/notebooks/tables_guide.ipynb b/docs/src/notebooks/tables_guide.ipynb
new file mode 100644
index 00000000..1d6c5f0e
--- /dev/null
+++ b/docs/src/notebooks/tables_guide.ipynb
@@ -0,0 +1,831 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "d24eb4c6-e246-44ca-ba7c-6eae7923bd4c",
+ "metadata": {},
+ "source": [
+ "## LanceDB Tables\n",
+ "A Table is a collection of Records in a LanceDB Database.\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "id": "c1b4e34b-a49c-471d-a343-a5940bb5138a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install lancedb -qq"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "4e5a8d07-d9a1-48c1-913a-8e0629289579",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import lancedb\n",
+ "db = lancedb.connect(\"./.lancedb\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "66fb93d5-3551-406b-99b2-488442d61d06",
+ "metadata": {},
+ "source": [
+ "LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.\n",
+ "\n",
+ " ### From list of tuples or dictionaries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "5df12f66-8d99-43ad-8d0b-22189ec0a6b9",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "pyarrow.Table\n",
+ "vector: fixed_size_list[2]\n",
+ " child 0, item: float\n",
+ "lat: double\n",
+ "long: double\n",
+ "----\n",
+ "vector: [[[1.1,1.2],[0.2,1.8]]]\n",
+ "lat: [[45.5,40.1]]\n",
+ "long: [[-122.7,-74.1]]"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import lancedb\n",
+ "\n",
+ "db = lancedb.connect(\"./.lancedb\")\n",
+ "\n",
+ "data = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7},\n",
+ " {\"vector\": [0.2, 1.8], \"lat\": 40.1, \"long\": -74.1}]\n",
+ "\n",
+ "db.create_table(\"my_table\", data)\n",
+ "\n",
+ "db[\"my_table\"].head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "10ce802f-1a10-49ee-8ee3-a9bfb302d86c",
+ "metadata": {},
+ "source": [
+ "## From pandas DataFrame\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "f4d87ae9-0ccb-48eb-b31d-bb8f2370e47e",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "pyarrow.Table\n",
+ "vector: fixed_size_list[2]\n",
+ " child 0, item: float\n",
+ "lat: double\n",
+ "long: double\n",
+ "----\n",
+ "vector: [[[1.1,1.2],[0.2,1.8]]]\n",
+ "lat: [[45.5,40.1]]\n",
+ "long: [[-122.7,-74.1]]"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pandas as pd\n",
+ "\n",
+ "data = pd.DataFrame({\n",
+ " \"vector\": [[1.1, 1.2], [0.2, 1.8]],\n",
+ " \"lat\": [45.5, 40.1],\n",
+ " \"long\": [-122.7, -74.1]\n",
+ "})\n",
+ "\n",
+ "db.create_table(\"table2\", data)\n",
+ "\n",
+ "db[\"table2\"].head() "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4be81469-5b57-4f78-9c72-3938c0378d9d",
+ "metadata": {},
+ "source": [
+ "Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "25f34bcf-fca0-4431-8601-eac95d1bd347",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "vector: fixed_size_list[2]\n",
+ " child 0, item: float\n",
+ "lat: float\n",
+ "long: float"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pyarrow as pa\n",
+ "\n",
+ "custom_schema = pa.schema([\n",
+ "pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n",
+ "pa.field(\"lat\", pa.float32()),\n",
+ "pa.field(\"long\", pa.float32())\n",
+ "])\n",
+ "\n",
+ "table = db.create_table(\"table3\", data, schema=custom_schema, mode=\"overwrite\")\n",
+ "table.schema"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4df51925-7ca2-4005-9c72-38b3d26240c6",
+ "metadata": {},
+ "source": [
+ "### From PyArrow Tables\n",
+ "\n",
+ "You can also create LanceDB tables directly from pyarrow tables"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "90a880f6-be43-4c9d-ba65-0b05197c0f6f",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "vector: fixed_size_list[2]\n",
+ " child 0, item: float\n",
+ "item: string\n",
+ "price: double"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "table = pa.Table.from_arrays(\n",
+ " [\n",
+ " pa.array([[3.1, 4.1], [5.9, 26.5]],\n",
+ " pa.list_(pa.float32(), 2)),\n",
+ " pa.array([\"foo\", \"bar\"]),\n",
+ " pa.array([10.0, 20.0]),\n",
+ " ],\n",
+ " [\"vector\", \"item\", \"price\"],\n",
+ " )\n",
+ "\n",
+ "db = lancedb.connect(\"db\")\n",
+ "\n",
+ "tbl = db.create_table(\"test1\", table, mode=\"overwrite\")\n",
+ "tbl.schema"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0f36c51c-d902-449d-8292-700e53990c32",
+ "metadata": {},
+ "source": [
+ "### From Pydantic Models\n",
+ "\n",
+ "LanceDB supports to create Apache Arrow Schema from a Pydantic BaseModel."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "d81121d7-e4b7-447c-a48c-974b6ebb464a",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "movie_id: int64 not null\n",
+ "vector: fixed_size_list[128] not null\n",
+ " child 0, item: float\n",
+ "genres: string not null\n",
+ "title: string not null\n",
+ "imdb_id: int64 not null"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from lancedb.pydantic import vector, LanceModel\n",
+ "\n",
+ "class Content(LanceModel):\n",
+ " movie_id: int\n",
+ " vector: vector(128)\n",
+ " genres: str\n",
+ " title: str\n",
+ " imdb_id: int\n",
+ " \n",
+ " @property\n",
+ " def imdb_url(self) -> str:\n",
+ " return f\"https://www.imdb.com/title/tt{self.imdb_id}\"\n",
+ "\n",
+ "import pyarrow as pa\n",
+ "db = lancedb.connect(\"~/.lancedb\")\n",
+ "table_name = \"movielens_small\"\n",
+ "table = db.create_table(table_name, schema=Content)\n",
+ "table.schema"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "860e1f77-e860-46a9-98b7-b2979092ccd6",
+ "metadata": {},
+ "source": [
+ "### Using Iterators / Writing Large Datasets\n",
+ "\n",
+ "It is recommended to use itertators to add large datasets in batches when creating your table in one go. This does not create multiple versions of your dataset unlike manually adding batches using `table.add()`\n",
+ "\n",
+ "LanceDB additionally supports pyarrow's `RecordBatch` Iterators or other generators producing supported data types.\n",
+ "\n",
+ "## Here's an example using using `RecordBatch` iterator for creating tables."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "id": "bc247142-4e3c-41a2-b94c-8e00d2c2a508",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "LanceTable(table4)"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pyarrow as pa\n",
+ "\n",
+ "def make_batches():\n",
+ " for i in range(5):\n",
+ " yield pa.RecordBatch.from_arrays(\n",
+ " [\n",
+ " pa.array([[3.1, 4.1], [5.9, 26.5]],\n",
+ " pa.list_(pa.float32(), 2)),\n",
+ " pa.array([\"foo\", \"bar\"]),\n",
+ " pa.array([10.0, 20.0]),\n",
+ " ],\n",
+ " [\"vector\", \"item\", \"price\"],\n",
+ " )\n",
+ "\n",
+ "schema = pa.schema([\n",
+ " pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n",
+ " pa.field(\"item\", pa.utf8()),\n",
+ " pa.field(\"price\", pa.float32()),\n",
+ "])\n",
+ "\n",
+ "db.create_table(\"table4\", make_batches(), schema=schema)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "94f7dd2b-bae4-4bdf-8534-201437c31027",
+ "metadata": {},
+ "source": [
+ "### Using pandas `DataFrame` Iterator and Pydantic Schema\n",
+ "\n",
+ "You can set the schema via pyarrow schema object or using Pydantic object"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "id": "25ad3523-e0c9-4c28-b3df-38189c4e0e5f",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "vector: fixed_size_list[2] not null\n",
+ " child 0, item: float\n",
+ "item: string not null\n",
+ "price: double not null"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import pyarrow as pa\n",
+ "import pandas as pd\n",
+ "\n",
+ "class PydanticSchema(LanceModel):\n",
+ " vector: vector(2)\n",
+ " item: str\n",
+ " price: float\n",
+ "\n",
+ "def make_batches():\n",
+ " for i in range(5):\n",
+ " yield pd.DataFrame(\n",
+ " {\n",
+ " \"vector\": [[3.1, 4.1], [1, 1]],\n",
+ " \"item\": [\"foo\", \"bar\"],\n",
+ " \"price\": [10.0, 20.0],\n",
+ " })\n",
+ "\n",
+ "tbl = db.create_table(\"table5\", make_batches(), schema=PydanticSchema)\n",
+ "tbl.schema"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4aa955e9-fcd0-4c99-b644-f218f3bb3f1a",
+ "metadata": {},
+ "source": [
+ "## Creating Empty Table\n",
+ "\n",
+ "You can create an empty table by just passing the schema and later add to it using `table.add()`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "id": "2814173a-eacc-4dd8-a64d-6312b44582cc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import lancedb\n",
+ "from lancedb.pydantic import LanceModel, vector\n",
+ "\n",
+ "class Model(LanceModel):\n",
+ " vector: vector(2)\n",
+ "\n",
+ "tbl = db.create_table(\"table6\", schema=Model.to_arrow_schema())"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1d1b0f5c-a1d9-459f-8614-8376b6f577e1",
+ "metadata": {},
+ "source": [
+ "## Open Existing Tables\n",
+ "\n",
+ "If you forget the name of your table, you can always get a listing of all table names:\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "df9e13c0-41f6-437f-9dfa-2fd71d3d9c45",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['table6', 'table4', 'table5', 'movielens_small']"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "db.table_names()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "id": "9343f5ad-6024-42ee-ac2f-6c1471df8679",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " vector | \n",
+ " item | \n",
+ " price | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " [3.1, 4.1] | \n",
+ " foo | \n",
+ " 10.0 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " [5.9, 26.5] | \n",
+ " bar | \n",
+ " 20.0 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " [3.1, 4.1] | \n",
+ " foo | \n",
+ " 10.0 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " [5.9, 26.5] | \n",
+ " bar | \n",
+ " 20.0 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " [3.1, 4.1] | \n",
+ " foo | \n",
+ " 10.0 | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " [5.9, 26.5] | \n",
+ " bar | \n",
+ " 20.0 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " [3.1, 4.1] | \n",
+ " foo | \n",
+ " 10.0 | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " [5.9, 26.5] | \n",
+ " bar | \n",
+ " 20.0 | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " [3.1, 4.1] | \n",
+ " foo | \n",
+ " 10.0 | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " [5.9, 26.5] | \n",
+ " bar | \n",
+ " 20.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " vector | \n",
+ " item | \n",
+ " price | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ "