From 31fb3b3b5c4b1ac2f539445491ec2b57fc35a0e5 Mon Sep 17 00:00:00 2001
From: qzhu <qian@lancedb.com>
Date: Wed, 13 Nov 2024 21:57:05 -0800
Subject: [PATCH] first edit

---
 docs/mkdocs.yml                      | 25 +++++++----
 docs/src/cloud/best_practices.md     | 20 +++++++++
 docs/src/cloud/build_index.md        | 64 ++++++++++++++++++++++++++++
 docs/src/cloud/full_text_search.md   | 14 ++++++
 docs/src/cloud/hybrid_search.md      | 10 +++++
 docs/src/cloud/ingest_data.md        | 29 +++++++++++++
 docs/src/cloud/metadata_filtering.md | 33 ++++++++++++++
 docs/src/cloud/update_data.md        | 49 +++++++++++++++++++++
 docs/src/cloud/vector_search.md      | 21 +++++++++
 9 files changed, 257 insertions(+), 8 deletions(-)
 create mode 100644 docs/src/cloud/best_practices.md
 create mode 100644 docs/src/cloud/build_index.md
 create mode 100644 docs/src/cloud/full_text_search.md
 create mode 100644 docs/src/cloud/hybrid_search.md
 create mode 100644 docs/src/cloud/ingest_data.md
 create mode 100644 docs/src/cloud/metadata_filtering.md
 create mode 100644 docs/src/cloud/update_data.md
 create mode 100644 docs/src/cloud/vector_search.md

diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 0bb1ebbe..f931052f 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -222,10 +222,12 @@ nav:
           - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
       - ☁️ LanceDB Cloud:
           - Overview: cloud/index.md
-          - API reference:
-              - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/modules.md
-              - REST API: cloud/rest.md
+          - Quickstart: cloud/quickstart.md
+          - Best Practices: cloud/best_practices.md
+          # - API reference:
+          #     - 🐍 Python: python/saas-python.md
+          #     - 👾 JavaScript: javascript/modules.md
+          #     - REST API: cloud/rest.md
 
   - Quick start: basic.md
   - Concepts:
@@ -348,10 +350,17 @@ nav:
       - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
   - LanceDB Cloud:
       - Overview: cloud/index.md
-      - API reference:
-          - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/modules.md
-          - REST API: cloud/rest.md
+      - Quickstart: cloud/quickstart.md
+      - Work with data: 
+        - Ingest data: cloud/ingest_data.md
+        - Update data: cloud/update_data.md
+        - Build an index: cloud/build_index.md
+        - Vector search: cloud/vector_search.md
+        - Full-text search: cloud/full_text_search.md
+        - Hybrid search: cloud/hybrid_search.md
+        - Metadata Filtering: cloud/metadata_filtering.md
+      - Best Practices: cloud/best_practices.md
+      # - REST API: cloud/rest.md
 
 extra_css:
   - styles/global.css
diff --git a/docs/src/cloud/best_practices.md b/docs/src/cloud/best_practices.md
new file mode 100644
index 00000000..e1f08ccc
--- /dev/null
+++ b/docs/src/cloud/best_practices.md
@@ -0,0 +1,20 @@
+This section provides a set of recommended best practices to help you get the most out of LanceDB Cloud. By following these guidelines, you can optimize your usage of LanceDB Cloud, improve performance, and ensure a smooth experience.
+
+### Should the db connection be created once and keep it open?
+Yes! It is recommended to establish a single db connection and maintain it throughout your interaction with the tables within. 
+
+LanceDB uses `requests.Session()` for connection pooling, which automatically manages connection reuse and cleanup. This approach avoids the overhead of repeatedly establishing HTTP connections, significantly improving efficiency.
+
+### Should a single `open_table` call be made and maintained for subsequent table operations?
+`table = db.open_table()` should be called once and used for all subsequent table operations. If there are changes to the opened table, `table` always reflect the latest version of the data. 
+
+### Row id
+
+### What are the vector indexing types supported by LanceDB Cloud?
+We support `IVF_PQ` and `IVF_HNSW_SQ` as the `index_type` which is passed to `create_index`. LanceDB Cloud tunes the indexing parameters automatically to achieve the best tradeoff betweeln query latency and query quality.
+
+### Do I need to do anything when there is new data added to a table with an existing index?
+No! LanceDB Cloud triggers an asynchronous background job to index the new vectors. This process will either merge the new vectors into the existing index or initiate a complete re-indexing if needed. 
+
+There is a flag `fast_search` in `table.search()` that allows you to control whether the unindexed rows should be searched or not.  
+
diff --git a/docs/src/cloud/build_index.md b/docs/src/cloud/build_index.md
new file mode 100644
index 00000000..651a17b8
--- /dev/null
+++ b/docs/src/cloud/build_index.md
@@ -0,0 +1,64 @@
+LanceDB Cloud supports **vector index**, **scalar index** and **full-text search index**. Compared to open-source version, LanceDB Cloud focuses on **automation**:
+
+- If there is a single vector column in the table, the vector column can be inferred from the schema and the index will be automatically created. 
+
+- Indexing parameters will be automatically tuned for customer's data.
+
+## Vector index
+LanceDB has implemented the state-of-art indexing algorithms (more about [IVF-PQ](https://lancedb.github.io/lancedb/concepts/index_ivfpq/) and [HNSW](https://lancedb.github.io/lancedb/concepts/index_hnsw/)). We currently 
+support the _L2_, _Cosine_ and _Dot_ as distance calculation metrics. You can create multiple vector indices within a table.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:create_index"
+    ```
+
+## Scalar index
+LanceDB Cloud and LanceDB Enterprise supports several types of Scalar indices to accelerate search over scalar columns.
+
+- *BTREE*: The most common type is BTREE. This index is inspired by the btree data structure although only the first few layers of the btree are cached in memory. It will perform well on columns with a large number of unique values and few rows per value.
+- *BITMAP*: this index stores a bitmap for each unique value in the column. This index is useful for columns with a finite number of unique values and many rows per value. 
+    - For example, columns that represent "categories", "labels", or "tags"
+- *LABEL_LIST*: a special index that is used to index list columns whose values have a finite set of possibilities. 
+    - For example, a column that contains lists of tags (e.g. ["tag1", "tag2", "tag3"]) can be indexed with a LABEL_LIST index.
+
+You can create multiple scalar indices within a table.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_scalar_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:create_scalar_index"
+    ```
+
+## Full-text search index
+We provide performant full-text search on LanceDB Cloud, allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions.
+!!! note ""
+
+    `use_tantivy` is not available with `create_fts_index` on LanceDB Cloud as we used our native implementation, which has better performance comparing to tantivy.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:create_fts_index"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:create_fts_index"
+    ```
\ No newline at end of file
diff --git a/docs/src/cloud/full_text_search.md b/docs/src/cloud/full_text_search.md
new file mode 100644
index 00000000..7fbff966
--- /dev/null
+++ b/docs/src/cloud/full_text_search.md
@@ -0,0 +1,14 @@
+The full-text search allows you to 
+incorporate keyword-based search (based on BM25) in your retrieval solutions.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:full_text_search"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:full_text_search"
+    ```
\ No newline at end of file
diff --git a/docs/src/cloud/hybrid_search.md b/docs/src/cloud/hybrid_search.md
new file mode 100644
index 00000000..d1737ec7
--- /dev/null
+++ b/docs/src/cloud/hybrid_search.md
@@ -0,0 +1,10 @@
+We support hybrid search that combines semantic and full-text search via a 
+reranking algorithm of your choice, to get the best of both worlds. LanceDB 
+comes with [built-in rerankers](https://lancedb.github.io/lancedb/reranking/) 
+and you can implement you own _customized reranker_ as well. 
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:hybrid_search"
+    ```
\ No newline at end of file
diff --git a/docs/src/cloud/ingest_data.md b/docs/src/cloud/ingest_data.md
new file mode 100644
index 00000000..183af080
--- /dev/null
+++ b/docs/src/cloud/ingest_data.md
@@ -0,0 +1,29 @@
+## Insert data
+The LanceDB Cloud SDK for data ingestion remains consistent with our open-source version, 
+ensuring a seamless transition for existing OSS users. 
+!!! note "unsupported parameters in create_table"
+
+    The following two parameters: `mode="overwrite"` and `exist_ok`, are expected to be added by Nov, 2024.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:ingest_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    --8<-- "nodejs/examples/cloud.test.ts:ingest_data"
+    ```
+
+## Insert large datasets
+It is recommended to use itertators to add large datasets in batches when creating 
+your table in one go. Data will be automatically compacted for the best query performance.
+!!! info "batch size"
+
+    The batch size .
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:ingest_data_in_batch"
+    ```
\ No newline at end of file
diff --git a/docs/src/cloud/metadata_filtering.md b/docs/src/cloud/metadata_filtering.md
new file mode 100644
index 00000000..c0e7650b
--- /dev/null
+++ b/docs/src/cloud/metadata_filtering.md
@@ -0,0 +1,33 @@
+LanceDB Cloud supports rich filtering features of query results based on metadata fields. 
+
+By default, _post-filtering_ is performed on the top-k results returned by the vector search. 
+However, _pre-filtering_ is also an option that performs the filter prior to vector search. 
+This can be useful to narrow down on the search space on a very large dataset to reduce query 
+latency.
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:filtering"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:filtering"
+    ```
+We also support standard SQL expressions as predicates for filtering operations. 
+It can be used during vector search, update, and deletion operations.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:sql_filtering"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:sql_filtering"
+    ```
\ No newline at end of file
diff --git a/docs/src/cloud/update_data.md b/docs/src/cloud/update_data.md
new file mode 100644
index 00000000..c0578e9b
--- /dev/null
+++ b/docs/src/cloud/update_data.md
@@ -0,0 +1,49 @@
+LanceDB Cloud efficiently manages updates across many tables. 
+Currently, we offer _update_, _merge_insert_, and _delete_.
+
+## update
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:update_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:update_data"
+    ```
+
+## merge insert
+This merge insert can add rows, update rows, and remove rows all in a single transaction. 
+It combines new data from a source table with existing data in a target table by using a join.
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:merge_insert"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:merge_insert"
+    ```
+
+## delete
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:delete_data"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+
+    --8<-- "nodejs/examples/cloud.test.ts:connect_db_and_open_table"
+    --8<-- "nodejs/examples/cloud.test.ts:delete_data"
+    ```
\ No newline at end of file
diff --git a/docs/src/cloud/vector_search.md b/docs/src/cloud/vector_search.md
new file mode 100644
index 00000000..0a10faa7
--- /dev/null
+++ b/docs/src/cloud/vector_search.md
@@ -0,0 +1,21 @@
+Users can also tune the following parameters for better search quality.
+
+ - [nprobes](https://lancedb.github.io/lancedb/js/classes/VectorQuery/#nprobes): 
+ the number of partitions to search (probe).
+ - [refine factor](https://lancedb.github.io/lancedb/js/classes/VectorQuery/#refinefactor): 
+ a multiplier to control how many additional rows are taken during the refine step.
+
+ [Metadata filtering](filtering) combined with the vector search is also supported.
+
+=== "Python"
+
+    ```python
+    --8<-- "python/python/tests/docs/test_cloud.py:vector_search"
+    ```
+=== "Typescript"
+
+    ```typescript 
+    --8<-- "nodejs/examples/cloud.test.ts:imports"
+    
+    --8<-- "nodejs/examples/cloud.test.ts:vector_search"
+    ```
\ No newline at end of file