diff --git a/docs/web/demos/fineweb-edu-search.mdx b/docs/web/demos/fineweb-edu-search.mdx new file mode 100644 index 000000000..64d4decd4 --- /dev/null +++ b/docs/web/demos/fineweb-edu-search.mdx @@ -0,0 +1,20 @@ +--- +title: "Large scale vector search (1.5B)" +sidebarTitle: "Large Scale Vector Search" +description: "Semantic search over 1.5B+ FineWeb EDU embeddings." +icon: database +--- + +vector-search +education +bge-embeddings + + +Semantic search over curated FineWeb EDU pages, represented as 1.5B+ BGE embeddings. Inspect the +query plan to see how LanceDB serves vector search at billion scale. + + diff --git a/docs/web/demos/index.mdx b/docs/web/demos/index.mdx index a6a3e284a..fb1bdc49a 100644 --- a/docs/web/demos/index.mdx +++ b/docs/web/demos/index.mdx @@ -13,3 +13,5 @@ works. | [Semantic.art](/demos/semantic-art) | Multimodal art discovery using feelings, phrases, and images. | | [Wikipedia 41M hybrid search](/demos/wikipedia-search) | Full-text and vector search combined, over 41M+ entries. | | [Video search](/demos/video-search) | Natural-language search across a video library. | +| [Large scale vector search (1.5B)](/demos/fineweb-edu-search) | Semantic search over 1.5B+ FineWeb EDU embeddings. | +| [Robot data curation (DROID)](/demos/robot-data-curation) | Searching, scoring, and selecting robotics training examples. | diff --git a/docs/web/demos/robot-data-curation.mdx b/docs/web/demos/robot-data-curation.mdx new file mode 100644 index 000000000..b61f53afe --- /dev/null +++ b/docs/web/demos/robot-data-curation.mdx @@ -0,0 +1,21 @@ +--- +title: "Robot data curation (DROID)" +sidebarTitle: "Robot Data Curation" +description: "Search, score, and select robotics training examples from DROID." +icon: robot +--- + +multimodal +robotics +vector-search +fts + + +Search robot video frames from the DROID dataset, compute features, and select training examples in LanceDB. +Add captions and quality scores as columns, then write model errors back to guide the next training run. + + diff --git a/docs/web/docs.json b/docs/web/docs.json index ced9671a8..0f5b23a9f 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -545,7 +545,9 @@ "demos/index", "demos/semantic-art", "demos/wikipedia-search", - "demos/video-search" + "demos/video-search", + "demos/fineweb-edu-search", + "demos/robot-data-curation" ] } ] diff --git a/docs/web/static/assets/images/demos/droid_hero.jpg b/docs/web/static/assets/images/demos/droid_hero.jpg new file mode 100644 index 000000000..14c64b60b Binary files /dev/null and b/docs/web/static/assets/images/demos/droid_hero.jpg differ diff --git a/docs/web/static/assets/images/demos/fineweb_edu_hero.svg b/docs/web/static/assets/images/demos/fineweb_edu_hero.svg new file mode 100644 index 000000000..092b6051a --- /dev/null +++ b/docs/web/static/assets/images/demos/fineweb_edu_hero.svg @@ -0,0 +1,61 @@ + + Large scale vector search (1.5B) + Abstract visualization of billions of vectors clustered and searched + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1.5B vectors + Large scale LanceDB search + +