![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
\n",
+ "\n",
"\n",
"The code below is an example of hybrid search, a search algorithm that combines FTS and vector search in LanceDB.\n",
"\n",
@@ -18,24 +20,36 @@
"cell_type": "code",
"execution_count": null,
"id": "819fa612",
- "metadata": {},
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "819fa612",
+ "outputId": "f5593c76-573f-4a04-d0ce-aac7b7ee1466"
+ },
"outputs": [],
"source": [
"# Setup\n",
- "!pip install lancedb pandas langchain langchain-community pypdf openai cohere tiktoken sentence_transformers tantivy==0.20.1"
+ "!pip install lancedb pandas langchain langchain_openai langchain-community pypdf openai cohere tiktoken sentence_transformers tantivy==0.20.1"
]
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"id": "b6864d97-7f85-4d9c-bf05-e9cf9db29e81",
- "metadata": {},
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "b6864d97-7f85-4d9c-bf05-e9cf9db29e81",
+ "outputId": "6c6dd78d-3213-4bd8-9e74-5faba902e546"
+ },
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- " ········\n"
+ "··········\n"
]
}
],
@@ -49,24 +63,28 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 3,
"id": "cfce9804-cd1c-48c3-acd2-e74eb4e290c7",
- "metadata": {},
+ "metadata": {
+ "id": "cfce9804-cd1c-48c3-acd2-e74eb4e290c7"
+ },
"outputs": [],
"source": [
"def pretty_print(docs):\n",
" for doc in docs:\n",
- " print(doc + \"\\n\\n\") "
+ " print(doc + \"\\n\\n\")"
]
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"id": "efb22cec-5a06-46ac-91c3-53f9b9090109",
- "metadata": {},
+ "metadata": {
+ "id": "efb22cec-5a06-46ac-91c3-53f9b9090109"
+ },
"outputs": [],
"source": [
- "from langchain.document_loaders import PyPDFLoader\n",
+ "from langchain_community.document_loaders import PyPDFLoader\n",
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"\n",
"# Load $ABNB's financial report. This may take 1-2 minutes since the PDF is large\n",
@@ -85,64 +103,60 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": null,
"id": "d3c5ce69-0f75-44cb-9e49-9be665fc156e",
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "[2024-02-12T20:00:04Z WARN lance::dataset] No existing dataset at /Users/ayushchaurasia/langchain/airbnb.lance, it will be created\n"
- ]
- }
- ],
+ "metadata": {
+ "id": "d3c5ce69-0f75-44cb-9e49-9be665fc156e"
+ },
+ "outputs": [],
"source": [
"from langchain_community.vectorstores import LanceDB\n",
- "from langchain.embeddings.openai import OpenAIEmbeddings\n",
+ "from langchain_openai import OpenAIEmbeddings\n",
"import lancedb\n",
- "from lancedb.embeddings import get_registry\n",
- "from lancedb.pydantic import Vector, LanceModel\n",
"\n",
- "openai = get_registry().get(\"openai\").create()\n",
- "\n",
- "class Schema(LanceModel):\n",
- " text: str = openai.SourceField()\n",
- " vector: Vector(openai.ndims()) = openai.VectorField()\n",
"\n",
"embedding_function = OpenAIEmbeddings()\n",
"\n",
"db = lancedb.connect(\"~/langchain\")\n",
- "table = db.create_table(\n",
- " \"airbnb\",\n",
- " schema=Schema,\n",
- " mode=\"overwrite\",\n",
- ")\n",
"\n",
"# Load the document into LanceDB\n",
- "db = LanceDB.from_documents(docs, embedding_function, connection=table)"
+ "db = LanceDB.from_documents(docs, embedding_function, connection=db)"
]
},
{
"cell_type": "code",
- "execution_count": 11,
- "id": "4284e67e-3a39-4486-a060-11a18f7c0e1f",
+ "execution_count": null,
+ "id": "4bd12fb8",
"metadata": {},
"outputs": [],
"source": [
+ "table = db._table\n",
"table.create_fts_index(\"text\")"
]
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 19,
"id": "d959a80f-d568-48f4-9d14-7367bcc1ce8d",
- "metadata": {},
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 293
+ },
+ "id": "d959a80f-d568-48f4-9d14-7367bcc1ce8d",
+ "outputId": "b02f837d-7fa6-4ec2-b283-e170f5f67637"
+ },
"outputs": [
{
"data": {
+ "application/vnd.google.colaboratory.intrinsic+json": {
+ "summary": "{\n \"name\": \"table\",\n \"rows\": 5,\n \"fields\": [\n {\n \"column\": \"vector\",\n \"properties\": {\n \"dtype\": \"object\",\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"id\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 5,\n \"samples\": [\n \"ddcfa6b1-3de8-4933-a187-6aa7b7ae87b4\",\n \"47f5dd55-b3e7-4879-afba-5ca9eea7341b\",\n \"c391b1e1-6f66-41f2-82ff-18db5a218303\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"text\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 5,\n \"samples\": [\n \"Class A common stock, par value $0.0001 per share ABNB The Nasdaq Stock Market\\nSecurities registered pursuant to Section 12(g) of the Act:\\nNone______________\\nIndicate by check mark if the registrant is a well-known seasoned issuer, as defined in Rule 405 of the Securities Act. Yes \\u2612 No \\u2610 \\nIndicate by check mark if the registrant is not required to file reports pursuant to Section 13 or Section 15(d) of the Exchange Act. Yes \\u2610 No \\u2612 \\nIndicate by check mark whether the registrant (1) has filed all reports required to be filed by Section 13 or 15(d) of the Securities Exchange Act of 1934 during the preceding 12\\nmonths (or for such shorter period that the registrant was required to file such reports), and (2) has been subject to such filing requirements for the past 90 days. Yes \\u2612 No \\u2610 \\nIndicate by check mark whether the registrant has submitted electronically every Interactive Data File required to be submitted pursuant to Rule 405 of Regulation S-T (\\u00a7 232.405 of\",\n \"As of June 30, 2022, the aggregate market value of the Class A common stock held by non-affiliates of the registrant was approximately $35.1 billion based upon the closing price\\nreported for such date on the NASDAQ Global Select Market.\\nAs of February 3, 2023, 408,928,427 shares of the registrant's Class A common stock were outstanding 222,400,067 shares of the registrant's Class B common stock were\\noutstanding, no shares of the registrant\\u2019s Class C common stock were outstanding, and 9,200,000 shares of the registrant\\u2019s Class H common stock were outstanding.\\n______________\\nDOCUMENTS INCORPORATED BY REFERENCE\\nThe information required by Part III of this Report, to the extent not set forth herein, is incorporated herein by reference from the registrant\\u2019s definitive proxy statement relating to the\",\n \"this chapter) during the preceding 12 months (or for such shorter period that the registrant was required to submit such files). Yes \\u2612 No \\u2610 \\nIndicate by check mark whether the registrant is a large accelerated filer, an accelerated filer, a non-accelerated filer, a smaller reporting company, or an emerging growth company.\\nSee the definitions of \\u201clarge accelerated filer,\\u201d \\u201caccelerated filer,\\u201d \\u201csmaller reporting company\\u201d and \\u201cemerging growth company\\u201d in Rule 12b-2 of the Exchange Act.\\nLarge accelerated filer \\u2612 Accelerated filer\\u2610 \\nNon-accelerated filer \\u2610 Smaller reporting company\\u2610 \\nEmerging growth company\\u2610 \\nIf an emerging growth company, indicate by check mark if the registrant has elected not to use the extended transition period for complying with any new or revised financial\\naccounting standards provided pursuant to Section 13(a) of the Exchange Act. \\u2610\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"metadata\",\n \"properties\": {\n \"dtype\": \"object\",\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}",
+ "type": "dataframe"
+ },
"text/html": [
- "