mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-26 06:39:57 +00:00
3ffed89793
based on https://github.com/lancedb/lancedb/pull/713 - The Reranker api can be plugged into vector only or fts only search but this PR doesn't do that (see example - https://txt.cohere.com/rerank/) ### Default reranker -- `LinearCombinationReranker(weight=0.7, fill=1.0)` ``` table.search("hello", query_type="hybrid").rerank(normalize="score").to_pandas() ``` ### Available rerankers LinearCombinationReranker ``` from lancedb.rerankers import LinearCombinationReranker # Same as default table.search("hello", query_type="hybrid").rerank( normalize="score", reranker=LinearCombinationReranker() ).to_pandas() # with custom params reranker = LinearCombinationReranker(weight=0.3, fill=1.0) table.search("hello", query_type="hybrid").rerank( normalize="score", reranker=reranker ).to_pandas() ``` Cohere Reranker ``` from lancedb.rerankers import CohereReranker # default model.. English and multi-lingual supported. See docstring for available custom params table.search("hello", query_type="hybrid").rerank( normalize="rank", # score or rank reranker=CohereReranker() ).to_pandas() ``` CrossEncoderReranker ``` from lancedb.rerankers import CrossEncoderReranker table.search("hello", query_type="hybrid").rerank( normalize="rank", reranker=CrossEncoderReranker() ).to_pandas() ``` ## Using custom Reranker ``` from lancedb.reranker import Reranker class CustomReranker(Reranker): def rerank_hybrid(self, vector_result, fts_result): combined_res = self.merge_results(vector_results, fts_results) # or use custom combination logic # Custom rerank logic here return combined_res ``` - [x] Expand testing - [x] Make sure usage makes sense - [x] Run simple benchmarks for correctness (Seeing weird result from cohere reranker in the toy example) - Support diverse rerankers by default: - [x] Cross encoding - [x] Cohere - [x] Reciprocal Rank Fusion --------- Co-authored-by: Chang She <759245+changhiskhan@users.noreply.github.com> Co-authored-by: Prashanth Rao <35005448+prrao87@users.noreply.github.com>
68 lines
1.8 KiB
Python
68 lines
1.8 KiB
Python
import glob
|
|
from typing import Iterator
|
|
from pathlib import Path
|
|
|
|
glob_string = "../src/**/*.md"
|
|
excluded_globs = [
|
|
"../src/fts.md",
|
|
"../src/embedding.md",
|
|
"../src/examples/*.md",
|
|
"../src/integrations/voxel51.md",
|
|
"../src/guides/tables.md",
|
|
"../src/python/duckdb.md",
|
|
"../src/embeddings/*.md",
|
|
"../src/concepts/*.md",
|
|
"../src/ann_indexes.md",
|
|
"../src/basic.md",
|
|
"../src/hybrid_search.md",
|
|
]
|
|
|
|
python_prefix = "py"
|
|
python_file = ".py"
|
|
python_folder = "python"
|
|
|
|
files = glob.glob(glob_string, recursive=True)
|
|
excluded_files = [
|
|
f
|
|
for excluded_glob in excluded_globs
|
|
for f in glob.glob(excluded_glob, recursive=True)
|
|
]
|
|
|
|
|
|
def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
|
|
in_code_block = False
|
|
# Python code has strict indentation
|
|
strip_length = 0
|
|
skip_test = False
|
|
for line in lines:
|
|
if "skip-test" in line:
|
|
skip_test = True
|
|
if line.strip().startswith(prefix + python_prefix):
|
|
in_code_block = True
|
|
strip_length = len(line) - len(line.lstrip())
|
|
elif in_code_block and line.strip().startswith(suffix):
|
|
in_code_block = False
|
|
if not skip_test:
|
|
yield "\n"
|
|
skip_test = False
|
|
elif in_code_block:
|
|
if not skip_test:
|
|
yield line[strip_length:]
|
|
|
|
|
|
for file in filter(lambda file: file not in excluded_files, files):
|
|
with open(file, "r") as f:
|
|
lines = list(yield_lines(iter(f), "```", "```"))
|
|
|
|
if len(lines) > 0:
|
|
print(lines)
|
|
out_path = (
|
|
Path(python_folder)
|
|
/ Path(file).name.strip(".md")
|
|
/ (Path(file).name.strip(".md") + python_file)
|
|
)
|
|
print(out_path)
|
|
out_path.parent.mkdir(exist_ok=True, parents=True)
|
|
with open(out_path, "w") as out:
|
|
out.writelines(lines)
|