mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-08 04:42:57 +00:00
feat(python): Embedding fn support for gte-mlx/gte-large (#873)
have added testing and an example in the docstring, will be pushing a separate PR in recipe repo for rag example --------- Co-authored-by: Ayush Chaurasia <ayush.chaurarsia@gmail.com>
This commit is contained in:
@@ -79,7 +79,10 @@ def qanda_langchain(query):
|
||||
download_docs()
|
||||
docs = store_docs()
|
||||
|
||||
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200,)
|
||||
text_splitter = RecursiveCharacterTextSplitter(
|
||||
chunk_size=1000,
|
||||
chunk_overlap=200,
|
||||
)
|
||||
documents = text_splitter.split_documents(docs)
|
||||
embeddings = OpenAIEmbeddings()
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
|
||||
if not skip_test:
|
||||
yield line[strip_length:]
|
||||
|
||||
|
||||
for file in filter(lambda file: file not in excluded_files, files):
|
||||
with open(file, "r") as f:
|
||||
lines = list(yield_lines(iter(f), "```", "```"))
|
||||
|
||||
Reference in New Issue
Block a user