mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-03 18:32:55 +00:00
Add docs code testing & documentation syntax changes (#196)
- Creates testing files `md_testing.py` and `md_testing.js` for testing python and nodejs code in markdown files in the documentation This listens for HTML tags as well: `<!--[language] code code code...-->` will create a set-up file to create some mock tables or to fulfill some assumptions in the documentation. - Creates a github action workflow that triggers every push/pr to `docs/**` - Modifies documentation so tests run (mostly indentation, some small syntax errors and some missing imports) A list of excluded files that we need to take a closer look at later on: ```javascript const excludedFiles = [ "../src/fts.md", "../src/embedding.md", "../src/examples/serverless_lancedb_with_s3_and_lambda.md", "../src/examples/serverless_qa_bot_with_modal_and_langchain.md", "../src/examples/youtube_transcript_bot_with_nodejs.md", ]; ``` Many of them can't be done because we need the OpenAI API key :(. `fts.md` has some issues with the library, I believe this is still experimental? Closes #170 --------- Co-authored-by: Will Jones <willjones127@gmail.com>
This commit is contained in:
51
docs/test/md_testing.js
Normal file
51
docs/test/md_testing.js
Normal file
@@ -0,0 +1,51 @@
|
||||
const glob = require("glob");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const excludedFiles = [
|
||||
"../src/fts.md",
|
||||
"../src/embedding.md",
|
||||
"../src/examples/serverless_lancedb_with_s3_and_lambda.md",
|
||||
"../src/examples/serverless_qa_bot_with_modal_and_langchain.md",
|
||||
"../src/examples/youtube_transcript_bot_with_nodejs.md",
|
||||
];
|
||||
const nodePrefix = "javascript";
|
||||
const nodeFile = ".js";
|
||||
const nodeFolder = "node";
|
||||
const globString = "../src/**/*.md";
|
||||
const asyncPrefix = "(async () => {\n";
|
||||
const asyncSuffix = "})();";
|
||||
|
||||
function* yieldLines(lines, prefix, suffix) {
|
||||
let inCodeBlock = false;
|
||||
for (const line of lines) {
|
||||
if (line.trim().startsWith(prefix + nodePrefix)) {
|
||||
inCodeBlock = true;
|
||||
} else if (inCodeBlock && line.trim().startsWith(suffix)) {
|
||||
inCodeBlock = false;
|
||||
yield "\n";
|
||||
} else if (inCodeBlock) {
|
||||
yield line;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const files = glob.sync(globString, { recursive: true });
|
||||
|
||||
for (const file of files.filter((file) => !excludedFiles.includes(file))) {
|
||||
const lines = [];
|
||||
const data = fs.readFileSync(file, "utf-8");
|
||||
const fileLines = data.split("\n");
|
||||
|
||||
for (const line of yieldLines(fileLines, "```", "```")) {
|
||||
lines.push(line);
|
||||
}
|
||||
|
||||
if (lines.length > 0) {
|
||||
const fileName = path.basename(file, ".md");
|
||||
const outPath = path.join(nodeFolder, fileName, `${fileName}${nodeFile}`);
|
||||
console.log(outPath)
|
||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||
fs.writeFileSync(outPath, asyncPrefix + "\n" + lines.join("\n") + asyncSuffix);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user