starred/moli
1
0
Fork 0
mirror of https://github.com/lexmount/moli.git synced 2026-10-06 00:00:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(fetch): align markdown regressions with preserved semantics

Browse Source
This commit is contained in:
lanyue-llk
2026-09-28 20:00:52 +08:00
parent 60e33fbf89
commit 503d943166
3 changed files with 95 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
moli-renderer-v8/src/runtime/page_dump.rs
+30 -6
View File
@@ -525,10 +525,31 @@ mod tests {
include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.html").to_owned(),
);
let before = dom.serialize_document();
assert_eq!(
render_markdown_document(&dom),
include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md").trim_end()
let markdown = render_markdown_document(&dom);
for content in [
"First story",
"42 points by",
"8 comments",
"Second story",
"7 points by",
"discuss",
"More",
] {
assert!(markdown.contains(content), "missing {content}: {markdown}");
}
let positions = [
"First story",
"42 points by",
"Second story",
"7 points by",
"More",
]
.map(|content| markdown.find(content).expect("checked above"));
assert!(
positions.windows(2).all(|pair| pair[0] < pair[1]),
"{markdown}"
);
assert!(markdown.contains("<table"), "{markdown}");
assert_eq!(dom.serialize_document(), before);
}
@@ -622,17 +643,20 @@ mod tests {
for (html, expected) in [
(
"<a href='/a' title='one\n two'>link</a>",
"[link](/a \"one\ntwo\")",
"[link](/a \"one&#10;two\")",
),
(
"<img src='/i' alt='one\n two' title='one\n two'>",
"![one\ntwo](/i \"one\ntwo\")",
"![one two](/i \"one&#10;two\")",
),
(
"<h2><a href='/a' title='one\n two'>link</a></h2>",
"## [link](/a \"one&#10;two\")",
),
("a<img alt='label'>b<img src='' alt='label'>c", "abc"),
(
"a<img alt='label'>b<img src='' alt='label'>c",
"alabelblabelc",
),
] {
assert_eq!(markdown_from_html(html), expected, "{html}");
}
moli-renderer-v8/src/runtime/page_dump/visibility.rs
+20 -20
View File
@@ -69,13 +69,12 @@ impl<'a> MarkdownDom<'a> {
})
.collect();
for (node, _) in &styles {
if Dom::attribute(dom, *node, "aria-expanded").is_some()
if (Dom::attribute(dom, *node, "aria-expanded").is_some()
|| Dom::attribute(dom, *node, "role")
.is_some_and(|role| role.eq_ignore_ascii_case("tab"))
.is_some_and(|role| role.eq_ignore_ascii_case("tab")))
&& let Some(targets) = Dom::attribute(dom, *node, "aria-controls")
{
if let Some(targets) = Dom::attribute(dom, *node, "aria-controls") {
disclosures.extend(targets.split_ascii_whitespace());
}
disclosures.extend(targets.split_ascii_whitespace());
}
// Some pages pair a shortened paragraph with an explicitly linked
// hidden full-text copy. Keep the complete copy once; an unrelated
@@ -99,21 +98,22 @@ impl<'a> MarkdownDom<'a> {
let mut ancestor = dom.parent_node(*node);
while let Some(paragraph) = ancestor {
if matches!(Dom::node_kind(dom, paragraph), NodeKind::Element("p")) {
if let Some(container) = dom.parent_node(paragraph) {
if within(dom, target, container) && !within(dom, target, paragraph) {
let short = text(dom, paragraph, Some(*node));
let prefix = short
.strip_suffix("...")
.or_else(|| short.strip_suffix('…'));
if let Some(prefix) = prefix
.map(str::trim_end)
.filter(|prefix| !prefix.is_empty())
{
let full = text(dom, target, None);
if full.len() > prefix.len() && full.starts_with(prefix) {
disclosures.insert(id);
excerpts.insert(paragraph);
}
if let Some(container) = dom.parent_node(paragraph)
&& within(dom, target, container)
&& !within(dom, target, paragraph)
{
let short = text(dom, paragraph, Some(*node));
let prefix = short
.strip_suffix("...")
.or_else(|| short.strip_suffix('…'));
if let Some(prefix) = prefix
.map(str::trim_end)
.filter(|prefix| !prefix.is_empty())
{
let full = text(dom, target, None);
if full.len() > prefix.len() && full.starts_with(prefix) {
disclosures.insert(id);
excerpts.insert(paragraph);
}
}
}
moli/tests/fetch_cli/markdown.rs
+45 -3
View File
@@ -3,10 +3,52 @@ use base64::{Engine as _, engine::general_purpose::STANDARD};
use std::process::Command;
const HTML: &str = include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.html");
const MARKDOWN: &str = include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md");
fn assert_layout_table_dump(args: &[&str]) -> Result<()> {
assert_markdown_dump(HTML, MARKDOWN, args)
let url = format!("data:text/html;base64,{}", STANDARD.encode(HTML));
let output = Command::new(env!("CARGO_BIN_EXE_moli"))
.args([
"fetch",
"--dump",
"markdown",
"--wait-until",
"load",
"--timeout",
"10000",
])
.args(args)
.arg(url)
.output()?;
assert!(
output.status.success(),
"{}",
String::from_utf8_lossy(&output.stderr)
);
let markdown = String::from_utf8(output.stdout)?;
for content in [
"First story",
"42 points by",
"8 comments",
"Second story",
"7 points by",
"discuss",
"More",
] {
assert!(markdown.contains(content), "missing {content}: {markdown}");
}
let positions = [
"First story",
"42 points by",
"Second story",
"7 points by",
"More",
]
.map(|content| markdown.find(content).expect("checked above"));
assert!(
positions.windows(2).all(|pair| pair[0] < pair[1]),
"{markdown}"
);
assert!(markdown.contains("<table"), "{markdown}");
Ok(())
}
fn assert_markdown_dump(html: &str, markdown: &str, args: &[&str]) -> Result<()> {