From 503d943166d6ca4737943556b112e0dcd83458af Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 25 Sep 2026 00:20:12 +0800 Subject: [PATCH] test(fetch): align markdown regressions with preserved semantics --- moli-renderer-v8/src/runtime/page_dump.rs | 36 +++++++++++--- .../src/runtime/page_dump/visibility.rs | 40 ++++++++-------- moli/tests/fetch_cli/markdown.rs | 48 +++++++++++++++++-- 3 files changed, 95 insertions(+), 29 deletions(-) diff --git a/moli-renderer-v8/src/runtime/page_dump.rs b/moli-renderer-v8/src/runtime/page_dump.rs index 772a0f7070..c23ddf3729 100644 --- a/moli-renderer-v8/src/runtime/page_dump.rs +++ b/moli-renderer-v8/src/runtime/page_dump.rs @@ -525,10 +525,31 @@ mod tests { include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.html").to_owned(), ); let before = dom.serialize_document(); - assert_eq!( - render_markdown_document(&dom), - include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md").trim_end() + let markdown = render_markdown_document(&dom); + for content in [ + "First story", + "42 points by", + "8 comments", + "Second story", + "7 points by", + "discuss", + "More", + ] { + assert!(markdown.contains(content), "missing {content}: {markdown}"); + } + let positions = [ + "First story", + "42 points by", + "Second story", + "7 points by", + "More", + ] + .map(|content| markdown.find(content).expect("checked above")); + assert!( + positions.windows(2).all(|pair| pair[0] < pair[1]), + "{markdown}" ); + assert!(markdown.contains("link", - "[link](/a \"one\ntwo\")", + "[link](/a \"one two\")", ), ( "one\n two", - "![one\ntwo](/i \"one\ntwo\")", + "![one two](/i \"one two\")", ), ( "

link

", "## [link](/a \"one two\")", ), - ("alabelblabelc", "abc"), + ( + "alabelblabelc", + "alabelblabelc", + ), ] { assert_eq!(markdown_from_html(html), expected, "{html}"); } diff --git a/moli-renderer-v8/src/runtime/page_dump/visibility.rs b/moli-renderer-v8/src/runtime/page_dump/visibility.rs index 2a35bcec22..b110094b1b 100644 --- a/moli-renderer-v8/src/runtime/page_dump/visibility.rs +++ b/moli-renderer-v8/src/runtime/page_dump/visibility.rs @@ -69,13 +69,12 @@ impl<'a> MarkdownDom<'a> { }) .collect(); for (node, _) in &styles { - if Dom::attribute(dom, *node, "aria-expanded").is_some() + if (Dom::attribute(dom, *node, "aria-expanded").is_some() || Dom::attribute(dom, *node, "role") - .is_some_and(|role| role.eq_ignore_ascii_case("tab")) + .is_some_and(|role| role.eq_ignore_ascii_case("tab"))) + && let Some(targets) = Dom::attribute(dom, *node, "aria-controls") { - if let Some(targets) = Dom::attribute(dom, *node, "aria-controls") { - disclosures.extend(targets.split_ascii_whitespace()); - } + disclosures.extend(targets.split_ascii_whitespace()); } // Some pages pair a shortened paragraph with an explicitly linked // hidden full-text copy. Keep the complete copy once; an unrelated @@ -99,21 +98,22 @@ impl<'a> MarkdownDom<'a> { let mut ancestor = dom.parent_node(*node); while let Some(paragraph) = ancestor { if matches!(Dom::node_kind(dom, paragraph), NodeKind::Element("p")) { - if let Some(container) = dom.parent_node(paragraph) { - if within(dom, target, container) && !within(dom, target, paragraph) { - let short = text(dom, paragraph, Some(*node)); - let prefix = short - .strip_suffix("...") - .or_else(|| short.strip_suffix('…')); - if let Some(prefix) = prefix - .map(str::trim_end) - .filter(|prefix| !prefix.is_empty()) - { - let full = text(dom, target, None); - if full.len() > prefix.len() && full.starts_with(prefix) { - disclosures.insert(id); - excerpts.insert(paragraph); - } + if let Some(container) = dom.parent_node(paragraph) + && within(dom, target, container) + && !within(dom, target, paragraph) + { + let short = text(dom, paragraph, Some(*node)); + let prefix = short + .strip_suffix("...") + .or_else(|| short.strip_suffix('…')); + if let Some(prefix) = prefix + .map(str::trim_end) + .filter(|prefix| !prefix.is_empty()) + { + let full = text(dom, target, None); + if full.len() > prefix.len() && full.starts_with(prefix) { + disclosures.insert(id); + excerpts.insert(paragraph); } } } diff --git a/moli/tests/fetch_cli/markdown.rs b/moli/tests/fetch_cli/markdown.rs index 36711ea604..917047c302 100644 --- a/moli/tests/fetch_cli/markdown.rs +++ b/moli/tests/fetch_cli/markdown.rs @@ -3,10 +3,52 @@ use base64::{Engine as _, engine::general_purpose::STANDARD}; use std::process::Command; const HTML: &str = include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.html"); -const MARKDOWN: &str = include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md"); - fn assert_layout_table_dump(args: &[&str]) -> Result<()> { - assert_markdown_dump(HTML, MARKDOWN, args) + let url = format!("data:text/html;base64,{}", STANDARD.encode(HTML)); + let output = Command::new(env!("CARGO_BIN_EXE_moli")) + .args([ + "fetch", + "--dump", + "markdown", + "--wait-until", + "load", + "--timeout", + "10000", + ]) + .args(args) + .arg(url) + .output()?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let markdown = String::from_utf8(output.stdout)?; + for content in [ + "First story", + "42 points by", + "8 comments", + "Second story", + "7 points by", + "discuss", + "More", + ] { + assert!(markdown.contains(content), "missing {content}: {markdown}"); + } + let positions = [ + "First story", + "42 points by", + "Second story", + "7 points by", + "More", + ] + .map(|content| markdown.find(content).expect("checked above")); + assert!( + positions.windows(2).all(|pair| pair[0] < pair[1]), + "{markdown}" + ); + assert!(markdown.contains(" Result<()> {