From 413bf13007814fa3ada6360ffd85a68ddcc83ce0 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:05:04 +0800 Subject: [PATCH] fix(html2md): share structural boundaries and preserve fallback targets --- moli-html2md/README.md | 4 +- moli-html2md/src/content.rs | 5 ++ moli-html2md/src/html_fallback.rs | 72 +++++++++++++++-------- moli-html2md/src/machine.rs | 35 +++++------ moli-html2md/src/media.rs | 22 ++++++- moli-html2md/tests/review_semantics.rs | 48 +++++++++++++++ moli-renderer-v8/src/runtime/page_dump.rs | 27 ++------- moli/tests/fetch_cli/markdown.rs | 4 +- 8 files changed, 146 insertions(+), 71 deletions(-) diff --git a/moli-html2md/README.md b/moli-html2md/README.md index e6a4cc28e3..08a8921651 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -71,13 +71,13 @@ missing trailing cells in short rows; the converter does not allocate a padded g Tables with `role="presentation"` or `role="none"` retain ordinary block expansion, in DOM order with blank lines between cells, even when structurally simple. Role matching ignores ASCII case and surrounding HTML whitespace. Nested layout -wrappers expand into readable blocks. Tables with headings, captions, explicit +wrappers expand into readable blocks. Tables with header cells, captions, explicit data roles, header relationships or real cell spans retain data-table structure. Complex data tables use content-only HTML: non-unit spans, multiple `thead` rows, rows wider than an explicit header, or cells containing nested tables, lists, headings, quotes or preformatted blocks. This retains header, cell and nested-table associations that Markdown pipe tables cannot express. Inner simple tables can still produce -GFM when an explicitly presentational containing table is expanded. Scripts, +GFM when a presentational or layout-only containing table is expanded. Scripts, styles and event handlers are omitted from HTML fallback output. Supported output includes headings, paragraphs, emphasis, strikethrough, links, diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index 5ca069222a..12ca8fa6de 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -135,3 +135,8 @@ pub(crate) fn class_language(class: Option<&str>) -> Option<&str> { .split_ascii_whitespace() .find_map(|part| part.strip_prefix("language-").filter(|s| !s.is_empty())) } + +/// These elements already own their document/list/choice boundaries. +pub(crate) fn owns_boundaries(tag: &str) -> bool { + crate::machine::is_structural_block(tag) || matches!(tag, "select" | "optgroup" | "option") +} diff --git a/moli-html2md/src/html_fallback.rs b/moli-html2md/src/html_fallback.rs index cd68825386..21a834c28b 100644 --- a/moli-html2md/src/html_fallback.rs +++ b/moli-html2md/src/html_fallback.rs @@ -4,8 +4,8 @@ use std::fmt::Write as _; use crate::{Dom, NodeKind}; /// Markdown has no merged cells, nested tables, or block content inside cells. -/// Detect those structures before conversion so a table is traversed at most -/// twice, even when it contains thousands of nested tables. +/// Each structural pre-scan stops at nested tables; nested scopes are disjoint. +/// Conversion and target discovery keep their own bounded linear traversals. pub(crate) fn needed( dom: &D, root: D::NodeId, @@ -138,8 +138,9 @@ enum Task { Literal(&'static str), } -/// HTML fallback preserves structural content and referenced navigation targets. Keep content markup only; scripts, styles, event handlers -/// and unrelated presentation attributes are never copied. +/// HTML fallback preserves structural content and referenced navigation targets. +/// Keep content markup only; scripts, styles, event handlers and unrelated +/// presentation attributes are never copied. pub(crate) fn render( dom: &D, root: D::NodeId, @@ -147,11 +148,11 @@ pub(crate) fn render( limit: usize, anchor_targets: &crate::anchors::Targets, default_language: Option<&str>, + emitted_anchors: &mut HashSet, ) -> String { let inline_root = matches!(dom.node_kind(root), NodeKind::Element("code")); let keep_source_lines = matches!(dom.node_kind(root), NodeKind::Element("pre")); let mut output = String::new(); - let mut emitted_anchors = HashSet::new(); let mut tasks = vec![Task::Node(root, depth, false)]; while let Some(task) = tasks.pop() { match task { @@ -188,10 +189,13 @@ pub(crate) fn render( } let code = code || matches!(dom.node_kind(node), NodeKind::Element("pre" | "code")); if let NodeKind::Element(tag) = dom.node_kind(node) { - if dom.has_block_layout(node) && !has_block_tag(tag) { + if dom.has_block_layout(node) + && !has_block_tag(tag) + && !crate::content::owns_boundaries(tag) + { output.push_str("
"); tasks.push(Task::Close("div".to_owned())); - } else if dom.has_text_boundary(node) { + } else if dom.has_text_boundary(node) && !crate::content::owns_boundaries(tag) { output.push(' '); tasks.push(Task::Literal(" ")); } @@ -243,25 +247,42 @@ pub(crate) fn render( } if tag == "img" { let label = crate::content::image_label(dom, node); + let targets: Vec<_> = anchor_targets + .targets(dom, node) + .into_iter() + .filter(|id| emitted_anchors.insert((*id).to_owned())) + .collect(); if let Some(src) = crate::media::source(dom, node) { + for id in targets.iter().skip(1) { + output.push_str(&crate::anchors::markup(id)); + } output.push_str("\"");"); + output.push('"'); + if let Some(id) = targets.first() { + output.push_str(" id=\""); + escape(id, &mut output); + output.push('"'); + } + if let Some(title) = dom.attribute(node, "title") { + output.push_str(" title=\""); + escape(title, &mut output); + output.push('"'); + } + output.push('>'); } else { + for id in targets { + output.push_str(&crate::anchors::markup(id)); + } escape(&label, &mut output); } continue; } if matches!(tag, "iframe" | "video" | "audio") { let urls = crate::media::sources(dom, node, tag, depth, limit); - let poster = (tag == "video") - .then(|| dom.attribute(node, "poster")) - .flatten() - .filter(|src| { - !src.trim().is_empty() && crate::media::safe_url(src, true) - }); + let poster = crate::media::poster(dom, node, tag); if !urls.is_empty() || poster.is_some() { let block = tag == "audio"; output.push_str(if block { "
" } else { " " }); @@ -272,6 +293,7 @@ pub(crate) fn render( escape(&crate::media::poster_label(dom, node), &mut output); output.push_str("\"> "); } + let has_source = !urls.is_empty(); for url in urls { output.push_str("( if block { output.push_str("
"); } - continue; + if has_source { + continue; + } } } let retained = if allowed(tag) && (tag != "span" - || dom.attribute(node, "id").is_some() + || !anchor_targets.targets(dom, node).is_empty() || dom.attribute(node, "title").is_some()) { Some(tag) @@ -308,6 +332,14 @@ pub(crate) fn render( ) { begin_html_line(&mut output); } + let targets: Vec<_> = anchor_targets + .targets(dom, node) + .into_iter() + .filter(|id| emitted_anchors.insert((*id).to_owned())) + .collect(); + for id in targets.iter().skip(1) { + output.push_str(&crate::anchors::markup(id)); + } output.push('<'); output.push_str(tag); for name in [ @@ -338,11 +370,6 @@ pub(crate) fn render( output.push('"'); } } - let targets: Vec<_> = anchor_targets - .targets(dom, node) - .into_iter() - .filter(|id| emitted_anchors.insert((*id).to_owned())) - .collect(); if let Some(id) = targets.first() { output.push_str(" id=\""); escape(id, &mut output); @@ -368,9 +395,6 @@ pub(crate) fn render( if tag == "pre" { output.push('\n'); } - for id in targets.iter().skip(1) { - output.push_str(&crate::anchors::markup(id)); - } if !matches!(tag, "br" | "hr" | "img") { tasks.push(Task::Close(tag.to_owned())); } diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index ac37853c8a..e9758d1a73 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -11,6 +11,7 @@ enum Task<'a, Id> { MathChildren(Option, usize), Boundary, ChoiceBoundary, + InlineSpace, Fallback(String, usize), PopStyle, EndLink(usize), @@ -88,6 +89,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { | Task::MathChildren(None, _) | Task::RawChildren(None, _, _) => {} Task::Boundary => self.writer().boundary(2), + Task::InlineSpace => self.writer().text(" "), Task::ChoiceBoundary => self.writer().boundary(1), Task::Fallback(label, before) => { if self.writer().content_mark() == before { @@ -223,12 +225,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if depth >= self.options.max_depth { return; } - if self.dom.has_block_layout(node) { + let owns_boundaries = matches!(self.dom.node_kind(node), NodeKind::Element(tag) if crate::content::owns_boundaries(tag)); + if !owns_boundaries && self.dom.has_block_layout(node) { self.writer().boundary(2); self.tasks.push(Task::Boundary); - } else if self.dom.has_text_boundary(node) { - self.writer().boundary(1); - self.tasks.push(Task::ChoiceBoundary); + } else if !owns_boundaries && self.dom.has_text_boundary(node) { + self.writer().text(" "); + self.tasks.push(Task::InlineSpace); } let tag = match self.dom.node_kind(node) { NodeKind::Document => { @@ -375,6 +378,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.options.max_depth, &self.anchor_targets, self.options.default_code_language.as_deref(), + &mut self.emitted_anchors, ); if tag == "pre" { self.writer().block(html.into(), 2, 2); @@ -416,6 +420,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.options.max_depth, &self.anchor_targets, self.options.default_code_language.as_deref(), + &mut self.emitted_anchors, ); self.writer().block(html.into(), 2, 2); return; @@ -447,8 +452,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { ); let complex_table = crate::html_fallback::needed(self.dom, node, depth, self.options.max_depth); - if !presentation - && structural_table + if structural_table && (complex_table || (!self.anchor_targets.is_empty() && crate::anchors::contains( @@ -465,17 +469,12 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.options.max_depth, &self.anchor_targets, self.options.default_code_language.as_deref(), + &mut self.emitted_anchors, ); self.writer().block(html.into(), 2, 2); return; } - { - for id in self.anchor_targets.targets(self.dom, node) { - if self.emitted_anchors.insert(id.to_owned()) { - self.writer().inline_html(&crate::anchors::markup(id)); - } - } - } + self.emit_own_targets(node); if !complex_table && let Some(mut table) = Table::from_dom(self.dom, node, depth, self.options.max_depth) @@ -488,8 +487,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } return; } - if !presentation - && structural_table + if structural_table && crate::html_fallback::has_cells( self.dom, node, @@ -504,6 +502,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.options.max_depth, &self.anchor_targets, self.options.default_code_language.as_deref(), + &mut self.emitted_anchors, ); self.writer().block(html.into(), 2, 2); return; @@ -558,9 +557,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool { let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth); - let poster = (tag == "video") - .then(|| crate::media::direct_attribute(self.dom, node, "poster", true)) - .flatten(); + let poster = crate::media::poster(self.dom, node, tag); if urls.is_empty() && poster.is_none() { return false; } @@ -831,7 +828,7 @@ fn is_block(tag: &str) -> bool { ) } -fn is_structural_block(tag: &str) -> bool { +pub(crate) fn is_structural_block(tag: &str) -> bool { matches!( tag, "address" diff --git a/moli-html2md/src/media.rs b/moli-html2md/src/media.rs index f5f1dbb034..530ddfd635 100644 --- a/moli-html2md/src/media.rs +++ b/moli-html2md/src/media.rs @@ -273,8 +273,12 @@ pub(crate) fn sources<'a, D: Dom + ?Sized>( } pub(crate) fn label<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId, tag: &str) -> &'a str { - dom.attribute(node, "title") + dom.attribute(node, "aria-label") .filter(|label| !label.trim().is_empty()) + .or_else(|| { + dom.attribute(node, "title") + .filter(|label| !label.trim().is_empty()) + }) .unwrap_or(match tag { "video" => "Video", "audio" => "Audio", @@ -295,3 +299,19 @@ pub(crate) fn safe_link_url(value: &str) -> bool { && !normalized.starts_with("data:text/html") && !normalized.starts_with("data:application/xhtml+xml") } + +pub(crate) fn poster_label(dom: &D, node: D::NodeId) -> String { + format!("{} poster", label(dom, node, "video")) +} + +pub(crate) fn poster<'a, D: Dom + ?Sized>( + dom: &'a D, + node: D::NodeId, + tag: &str, +) -> Option<&'a str> { + (tag == "video") + .then(|| dom.attribute(node, "poster")) + .flatten() + .map(str::trim) + .filter(|src| !src.is_empty() && safe_url(src, true)) +} diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs index b635872e66..4a8088ceaf 100644 --- a/moli-html2md/tests/review_semantics.rs +++ b/moli-html2md/tests/review_semantics.rs @@ -937,3 +937,51 @@ fn merged_headerless_data_is_preserved_without_treating_empty_hints_as_headers() assert!(result.contains("/cover.jpg"), "{result}"); assert!(result.contains("Home"), "{result}"); } + +#[test] +fn fallback_targets_and_media_preserve_navigation_without_duplicate_ids() { + for html in [ + "F
C
Chart
", + "N
C
", + "X

P

C
Cell
", + ] { + let output = markdown(html); + let target = if html.contains("#fig") { + "fig" + } else if html.contains("#note") { + "note" + } else { + "x" + }; + assert_eq!( + output.matches(&format!("id=\"{target}\"")).count(), + 1, + "{output}" + ); + if target == "fig" { + assert!(output.contains("title=\"Details\""), "{output}"); + } + } + let output = markdown( + "12
C
linked textplain
", + ); + let html = rendered_html(&output); + assert!( + html.contains("href=\"/x\" id=\"one\">linked text"), + "{html}" + ); + assert!( + !output.contains("plain") && !output.contains("C" + )); + let html = rendered_html(&output); + assert!( + html.contains("/cover.jpg") && html.contains("href=\"/v.mp4\""), + "{output}" + ); + } +} diff --git a/moli-renderer-v8/src/runtime/page_dump.rs b/moli-renderer-v8/src/runtime/page_dump.rs index 397678970a..f5f636dd93 100644 --- a/moli-renderer-v8/src/runtime/page_dump.rs +++ b/moli-renderer-v8/src/runtime/page_dump.rs @@ -394,30 +394,11 @@ mod tests { ); let before = dom.serialize_document(); let markdown = render_markdown_document(&dom); - for content in [ - "First story", - "42 points by", - "8 comments", - "Second story", - "7 points by", - "discuss", - "More", - ] { - assert!(markdown.contains(content), "missing {content}: {markdown}"); - } - let positions = [ - "First story", - "42 points by", - "Second story", - "7 points by", - "More", - ] - .map(|content| markdown.find(content).expect("checked above")); - assert!( - positions.windows(2).all(|pair| pair[0] < pair[1]), - "{markdown}" + assert_eq!( + markdown, + include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md") + .trim_end_matches('\n') ); - assert!(markdown.contains(" Result<()> String::from_utf8_lossy(&output.stderr) ); assert_eq!( - String::from_utf8(output.stdout)?.trim_end(), - markdown.trim_end() + String::from_utf8(output.stdout)?, + format!("{}\n", markdown.trim_end_matches('\n')) ); Ok(()) }