fix(html2md): share structural boundaries and preserve fallback targets

This commit is contained in:
lanyue-llk
2026-10-28 03:43:27 +07:00
committed by KEVIN LIANG
parent 9343290ad6
commit 413bf13007
8 changed files with 146 additions and 71 deletions
+2 -2
View File
@@ -71,13 +71,13 @@ missing trailing cells in short rows; the converter does not allocate a padded g
Tables with `role="presentation"` or `role="none"` retain ordinary block expansion,
in DOM order with blank lines between cells, even when structurally simple. Role
matching ignores ASCII case and surrounding HTML whitespace. Nested layout
wrappers expand into readable blocks. Tables with headings, captions, explicit
wrappers expand into readable blocks. Tables with header cells, captions, explicit
data roles, header relationships or real cell spans retain data-table structure.
Complex data tables use content-only HTML: non-unit spans, multiple `thead` rows, rows wider than
an explicit header, or cells containing nested tables, lists, headings, quotes
or preformatted blocks. This retains header, cell and nested-table associations
that Markdown pipe tables cannot express. Inner simple tables can still produce
GFM when an explicitly presentational containing table is expanded. Scripts,
GFM when a presentational or layout-only containing table is expanded. Scripts,
styles and event handlers are omitted from HTML fallback output.
Supported output includes headings, paragraphs, emphasis, strikethrough, links,
+5
View File
@@ -135,3 +135,8 @@ pub(crate) fn class_language(class: Option<&str>) -> Option<&str> {
.split_ascii_whitespace()
.find_map(|part| part.strip_prefix("language-").filter(|s| !s.is_empty()))
}
/// These elements already own their document/list/choice boundaries.
pub(crate) fn owns_boundaries(tag: &str) -> bool {
crate::machine::is_structural_block(tag) || matches!(tag, "select" | "optgroup" | "option")
}
+48 -24
View File
@@ -4,8 +4,8 @@ use std::fmt::Write as _;
use crate::{Dom, NodeKind};
/// Markdown has no merged cells, nested tables, or block content inside cells.
/// Detect those structures before conversion so a table is traversed at most
/// twice, even when it contains thousands of nested tables.
/// Each structural pre-scan stops at nested tables; nested scopes are disjoint.
/// Conversion and target discovery keep their own bounded linear traversals.
pub(crate) fn needed<D: Dom + ?Sized>(
dom: &D,
root: D::NodeId,
@@ -138,8 +138,9 @@ enum Task<Id> {
Literal(&'static str),
}
/// HTML fallback preserves structural content and referenced navigation targets. Keep content markup only; scripts, styles, event handlers
/// and unrelated presentation attributes are never copied.
/// HTML fallback preserves structural content and referenced navigation targets.
/// Keep content markup only; scripts, styles, event handlers and unrelated
/// presentation attributes are never copied.
pub(crate) fn render<D: Dom + ?Sized>(
dom: &D,
root: D::NodeId,
@@ -147,11 +148,11 @@ pub(crate) fn render<D: Dom + ?Sized>(
limit: usize,
anchor_targets: &crate::anchors::Targets,
default_language: Option<&str>,
emitted_anchors: &mut HashSet<String>,
) -> String {
let inline_root = matches!(dom.node_kind(root), NodeKind::Element("code"));
let keep_source_lines = matches!(dom.node_kind(root), NodeKind::Element("pre"));
let mut output = String::new();
let mut emitted_anchors = HashSet::new();
let mut tasks = vec![Task::Node(root, depth, false)];
while let Some(task) = tasks.pop() {
match task {
@@ -188,10 +189,13 @@ pub(crate) fn render<D: Dom + ?Sized>(
}
let code = code || matches!(dom.node_kind(node), NodeKind::Element("pre" | "code"));
if let NodeKind::Element(tag) = dom.node_kind(node) {
if dom.has_block_layout(node) && !has_block_tag(tag) {
if dom.has_block_layout(node)
&& !has_block_tag(tag)
&& !crate::content::owns_boundaries(tag)
{
output.push_str("<div>");
tasks.push(Task::Close("div".to_owned()));
} else if dom.has_text_boundary(node) {
} else if dom.has_text_boundary(node) && !crate::content::owns_boundaries(tag) {
output.push(' ');
tasks.push(Task::Literal(" "));
}
@@ -243,25 +247,42 @@ pub(crate) fn render<D: Dom + ?Sized>(
}
if tag == "img" {
let label = crate::content::image_label(dom, node);
let targets: Vec<_> = anchor_targets
.targets(dom, node)
.into_iter()
.filter(|id| emitted_anchors.insert((*id).to_owned()))
.collect();
if let Some(src) = crate::media::source(dom, node) {
for id in targets.iter().skip(1) {
output.push_str(&crate::anchors::markup(id));
}
output.push_str("<img src=\"");
escape(&src, &mut output);
output.push_str("\" alt=\"");
escape(&label, &mut output);
output.push_str("\">");
output.push('"');
if let Some(id) = targets.first() {
output.push_str(" id=\"");
escape(id, &mut output);
output.push('"');
}
if let Some(title) = dom.attribute(node, "title") {
output.push_str(" title=\"");
escape(title, &mut output);
output.push('"');
}
output.push('>');
} else {
for id in targets {
output.push_str(&crate::anchors::markup(id));
}
escape(&label, &mut output);
}
continue;
}
if matches!(tag, "iframe" | "video" | "audio") {
let urls = crate::media::sources(dom, node, tag, depth, limit);
let poster = (tag == "video")
.then(|| dom.attribute(node, "poster"))
.flatten()
.filter(|src| {
!src.trim().is_empty() && crate::media::safe_url(src, true)
});
let poster = crate::media::poster(dom, node, tag);
if !urls.is_empty() || poster.is_some() {
let block = tag == "audio";
output.push_str(if block { "<div>" } else { " " });
@@ -272,6 +293,7 @@ pub(crate) fn render<D: Dom + ?Sized>(
escape(&crate::media::poster_label(dom, node), &mut output);
output.push_str("\"> ");
}
let has_source = !urls.is_empty();
for url in urls {
output.push_str("<a href=\"");
escape(url, &mut output);
@@ -282,12 +304,14 @@ pub(crate) fn render<D: Dom + ?Sized>(
if block {
output.push_str("</div>");
}
continue;
if has_source {
continue;
}
}
}
let retained = if allowed(tag)
&& (tag != "span"
|| dom.attribute(node, "id").is_some()
|| !anchor_targets.targets(dom, node).is_empty()
|| dom.attribute(node, "title").is_some())
{
Some(tag)
@@ -308,6 +332,14 @@ pub(crate) fn render<D: Dom + ?Sized>(
) {
begin_html_line(&mut output);
}
let targets: Vec<_> = anchor_targets
.targets(dom, node)
.into_iter()
.filter(|id| emitted_anchors.insert((*id).to_owned()))
.collect();
for id in targets.iter().skip(1) {
output.push_str(&crate::anchors::markup(id));
}
output.push('<');
output.push_str(tag);
for name in [
@@ -338,11 +370,6 @@ pub(crate) fn render<D: Dom + ?Sized>(
output.push('"');
}
}
let targets: Vec<_> = anchor_targets
.targets(dom, node)
.into_iter()
.filter(|id| emitted_anchors.insert((*id).to_owned()))
.collect();
if let Some(id) = targets.first() {
output.push_str(" id=\"");
escape(id, &mut output);
@@ -368,9 +395,6 @@ pub(crate) fn render<D: Dom + ?Sized>(
if tag == "pre" {
output.push('\n');
}
for id in targets.iter().skip(1) {
output.push_str(&crate::anchors::markup(id));
}
if !matches!(tag, "br" | "hr" | "img") {
tasks.push(Task::Close(tag.to_owned()));
}
+16 -19
View File
@@ -11,6 +11,7 @@ enum Task<'a, Id> {
MathChildren(Option<Id>, usize),
Boundary,
ChoiceBoundary,
InlineSpace,
Fallback(String, usize),
PopStyle,
EndLink(usize),
@@ -88,6 +89,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
| Task::MathChildren(None, _)
| Task::RawChildren(None, _, _) => {}
Task::Boundary => self.writer().boundary(2),
Task::InlineSpace => self.writer().text(" "),
Task::ChoiceBoundary => self.writer().boundary(1),
Task::Fallback(label, before) => {
if self.writer().content_mark() == before {
@@ -223,12 +225,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
if depth >= self.options.max_depth {
return;
}
if self.dom.has_block_layout(node) {
let owns_boundaries = matches!(self.dom.node_kind(node), NodeKind::Element(tag) if crate::content::owns_boundaries(tag));
if !owns_boundaries && self.dom.has_block_layout(node) {
self.writer().boundary(2);
self.tasks.push(Task::Boundary);
} else if self.dom.has_text_boundary(node) {
self.writer().boundary(1);
self.tasks.push(Task::ChoiceBoundary);
} else if !owns_boundaries && self.dom.has_text_boundary(node) {
self.writer().text(" ");
self.tasks.push(Task::InlineSpace);
}
let tag = match self.dom.node_kind(node) {
NodeKind::Document => {
@@ -375,6 +378,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
self.options.max_depth,
&self.anchor_targets,
self.options.default_code_language.as_deref(),
&mut self.emitted_anchors,
);
if tag == "pre" {
self.writer().block(html.into(), 2, 2);
@@ -416,6 +420,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
self.options.max_depth,
&self.anchor_targets,
self.options.default_code_language.as_deref(),
&mut self.emitted_anchors,
);
self.writer().block(html.into(), 2, 2);
return;
@@ -447,8 +452,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
);
let complex_table =
crate::html_fallback::needed(self.dom, node, depth, self.options.max_depth);
if !presentation
&& structural_table
if structural_table
&& (complex_table
|| (!self.anchor_targets.is_empty()
&& crate::anchors::contains(
@@ -465,17 +469,12 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
self.options.max_depth,
&self.anchor_targets,
self.options.default_code_language.as_deref(),
&mut self.emitted_anchors,
);
self.writer().block(html.into(), 2, 2);
return;
}
{
for id in self.anchor_targets.targets(self.dom, node) {
if self.emitted_anchors.insert(id.to_owned()) {
self.writer().inline_html(&crate::anchors::markup(id));
}
}
}
self.emit_own_targets(node);
if !complex_table
&& let Some(mut table) =
Table::from_dom(self.dom, node, depth, self.options.max_depth)
@@ -488,8 +487,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
}
return;
}
if !presentation
&& structural_table
if structural_table
&& crate::html_fallback::has_cells(
self.dom,
node,
@@ -504,6 +502,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
self.options.max_depth,
&self.anchor_targets,
self.options.default_code_language.as_deref(),
&mut self.emitted_anchors,
);
self.writer().block(html.into(), 2, 2);
return;
@@ -558,9 +557,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool {
let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth);
let poster = (tag == "video")
.then(|| crate::media::direct_attribute(self.dom, node, "poster", true))
.flatten();
let poster = crate::media::poster(self.dom, node, tag);
if urls.is_empty() && poster.is_none() {
return false;
}
@@ -831,7 +828,7 @@ fn is_block(tag: &str) -> bool {
)
}
fn is_structural_block(tag: &str) -> bool {
pub(crate) fn is_structural_block(tag: &str) -> bool {
matches!(
tag,
"address"
+21 -1
View File
@@ -273,8 +273,12 @@ pub(crate) fn sources<'a, D: Dom + ?Sized>(
}
pub(crate) fn label<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId, tag: &str) -> &'a str {
dom.attribute(node, "title")
dom.attribute(node, "aria-label")
.filter(|label| !label.trim().is_empty())
.or_else(|| {
dom.attribute(node, "title")
.filter(|label| !label.trim().is_empty())
})
.unwrap_or(match tag {
"video" => "Video",
"audio" => "Audio",
@@ -295,3 +299,19 @@ pub(crate) fn safe_link_url(value: &str) -> bool {
&& !normalized.starts_with("data:text/html")
&& !normalized.starts_with("data:application/xhtml+xml")
}
pub(crate) fn poster_label<D: Dom + ?Sized>(dom: &D, node: D::NodeId) -> String {
format!("{} poster", label(dom, node, "video"))
}
pub(crate) fn poster<'a, D: Dom + ?Sized>(
dom: &'a D,
node: D::NodeId,
tag: &str,
) -> Option<&'a str> {
(tag == "video")
.then(|| dom.attribute(node, "poster"))
.flatten()
.map(str::trim)
.filter(|src| !src.is_empty() && safe_url(src, true))
}
+48
View File
@@ -937,3 +937,51 @@ fn merged_headerless_data_is_preserved_without_treating_empty_hints_as_headers()
assert!(result.contains("/cover.jpg"), "{result}");
assert!(result.contains("Home"), "{result}");
}
#[test]
fn fallback_targets_and_media_preserve_navigation_without_duplicate_ids() {
for html in [
"<a href='#fig'>F</a><table><caption>C</caption><tr><td><img id='fig' src='/a.png' alt='Chart' title='Details'></td></tr></table>",
"<a href='#note'>N</a><table><caption>C</caption><tr><td><textarea id='note'>x</textarea></td></tr></table>",
"<a href='#x'>X</a><p id='x'>P</p><table><caption>C</caption><tr><td id='x'>Cell</td></tr></table>",
] {
let output = markdown(html);
let target = if html.contains("#fig") {
"fig"
} else if html.contains("#note") {
"note"
} else {
"x"
};
assert_eq!(
output.matches(&format!("id=\"{target}\"")).count(),
1,
"{output}"
);
if target == "fig" {
assert!(output.contains("title=\"Details\""), "{output}");
}
}
let output = markdown(
"<a href='#one'>1</a><a href='#two'>2</a><table><caption>C</caption><tr><td><a id='one' name='two' href='/x'>linked text</a><span id='unreferenced'>plain</span><video poster=' ' src='/v.mp4'></video></td></tr></table>",
);
let html = rendered_html(&output);
assert!(
html.contains("href=\"/x\" id=\"one\">linked text</a>"),
"{html}"
);
assert!(
!output.contains("<span>plain") && !output.contains("<img"),
"{output}"
);
for span in ["", "colspan='2'"] {
let output = markdown(&format!(
"<table><caption>C</caption><tr><td {span}><video poster='/cover.jpg' src='blob:id'><a href='/v.mp4'>Download</a></video></td></tr></table>"
));
let html = rendered_html(&output);
assert!(
html.contains("/cover.jpg") && html.contains("href=\"/v.mp4\""),
"{output}"
);
}
}
+4 -23
View File
@@ -394,30 +394,11 @@ mod tests {
);
let before = dom.serialize_document();
let markdown = render_markdown_document(&dom);
for content in [
"First story",
"42 points by",
"8 comments",
"Second story",
"7 points by",
"discuss",
"More",
] {
assert!(markdown.contains(content), "missing {content}: {markdown}");
}
let positions = [
"First story",
"42 points by",
"Second story",
"7 points by",
"More",
]
.map(|content| markdown.find(content).expect("checked above"));
assert!(
positions.windows(2).all(|pair| pair[0] < pair[1]),
"{markdown}"
assert_eq!(
markdown,
include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md")
.trim_end_matches('\n')
);
assert!(markdown.contains("<table"), "{markdown}");
assert_eq!(dom.serialize_document(), before);
}
+2 -2
View File
@@ -32,8 +32,8 @@ fn assert_markdown_dump(html: &str, markdown: &str, args: &[&str]) -> Result<()>
String::from_utf8_lossy(&output.stderr)
);
assert_eq!(
String::from_utf8(output.stdout)?.trim_end(),
markdown.trim_end()
String::from_utf8(output.stdout)?,
format!("{}\n", markdown.trim_end_matches('\n'))
);
Ok(())
}