mirror of
https://github.com/lexmount/moli.git
synced 2026-11-17 00:08:42 +00:00
fix(html2md): share structural boundaries and preserve fallback targets
This commit is contained in:
@@ -71,13 +71,13 @@ missing trailing cells in short rows; the converter does not allocate a padded g
|
||||
Tables with `role="presentation"` or `role="none"` retain ordinary block expansion,
|
||||
in DOM order with blank lines between cells, even when structurally simple. Role
|
||||
matching ignores ASCII case and surrounding HTML whitespace. Nested layout
|
||||
wrappers expand into readable blocks. Tables with headings, captions, explicit
|
||||
wrappers expand into readable blocks. Tables with header cells, captions, explicit
|
||||
data roles, header relationships or real cell spans retain data-table structure.
|
||||
Complex data tables use content-only HTML: non-unit spans, multiple `thead` rows, rows wider than
|
||||
an explicit header, or cells containing nested tables, lists, headings, quotes
|
||||
or preformatted blocks. This retains header, cell and nested-table associations
|
||||
that Markdown pipe tables cannot express. Inner simple tables can still produce
|
||||
GFM when an explicitly presentational containing table is expanded. Scripts,
|
||||
GFM when a presentational or layout-only containing table is expanded. Scripts,
|
||||
styles and event handlers are omitted from HTML fallback output.
|
||||
|
||||
Supported output includes headings, paragraphs, emphasis, strikethrough, links,
|
||||
|
||||
@@ -135,3 +135,8 @@ pub(crate) fn class_language(class: Option<&str>) -> Option<&str> {
|
||||
.split_ascii_whitespace()
|
||||
.find_map(|part| part.strip_prefix("language-").filter(|s| !s.is_empty()))
|
||||
}
|
||||
|
||||
/// These elements already own their document/list/choice boundaries.
|
||||
pub(crate) fn owns_boundaries(tag: &str) -> bool {
|
||||
crate::machine::is_structural_block(tag) || matches!(tag, "select" | "optgroup" | "option")
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@ use std::fmt::Write as _;
|
||||
use crate::{Dom, NodeKind};
|
||||
|
||||
/// Markdown has no merged cells, nested tables, or block content inside cells.
|
||||
/// Detect those structures before conversion so a table is traversed at most
|
||||
/// twice, even when it contains thousands of nested tables.
|
||||
/// Each structural pre-scan stops at nested tables; nested scopes are disjoint.
|
||||
/// Conversion and target discovery keep their own bounded linear traversals.
|
||||
pub(crate) fn needed<D: Dom + ?Sized>(
|
||||
dom: &D,
|
||||
root: D::NodeId,
|
||||
@@ -138,8 +138,9 @@ enum Task<Id> {
|
||||
Literal(&'static str),
|
||||
}
|
||||
|
||||
/// HTML fallback preserves structural content and referenced navigation targets. Keep content markup only; scripts, styles, event handlers
|
||||
/// and unrelated presentation attributes are never copied.
|
||||
/// HTML fallback preserves structural content and referenced navigation targets.
|
||||
/// Keep content markup only; scripts, styles, event handlers and unrelated
|
||||
/// presentation attributes are never copied.
|
||||
pub(crate) fn render<D: Dom + ?Sized>(
|
||||
dom: &D,
|
||||
root: D::NodeId,
|
||||
@@ -147,11 +148,11 @@ pub(crate) fn render<D: Dom + ?Sized>(
|
||||
limit: usize,
|
||||
anchor_targets: &crate::anchors::Targets,
|
||||
default_language: Option<&str>,
|
||||
emitted_anchors: &mut HashSet<String>,
|
||||
) -> String {
|
||||
let inline_root = matches!(dom.node_kind(root), NodeKind::Element("code"));
|
||||
let keep_source_lines = matches!(dom.node_kind(root), NodeKind::Element("pre"));
|
||||
let mut output = String::new();
|
||||
let mut emitted_anchors = HashSet::new();
|
||||
let mut tasks = vec![Task::Node(root, depth, false)];
|
||||
while let Some(task) = tasks.pop() {
|
||||
match task {
|
||||
@@ -188,10 +189,13 @@ pub(crate) fn render<D: Dom + ?Sized>(
|
||||
}
|
||||
let code = code || matches!(dom.node_kind(node), NodeKind::Element("pre" | "code"));
|
||||
if let NodeKind::Element(tag) = dom.node_kind(node) {
|
||||
if dom.has_block_layout(node) && !has_block_tag(tag) {
|
||||
if dom.has_block_layout(node)
|
||||
&& !has_block_tag(tag)
|
||||
&& !crate::content::owns_boundaries(tag)
|
||||
{
|
||||
output.push_str("<div>");
|
||||
tasks.push(Task::Close("div".to_owned()));
|
||||
} else if dom.has_text_boundary(node) {
|
||||
} else if dom.has_text_boundary(node) && !crate::content::owns_boundaries(tag) {
|
||||
output.push(' ');
|
||||
tasks.push(Task::Literal(" "));
|
||||
}
|
||||
@@ -243,25 +247,42 @@ pub(crate) fn render<D: Dom + ?Sized>(
|
||||
}
|
||||
if tag == "img" {
|
||||
let label = crate::content::image_label(dom, node);
|
||||
let targets: Vec<_> = anchor_targets
|
||||
.targets(dom, node)
|
||||
.into_iter()
|
||||
.filter(|id| emitted_anchors.insert((*id).to_owned()))
|
||||
.collect();
|
||||
if let Some(src) = crate::media::source(dom, node) {
|
||||
for id in targets.iter().skip(1) {
|
||||
output.push_str(&crate::anchors::markup(id));
|
||||
}
|
||||
output.push_str("<img src=\"");
|
||||
escape(&src, &mut output);
|
||||
output.push_str("\" alt=\"");
|
||||
escape(&label, &mut output);
|
||||
output.push_str("\">");
|
||||
output.push('"');
|
||||
if let Some(id) = targets.first() {
|
||||
output.push_str(" id=\"");
|
||||
escape(id, &mut output);
|
||||
output.push('"');
|
||||
}
|
||||
if let Some(title) = dom.attribute(node, "title") {
|
||||
output.push_str(" title=\"");
|
||||
escape(title, &mut output);
|
||||
output.push('"');
|
||||
}
|
||||
output.push('>');
|
||||
} else {
|
||||
for id in targets {
|
||||
output.push_str(&crate::anchors::markup(id));
|
||||
}
|
||||
escape(&label, &mut output);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if matches!(tag, "iframe" | "video" | "audio") {
|
||||
let urls = crate::media::sources(dom, node, tag, depth, limit);
|
||||
let poster = (tag == "video")
|
||||
.then(|| dom.attribute(node, "poster"))
|
||||
.flatten()
|
||||
.filter(|src| {
|
||||
!src.trim().is_empty() && crate::media::safe_url(src, true)
|
||||
});
|
||||
let poster = crate::media::poster(dom, node, tag);
|
||||
if !urls.is_empty() || poster.is_some() {
|
||||
let block = tag == "audio";
|
||||
output.push_str(if block { "<div>" } else { " " });
|
||||
@@ -272,6 +293,7 @@ pub(crate) fn render<D: Dom + ?Sized>(
|
||||
escape(&crate::media::poster_label(dom, node), &mut output);
|
||||
output.push_str("\"> ");
|
||||
}
|
||||
let has_source = !urls.is_empty();
|
||||
for url in urls {
|
||||
output.push_str("<a href=\"");
|
||||
escape(url, &mut output);
|
||||
@@ -282,12 +304,14 @@ pub(crate) fn render<D: Dom + ?Sized>(
|
||||
if block {
|
||||
output.push_str("</div>");
|
||||
}
|
||||
continue;
|
||||
if has_source {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
let retained = if allowed(tag)
|
||||
&& (tag != "span"
|
||||
|| dom.attribute(node, "id").is_some()
|
||||
|| !anchor_targets.targets(dom, node).is_empty()
|
||||
|| dom.attribute(node, "title").is_some())
|
||||
{
|
||||
Some(tag)
|
||||
@@ -308,6 +332,14 @@ pub(crate) fn render<D: Dom + ?Sized>(
|
||||
) {
|
||||
begin_html_line(&mut output);
|
||||
}
|
||||
let targets: Vec<_> = anchor_targets
|
||||
.targets(dom, node)
|
||||
.into_iter()
|
||||
.filter(|id| emitted_anchors.insert((*id).to_owned()))
|
||||
.collect();
|
||||
for id in targets.iter().skip(1) {
|
||||
output.push_str(&crate::anchors::markup(id));
|
||||
}
|
||||
output.push('<');
|
||||
output.push_str(tag);
|
||||
for name in [
|
||||
@@ -338,11 +370,6 @@ pub(crate) fn render<D: Dom + ?Sized>(
|
||||
output.push('"');
|
||||
}
|
||||
}
|
||||
let targets: Vec<_> = anchor_targets
|
||||
.targets(dom, node)
|
||||
.into_iter()
|
||||
.filter(|id| emitted_anchors.insert((*id).to_owned()))
|
||||
.collect();
|
||||
if let Some(id) = targets.first() {
|
||||
output.push_str(" id=\"");
|
||||
escape(id, &mut output);
|
||||
@@ -368,9 +395,6 @@ pub(crate) fn render<D: Dom + ?Sized>(
|
||||
if tag == "pre" {
|
||||
output.push('\n');
|
||||
}
|
||||
for id in targets.iter().skip(1) {
|
||||
output.push_str(&crate::anchors::markup(id));
|
||||
}
|
||||
if !matches!(tag, "br" | "hr" | "img") {
|
||||
tasks.push(Task::Close(tag.to_owned()));
|
||||
}
|
||||
|
||||
+16
-19
@@ -11,6 +11,7 @@ enum Task<'a, Id> {
|
||||
MathChildren(Option<Id>, usize),
|
||||
Boundary,
|
||||
ChoiceBoundary,
|
||||
InlineSpace,
|
||||
Fallback(String, usize),
|
||||
PopStyle,
|
||||
EndLink(usize),
|
||||
@@ -88,6 +89,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
| Task::MathChildren(None, _)
|
||||
| Task::RawChildren(None, _, _) => {}
|
||||
Task::Boundary => self.writer().boundary(2),
|
||||
Task::InlineSpace => self.writer().text(" "),
|
||||
Task::ChoiceBoundary => self.writer().boundary(1),
|
||||
Task::Fallback(label, before) => {
|
||||
if self.writer().content_mark() == before {
|
||||
@@ -223,12 +225,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
if depth >= self.options.max_depth {
|
||||
return;
|
||||
}
|
||||
if self.dom.has_block_layout(node) {
|
||||
let owns_boundaries = matches!(self.dom.node_kind(node), NodeKind::Element(tag) if crate::content::owns_boundaries(tag));
|
||||
if !owns_boundaries && self.dom.has_block_layout(node) {
|
||||
self.writer().boundary(2);
|
||||
self.tasks.push(Task::Boundary);
|
||||
} else if self.dom.has_text_boundary(node) {
|
||||
self.writer().boundary(1);
|
||||
self.tasks.push(Task::ChoiceBoundary);
|
||||
} else if !owns_boundaries && self.dom.has_text_boundary(node) {
|
||||
self.writer().text(" ");
|
||||
self.tasks.push(Task::InlineSpace);
|
||||
}
|
||||
let tag = match self.dom.node_kind(node) {
|
||||
NodeKind::Document => {
|
||||
@@ -375,6 +378,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
self.options.max_depth,
|
||||
&self.anchor_targets,
|
||||
self.options.default_code_language.as_deref(),
|
||||
&mut self.emitted_anchors,
|
||||
);
|
||||
if tag == "pre" {
|
||||
self.writer().block(html.into(), 2, 2);
|
||||
@@ -416,6 +420,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
self.options.max_depth,
|
||||
&self.anchor_targets,
|
||||
self.options.default_code_language.as_deref(),
|
||||
&mut self.emitted_anchors,
|
||||
);
|
||||
self.writer().block(html.into(), 2, 2);
|
||||
return;
|
||||
@@ -447,8 +452,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
);
|
||||
let complex_table =
|
||||
crate::html_fallback::needed(self.dom, node, depth, self.options.max_depth);
|
||||
if !presentation
|
||||
&& structural_table
|
||||
if structural_table
|
||||
&& (complex_table
|
||||
|| (!self.anchor_targets.is_empty()
|
||||
&& crate::anchors::contains(
|
||||
@@ -465,17 +469,12 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
self.options.max_depth,
|
||||
&self.anchor_targets,
|
||||
self.options.default_code_language.as_deref(),
|
||||
&mut self.emitted_anchors,
|
||||
);
|
||||
self.writer().block(html.into(), 2, 2);
|
||||
return;
|
||||
}
|
||||
{
|
||||
for id in self.anchor_targets.targets(self.dom, node) {
|
||||
if self.emitted_anchors.insert(id.to_owned()) {
|
||||
self.writer().inline_html(&crate::anchors::markup(id));
|
||||
}
|
||||
}
|
||||
}
|
||||
self.emit_own_targets(node);
|
||||
if !complex_table
|
||||
&& let Some(mut table) =
|
||||
Table::from_dom(self.dom, node, depth, self.options.max_depth)
|
||||
@@ -488,8 +487,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
}
|
||||
return;
|
||||
}
|
||||
if !presentation
|
||||
&& structural_table
|
||||
if structural_table
|
||||
&& crate::html_fallback::has_cells(
|
||||
self.dom,
|
||||
node,
|
||||
@@ -504,6 +502,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
self.options.max_depth,
|
||||
&self.anchor_targets,
|
||||
self.options.default_code_language.as_deref(),
|
||||
&mut self.emitted_anchors,
|
||||
);
|
||||
self.writer().block(html.into(), 2, 2);
|
||||
return;
|
||||
@@ -558,9 +557,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
|
||||
fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool {
|
||||
let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth);
|
||||
let poster = (tag == "video")
|
||||
.then(|| crate::media::direct_attribute(self.dom, node, "poster", true))
|
||||
.flatten();
|
||||
let poster = crate::media::poster(self.dom, node, tag);
|
||||
if urls.is_empty() && poster.is_none() {
|
||||
return false;
|
||||
}
|
||||
@@ -831,7 +828,7 @@ fn is_block(tag: &str) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
fn is_structural_block(tag: &str) -> bool {
|
||||
pub(crate) fn is_structural_block(tag: &str) -> bool {
|
||||
matches!(
|
||||
tag,
|
||||
"address"
|
||||
|
||||
@@ -273,8 +273,12 @@ pub(crate) fn sources<'a, D: Dom + ?Sized>(
|
||||
}
|
||||
|
||||
pub(crate) fn label<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId, tag: &str) -> &'a str {
|
||||
dom.attribute(node, "title")
|
||||
dom.attribute(node, "aria-label")
|
||||
.filter(|label| !label.trim().is_empty())
|
||||
.or_else(|| {
|
||||
dom.attribute(node, "title")
|
||||
.filter(|label| !label.trim().is_empty())
|
||||
})
|
||||
.unwrap_or(match tag {
|
||||
"video" => "Video",
|
||||
"audio" => "Audio",
|
||||
@@ -295,3 +299,19 @@ pub(crate) fn safe_link_url(value: &str) -> bool {
|
||||
&& !normalized.starts_with("data:text/html")
|
||||
&& !normalized.starts_with("data:application/xhtml+xml")
|
||||
}
|
||||
|
||||
pub(crate) fn poster_label<D: Dom + ?Sized>(dom: &D, node: D::NodeId) -> String {
|
||||
format!("{} poster", label(dom, node, "video"))
|
||||
}
|
||||
|
||||
pub(crate) fn poster<'a, D: Dom + ?Sized>(
|
||||
dom: &'a D,
|
||||
node: D::NodeId,
|
||||
tag: &str,
|
||||
) -> Option<&'a str> {
|
||||
(tag == "video")
|
||||
.then(|| dom.attribute(node, "poster"))
|
||||
.flatten()
|
||||
.map(str::trim)
|
||||
.filter(|src| !src.is_empty() && safe_url(src, true))
|
||||
}
|
||||
|
||||
@@ -937,3 +937,51 @@ fn merged_headerless_data_is_preserved_without_treating_empty_hints_as_headers()
|
||||
assert!(result.contains("/cover.jpg"), "{result}");
|
||||
assert!(result.contains("Home"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_targets_and_media_preserve_navigation_without_duplicate_ids() {
|
||||
for html in [
|
||||
"<a href='#fig'>F</a><table><caption>C</caption><tr><td><img id='fig' src='/a.png' alt='Chart' title='Details'></td></tr></table>",
|
||||
"<a href='#note'>N</a><table><caption>C</caption><tr><td><textarea id='note'>x</textarea></td></tr></table>",
|
||||
"<a href='#x'>X</a><p id='x'>P</p><table><caption>C</caption><tr><td id='x'>Cell</td></tr></table>",
|
||||
] {
|
||||
let output = markdown(html);
|
||||
let target = if html.contains("#fig") {
|
||||
"fig"
|
||||
} else if html.contains("#note") {
|
||||
"note"
|
||||
} else {
|
||||
"x"
|
||||
};
|
||||
assert_eq!(
|
||||
output.matches(&format!("id=\"{target}\"")).count(),
|
||||
1,
|
||||
"{output}"
|
||||
);
|
||||
if target == "fig" {
|
||||
assert!(output.contains("title=\"Details\""), "{output}");
|
||||
}
|
||||
}
|
||||
let output = markdown(
|
||||
"<a href='#one'>1</a><a href='#two'>2</a><table><caption>C</caption><tr><td><a id='one' name='two' href='/x'>linked text</a><span id='unreferenced'>plain</span><video poster=' ' src='/v.mp4'></video></td></tr></table>",
|
||||
);
|
||||
let html = rendered_html(&output);
|
||||
assert!(
|
||||
html.contains("href=\"/x\" id=\"one\">linked text</a>"),
|
||||
"{html}"
|
||||
);
|
||||
assert!(
|
||||
!output.contains("<span>plain") && !output.contains("<img"),
|
||||
"{output}"
|
||||
);
|
||||
for span in ["", "colspan='2'"] {
|
||||
let output = markdown(&format!(
|
||||
"<table><caption>C</caption><tr><td {span}><video poster='/cover.jpg' src='blob:id'><a href='/v.mp4'>Download</a></video></td></tr></table>"
|
||||
));
|
||||
let html = rendered_html(&output);
|
||||
assert!(
|
||||
html.contains("/cover.jpg") && html.contains("href=\"/v.mp4\""),
|
||||
"{output}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -394,30 +394,11 @@ mod tests {
|
||||
);
|
||||
let before = dom.serialize_document();
|
||||
let markdown = render_markdown_document(&dom);
|
||||
for content in [
|
||||
"First story",
|
||||
"42 points by",
|
||||
"8 comments",
|
||||
"Second story",
|
||||
"7 points by",
|
||||
"discuss",
|
||||
"More",
|
||||
] {
|
||||
assert!(markdown.contains(content), "missing {content}: {markdown}");
|
||||
}
|
||||
let positions = [
|
||||
"First story",
|
||||
"42 points by",
|
||||
"Second story",
|
||||
"7 points by",
|
||||
"More",
|
||||
]
|
||||
.map(|content| markdown.find(content).expect("checked above"));
|
||||
assert!(
|
||||
positions.windows(2).all(|pair| pair[0] < pair[1]),
|
||||
"{markdown}"
|
||||
assert_eq!(
|
||||
markdown,
|
||||
include_str!("../../../moli-html2md/tests/fixtures/hacker-news-layout.md")
|
||||
.trim_end_matches('\n')
|
||||
);
|
||||
assert!(markdown.contains("<table"), "{markdown}");
|
||||
assert_eq!(dom.serialize_document(), before);
|
||||
}
|
||||
|
||||
|
||||
@@ -32,8 +32,8 @@ fn assert_markdown_dump(html: &str, markdown: &str, args: &[&str]) -> Result<()>
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert_eq!(
|
||||
String::from_utf8(output.stdout)?.trim_end(),
|
||||
markdown.trim_end()
|
||||
String::from_utf8(output.stdout)?,
|
||||
format!("{}\n", markdown.trim_end_matches('\n'))
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user