");
+ tasks.push(Task::Close("div".to_owned()));
+ } else if dom.has_text_boundary(node) {
+ output.push(' ');
+ tasks.push(Task::Literal(" "));
+ }
+ }
+ if let NodeKind::Element(tag) = dom.node_kind(node)
+ && !allowed(tag)
+ {
+ for id in anchor_targets.targets(dom, node) {
+ if emitted_anchors.insert(id.to_owned()) {
+ output.push_str(&crate::anchors::markup(id));
+ }
+ }
+ }
+ match dom.node_kind(node) {
+ NodeKind::Text(text) => escape(text, &mut output),
+ NodeKind::Element(
+ "head" | "title" | "script" | "style" | "noscript" | "template",
+ )
+ | NodeKind::Other => {}
+ NodeKind::Element("math") => {
+ for id in crate::anchors::within(dom, node, limit - depth, anchor_targets) {
+ if emitted_anchors.insert(id.clone()) {
+ output.push_str(&crate::anchors::markup(&id));
+ }
+ }
+ output.push_str(&crate::mathml::render(dom, node, limit - depth))
+ }
+ NodeKind::Element("input") => {
+ if let Some(value) = crate::form::input_text(dom, node) {
+ escape(&value, &mut output);
+ }
+ }
+ NodeKind::Element("textarea") => {
+ if let Some(value) = crate::form::textarea_text(dom, node) {
+ escape(&value, &mut output);
+ } else {
+ tasks.push(Task::Siblings(dom.first_child(node), depth + 1));
+ }
+ }
+ NodeKind::Element(tag) => {
+ if crate::visibility::nonrendered_serialized_state(dom, node) {
+ continue;
+ }
+ if matches!(tag, "iframe" | "video" | "audio") {
+ let urls = crate::media::sources(dom, node, tag, depth, limit);
+ if !urls.is_empty() {
+ output.push_str("
");
+ continue;
+ }
+ }
+ let retained = if allowed(tag) { Some(tag) } else { None };
+ if let Some(tag) = retained {
+ output.push('<');
+ output.push_str(tag);
+ for name in [
+ "rowspan", "colspan", "scope", "headers", "id", "href", "src",
+ "alt", "title", "start", "reversed", "value", "label", "selected",
+ "multiple", "disabled", "name",
+ ] {
+ let image_source = if name == "src" && tag == "img" {
+ crate::media::source(dom, node)
+ } else {
+ None
+ };
+ let value = if name == "src" && tag == "img" {
+ image_source.as_deref()
+ } else {
+ dom.attribute(node, name)
+ };
+ if let Some(value) = value {
+ if matches!(name, "href" | "src")
+ && !crate::media::safe_url(value, name == "src")
+ {
+ continue;
+ }
+ output.push(' ');
+ output.push_str(name);
+ output.push_str("=\"");
+ escape(value, &mut output);
+ output.push('"');
+ }
+ }
+ output.push('>');
+ if !matches!(tag, "br" | "hr" | "img") {
+ tasks.push(Task::Close(tag.to_owned()));
+ }
+ }
+ if let Some((math, relative_depth)) =
+ crate::mathml::primary_alternative(dom, node, limit - depth)
+ {
+ for id in
+ crate::anchors::within(dom, node, limit - depth, anchor_targets)
+ {
+ if emitted_anchors.insert(id.clone()) {
+ output.push_str(&crate::anchors::markup(&id));
+ }
+ }
+ output.push_str(&crate::mathml::render(
+ dom,
+ math,
+ limit - depth - relative_depth,
+ ));
+ } else {
+ if let Some(label) = crate::content::fallback_text(dom, node)
+ && !crate::content::has_readable_content(dom, node, limit - depth)
+ {
+ escape(&label, &mut output);
+ } else {
+ tasks.push(Task::Siblings(dom.first_child(node), depth + 1));
+ }
+ }
+ }
+ NodeKind::Document => {
+ tasks.push(Task::Siblings(dom.first_child(node), depth + 1))
+ }
+ }
+ }
+ }
+ }
+ output
+}
+
+fn allowed(tag: &str) -> bool {
+ matches!(
+ tag,
+ "table"
+ | "thead"
+ | "tbody"
+ | "tfoot"
+ | "tr"
+ | "td"
+ | "th"
+ | "caption"
+ | "select"
+ | "optgroup"
+ | "option"
+ | "button"
+ | "textarea"
+ | "p"
+ | "div"
+ | "span"
+ | "a"
+ | "img"
+ | "br"
+ | "hr"
+ | "strong"
+ | "b"
+ | "em"
+ | "i"
+ | "s"
+ | "del"
+ | "sup"
+ | "sub"
+ | "code"
+ | "pre"
+ | "blockquote"
+ | "ul"
+ | "ol"
+ | "li"
+ | "h1"
+ | "h2"
+ | "h3"
+ | "h4"
+ | "h5"
+ | "h6"
+ | "dl"
+ | "dt"
+ | "dd"
+ | "details"
+ | "summary"
+ )
+}
+
+fn escape(text: &str, output: &mut String) {
+ for ch in text.chars() {
+ match ch {
+ '&' => output.push_str("&"),
+ '<' => output.push_str("<"),
+ '>' => output.push_str(">"),
+ '"' => output.push_str("""),
+ // Raw HTML blocks end at an empty Markdown line. Entities preserve
+ // preformatted line endings without prematurely ending the block.
+ '\n' => output.push_str("
"),
+ '\r' => output.push_str("
"),
+ _ => output.push(ch),
+ }
+ }
+}
diff --git a/moli-html2md/src/lib.rs b/moli-html2md/src/lib.rs
index ec7a16b8cd5..8bbec691b5d 100644
--- a/moli-html2md/src/lib.rs
+++ b/moli-html2md/src/lib.rs
@@ -1,7 +1,9 @@
+mod anchors;
mod content;
mod converter;
mod dom;
mod form;
+mod html_table;
mod machine;
mod math;
mod mathml;
@@ -16,10 +18,10 @@ pub use converter::{Converter, convert};
pub use dom::{Dom, NodeKind};
pub use options::Options;
+pub use media::{SrcsetCandidate, parse_srcset};
+
#[cfg(test)]
extern crate self as moli_html2md;
#[cfg(test)]
mod copy_tests;
-
-pub use media::{SrcsetCandidate, parse_srcset};
diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs
index 9351431ba17..ac4ca227668 100644
--- a/moli-html2md/src/machine.rs
+++ b/moli-html2md/src/machine.rs
@@ -3,6 +3,7 @@ use crate::table::Table;
use crate::visibility::nonrendered_serialized_state;
use crate::writer::{Style, Writer, longest_run};
use crate::{Dom, NodeKind, Options};
+use std::collections::HashSet;
enum Task<'a, Id> {
Visit(Id, usize),
@@ -41,6 +42,8 @@ struct Machine<'a, D: Dom + ?Sized> {
tables: Vec
>,
raw: String,
serial: usize,
+ anchor_targets: crate::anchors::Targets,
+ emitted_anchors: HashSet,
}
pub(crate) fn convert(dom: &D, root: D::NodeId, options: &Options) -> String {
@@ -53,6 +56,8 @@ pub(crate) fn convert(dom: &D, root: D::NodeId, options: &Optio
tables: Vec::new(),
raw: String::new(),
serial: 0,
+ anchor_targets: crate::anchors::referenced(dom, root, options.max_depth),
+ emitted_anchors: HashSet::new(),
};
machine.run();
machine.take_writer().into_string()
@@ -210,6 +215,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
if depth >= self.options.max_depth {
return;
}
+ if self.dom.has_block_layout(node) {
+ self.writer().boundary(2);
+ self.tasks.push(Task::Boundary);
+ } else if self.dom.has_text_boundary(node) {
+ self.writer().boundary(1);
+ self.tasks.push(Task::ChoiceBoundary);
+ }
let tag = match self.dom.node_kind(node) {
NodeKind::Document => {
self.children(node, depth + 1);
@@ -225,11 +237,21 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
if nonrendered_serialized_state(self.dom, node) {
return;
}
- if let Some(table) = self.tables.last_mut() {
- table.visit_element(tag);
+ if tag != "table"
+ && !matches!(
+ tag,
+ "head" | "script" | "style" | "noscript" | "template" | "title"
+ )
+ {
+ for id in self.anchor_targets.targets(self.dom, node) {
+ if self.emitted_anchors.insert(id.to_owned()) {
+ self.writer().inline_html(&crate::anchors::markup(id));
+ }
+ }
}
match tag {
"math" => {
+ self.emit_anchors_within(node, self.options.max_depth - depth);
let markup = crate::mathml::render(self.dom, node, self.options.max_depth - depth);
self.writer().inline_html(&markup);
return;
@@ -313,6 +335,39 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
}
"iframe" | "video" | "audio" if self.media(node, tag, depth) => return,
"code" | "pre" => {
+ let remaining = self.options.max_depth - depth;
+ if subtree_has_link(self.dom, node, remaining)
+ || (!self.anchor_targets.is_empty()
+ && crate::anchors::contains(
+ self.dom,
+ node,
+ remaining,
+ &self.anchor_targets,
+ ))
+ {
+ let html = crate::html_table::render(
+ self.dom,
+ node,
+ depth,
+ self.options.max_depth,
+ &self.anchor_targets,
+ );
+ if tag == "pre" {
+ self.writer().block(html.into(), 2, 2);
+ } else {
+ self.writer().inline_html(&html);
+ }
+ return;
+ }
+ if tag == "pre" && !self.anchor_targets.is_empty() {
+ for id in
+ crate::anchors::within(self.dom, node, remaining, &self.anchor_targets)
+ {
+ if self.emitted_anchors.insert(id.clone()) {
+ self.writer().inline_html(&crate::anchors::markup(&id));
+ }
+ }
+ }
self.raw.clear();
let end = if tag == "pre" {
Task::EndPre(self.language(node))
@@ -330,6 +385,24 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
return;
}
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
+ let remaining = self.options.max_depth - depth;
+ if subtree_has_nested_heading(self.dom, node, remaining) {
+ self.writer().boundary(2);
+ self.tasks.push(Task::Boundary);
+ self.children(node, depth + 1);
+ return;
+ }
+ if subtree_has_block(self.dom, node, remaining) {
+ let html = crate::html_table::render(
+ self.dom,
+ node,
+ depth,
+ self.options.max_depth,
+ &self.anchor_targets,
+ );
+ self.writer().block(html.into(), 2, 2);
+ return;
+ }
self.capture();
self.writer().heading();
self.tasks
@@ -342,6 +415,32 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
"ul" | "ol" => self.start_list(node, tag == "ol"),
"li" => self.start_item(),
"table" => {
+ let presentation = self.dom.attribute(node, "role").is_some_and(|role| {
+ matches!(
+ role.trim().to_ascii_lowercase().as_str(),
+ "presentation" | "none"
+ )
+ });
+ if (!presentation
+ && crate::html_table::needed(self.dom, node, depth, self.options.max_depth))
+ || (!self.anchor_targets.is_empty()
+ && crate::anchors::contains(
+ self.dom,
+ node,
+ self.options.max_depth - depth,
+ &self.anchor_targets,
+ ))
+ {
+ let html = crate::html_table::render(
+ self.dom,
+ node,
+ depth,
+ self.options.max_depth,
+ &self.anchor_targets,
+ );
+ self.writer().block(html.into(), 2, 2);
+ return;
+ }
if let Some(mut table) =
Table::from_dom(self.dom, node, depth, self.options.max_depth)
{
@@ -353,6 +452,19 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
}
return;
}
+ if !presentation
+ && crate::html_table::has_cells(self.dom, node, depth, self.options.max_depth)
+ {
+ let html = crate::html_table::render(
+ self.dom,
+ node,
+ depth,
+ self.options.max_depth,
+ &self.anchor_targets,
+ );
+ self.writer().block(html.into(), 2, 2);
+ return;
+ }
self.writer().boundary(2);
self.tasks.push(Task::Boundary);
}
@@ -365,6 +477,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
if crate::mathml::primary_alternative(self.dom, node, self.options.max_depth - depth)
.is_some()
{
+ self.emit_anchors_within(node, self.options.max_depth - depth);
self.tasks
.push(Task::MathChildren(self.dom.first_child(node), depth + 1));
return;
@@ -378,6 +491,14 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
self.children(node, depth + 1);
}
+ fn emit_anchors_within(&mut self, node: D::NodeId, remaining: usize) {
+ for id in crate::anchors::within(self.dom, node, remaining, &self.anchor_targets) {
+ if self.emitted_anchors.insert(id.clone()) {
+ self.writer().inline_html(&crate::anchors::markup(&id));
+ }
+ }
+ }
+
fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool {
let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth);
let poster = (tag == "video")
@@ -585,6 +706,72 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
}
}
+fn subtree_has_nested_heading(dom: &D, root: D::NodeId, limit: usize) -> bool {
+ let mut pending = Vec::new();
+ let mut child = dom.first_child(root);
+ while let Some(id) = child {
+ pending.push((id, 1));
+ child = dom.next_sibling(id);
+ }
+ while let Some((node, depth)) = pending.pop() {
+ if matches!(
+ dom.node_kind(node),
+ NodeKind::Element("h1" | "h2" | "h3" | "h4" | "h5" | "h6")
+ ) {
+ return true;
+ }
+ if depth + 1 >= limit {
+ continue;
+ }
+ let mut child = dom.first_child(node);
+ while let Some(id) = child {
+ pending.push((id, depth + 1));
+ child = dom.next_sibling(id);
+ }
+ }
+ false
+}
+
+fn subtree_has_link(dom: &D, root: D::NodeId, limit: usize) -> bool {
+ let mut pending = vec![(dom.first_child(root), 1)];
+ while let Some((node, depth)) = pending.pop() {
+ let Some(node) = node else {
+ continue;
+ };
+ if depth >= limit {
+ continue;
+ }
+ pending.push((dom.next_sibling(node), depth));
+ if matches!(dom.node_kind(node), NodeKind::Element("a"))
+ && dom
+ .attribute(node, "href")
+ .is_some_and(|href| !href.is_empty() && crate::media::safe_url(href, false))
+ {
+ return true;
+ }
+ pending.push((dom.first_child(node), depth + 1));
+ }
+ false
+}
+
+fn subtree_has_block(dom: &D, root: D::NodeId, limit: usize) -> bool {
+ let mut pending = vec![(dom.first_child(root), 1)];
+ while let Some((node, depth)) = pending.pop() {
+ let Some(node) = node else {
+ continue;
+ };
+ if depth >= limit {
+ continue;
+ }
+ pending.push((dom.next_sibling(node), depth));
+ if matches!(dom.node_kind(node), NodeKind::Element(tag) if is_structural_block(tag)) {
+ return true;
+ }
+ pending.push((dom.first_child(node), depth + 1));
+ }
+ false
+}
+
fn class_language(class: Option<&str>) -> Option<&str> {
class?
.split_ascii_whitespace()
diff --git a/moli-html2md/src/table.rs b/moli-html2md/src/table.rs
index f29e72208ea..6a4ee474053 100644
--- a/moli-html2md/src/table.rs
+++ b/moli-html2md/src/table.rs
@@ -10,7 +10,6 @@ pub(crate) struct Table {
cells: std::vec::IntoIter| >,
content: Vec |