mirror of
https://github.com/lexmount/moli.git
synced 2026-11-19 16:03:39 +00:00
fix(html2md): retain complex tables and referenced fragments
This commit is contained in:
+17
-10
@@ -70,18 +70,22 @@ missing trailing cells in short rows; the converter does not allocate a padded g
|
||||
|
||||
Tables with `role="presentation"` or `role="none"` retain ordinary block expansion,
|
||||
in DOM order with blank lines between cells, even when structurally simple. Role
|
||||
matching ignores ASCII case and surrounding HTML whitespace. The same fallback
|
||||
applies to complex tables: non-unit spans, multiple `thead` rows, rows wider than
|
||||
matching ignores ASCII case and surrounding HTML whitespace. Complex tables use
|
||||
content-only HTML: non-unit spans, multiple `thead` rows, rows wider than
|
||||
an explicit header, or cells containing nested tables, lists, headings, quotes
|
||||
or preformatted blocks. Inner simple tables can still produce GFM when their
|
||||
containing layout table is expanded. Other roles, borders and CSS do not select
|
||||
another conversion path. Complex layouts remain readable blocks instead of raw
|
||||
HTML tables.
|
||||
or preformatted blocks. This retains header, cell and nested-table associations
|
||||
that Markdown pipe tables cannot express. Inner simple tables can still produce
|
||||
GFM when an explicitly presentational containing table is expanded. Scripts,
|
||||
styles and event handlers are omitted from HTML table output.
|
||||
|
||||
Supported output includes headings, paragraphs, emphasis, strikethrough, links,
|
||||
images, lists, blockquotes, hard breaks, media source links, readable form values,
|
||||
separated form choices, and fenced code. Inline HTML is used when
|
||||
images, lists, blockquotes, hard breaks, media source links, separated form choices,
|
||||
and fenced code. Inline HTML is used when
|
||||
Markdown delimiters cannot express an emphasis boundary.
|
||||
Superscripts and subscripts retain their HTML markers. Complete author-supplied
|
||||
TeX spans retain their commands and indices. Native MathML retains mathematical
|
||||
markup and presentation attributes, with alternate annotations omitted, so
|
||||
fractions, roots, matrices and powers do not collapse into ambiguous plain text.
|
||||
Script/style/head/noscript/template subtrees are omitted.
|
||||
|
||||
`max_depth` counts the supplied root as depth zero; nodes at or beyond the limit
|
||||
@@ -94,8 +98,11 @@ spaces and blank lines. Empty blocks still separate surrounding text.
|
||||
|
||||
This is a structural content dump. It omits non-rendered serialized state blobs
|
||||
but retains content in collapsed panels and inactive tabs. It does not evaluate
|
||||
stylesheets, layout, or JavaScript, and cannot preserve all HTML presentation (for example, arbitrary
|
||||
ordered-list numbering or table spanning). DOM snapshots used by Moli's optional
|
||||
stylesheets, layout, or JavaScript. Callers may supply computed block boundaries
|
||||
through the optional DOM query. Moli's page exporter additionally filters current
|
||||
CSS-hidden UI while retaining explicitly linked disclosure content. The converter
|
||||
cannot preserve all HTML presentation (for example, arbitrary ordered-list numbering).
|
||||
DOM snapshots used by Moli's optional
|
||||
strip/base/frame transformations remain the responsibility of the caller.
|
||||
|
||||
## Verification
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::{Dom, NodeKind};
|
||||
|
||||
pub(crate) struct Targets {
|
||||
referenced: HashSet<String>,
|
||||
ids: HashSet<String>,
|
||||
}
|
||||
|
||||
impl Targets {
|
||||
pub(crate) fn is_empty(&self) -> bool {
|
||||
self.referenced.is_empty()
|
||||
}
|
||||
|
||||
pub(crate) fn contains(&self, value: &str) -> bool {
|
||||
self.referenced.contains(value)
|
||||
}
|
||||
|
||||
pub(crate) fn targets<'a, D: Dom + ?Sized>(&self, dom: &'a D, node: D::NodeId) -> Vec<&'a str> {
|
||||
let mut targets = Vec::with_capacity(2);
|
||||
if let Some(id) = dom.attribute(node, "id").filter(|id| self.contains(id)) {
|
||||
targets.push(id);
|
||||
}
|
||||
if dom.node_kind(node) == NodeKind::Element("a")
|
||||
&& let Some(name) = dom
|
||||
.attribute(node, "name")
|
||||
.filter(|name| self.contains(name) && !self.ids.contains(*name))
|
||||
&& !targets.contains(&name)
|
||||
{
|
||||
targets.push(name);
|
||||
}
|
||||
targets
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn referenced<D: Dom + ?Sized>(dom: &D, root: D::NodeId, limit: usize) -> Targets {
|
||||
let mut available = HashSet::new();
|
||||
let mut element_ids = HashSet::new();
|
||||
if !dom.may_have_fragment_links() {
|
||||
return Targets {
|
||||
referenced: available,
|
||||
ids: element_ids,
|
||||
};
|
||||
}
|
||||
walk(dom, root, limit, |node| {
|
||||
for attribute in ["id", "name"] {
|
||||
if attribute == "name" && dom.node_kind(node) != NodeKind::Element("a") {
|
||||
continue;
|
||||
}
|
||||
if let Some(value) = dom.attribute(node, attribute) {
|
||||
available.insert(value.to_owned());
|
||||
if attribute == "id" {
|
||||
element_ids.insert(value.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
});
|
||||
let mut referenced = HashSet::new();
|
||||
walk(dom, root, limit, |node| {
|
||||
if let Some(fragment) = dom
|
||||
.attribute(node, "href")
|
||||
.and_then(|href| href.strip_prefix('#'))
|
||||
&& !fragment.is_empty()
|
||||
{
|
||||
// HTML fragment navigation tries the literal fragment first and
|
||||
// only percent-decodes it when no literal target exists.
|
||||
if available.contains(fragment) {
|
||||
referenced.insert(fragment.to_owned());
|
||||
} else {
|
||||
let decoded = decode_fragment(fragment);
|
||||
if available.contains(&decoded) {
|
||||
referenced.insert(decoded);
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
});
|
||||
Targets {
|
||||
referenced,
|
||||
ids: element_ids,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn contains<D: Dom + ?Sized>(
|
||||
dom: &D,
|
||||
root: D::NodeId,
|
||||
limit: usize,
|
||||
ids: &Targets,
|
||||
) -> bool {
|
||||
walk(dom, root, limit, |node| !ids.targets(dom, node).is_empty())
|
||||
}
|
||||
|
||||
pub(crate) fn within<D: Dom + ?Sized>(
|
||||
dom: &D,
|
||||
root: D::NodeId,
|
||||
limit: usize,
|
||||
ids: &Targets,
|
||||
) -> Vec<String> {
|
||||
let mut found = Vec::new();
|
||||
walk(dom, root, limit, |node| {
|
||||
for id in ids.targets(dom, node) {
|
||||
if !found.iter().any(|existing| existing == id) {
|
||||
found.push(id.to_owned());
|
||||
}
|
||||
}
|
||||
false
|
||||
});
|
||||
found
|
||||
}
|
||||
|
||||
fn walk<D: Dom + ?Sized>(
|
||||
dom: &D,
|
||||
root: D::NodeId,
|
||||
limit: usize,
|
||||
mut found: impl FnMut(D::NodeId) -> bool,
|
||||
) -> bool {
|
||||
let mut stack = vec![(Some(root), 0, false)];
|
||||
while let Some((node, depth, siblings)) = stack.pop() {
|
||||
let Some(node) = node else {
|
||||
continue;
|
||||
};
|
||||
if siblings {
|
||||
stack.push((dom.next_sibling(node), depth, true));
|
||||
}
|
||||
if depth >= limit {
|
||||
continue;
|
||||
}
|
||||
if matches!(
|
||||
dom.node_kind(node),
|
||||
NodeKind::Other
|
||||
| NodeKind::Element("head" | "script" | "style" | "noscript" | "template")
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if found(node) {
|
||||
return true;
|
||||
}
|
||||
stack.push((dom.first_child(node), depth + 1, true));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn decode_fragment(fragment: &str) -> String {
|
||||
let bytes = fragment.as_bytes();
|
||||
let mut decoded = Vec::with_capacity(bytes.len());
|
||||
let mut offset = 0;
|
||||
while offset < bytes.len() {
|
||||
if bytes[offset] == b'%' && offset + 2 < bytes.len() {
|
||||
let hex = |byte: u8| (byte as char).to_digit(16);
|
||||
if let (Some(high), Some(low)) = (hex(bytes[offset + 1]), hex(bytes[offset + 2])) {
|
||||
decoded.push((high * 16 + low) as u8);
|
||||
offset += 3;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
decoded.push(bytes[offset]);
|
||||
offset += 1;
|
||||
}
|
||||
String::from_utf8(decoded).unwrap_or_else(|_| fragment.to_owned())
|
||||
}
|
||||
|
||||
pub(crate) fn markup(id: &str) -> String {
|
||||
let mut result = String::from("<a id=\"");
|
||||
for character in id.chars() {
|
||||
match character {
|
||||
'&' => result.push_str("&"),
|
||||
'"' => result.push_str("""),
|
||||
'<' => result.push_str("<"),
|
||||
'>' => result.push_str(">"),
|
||||
'\n' => result.push_str(" "),
|
||||
'\r' => result.push_str(" "),
|
||||
character => result.push(character),
|
||||
}
|
||||
}
|
||||
result.push_str("\"></a>");
|
||||
result
|
||||
}
|
||||
@@ -28,7 +28,7 @@ fn nested_table_fallback_copies_text_linearly() {
|
||||
assert_eq!(actual.matches(&after).count(), depth);
|
||||
assert_eq!(
|
||||
support::rendered_html(&actual).matches("<table>").count(),
|
||||
1
|
||||
depth
|
||||
);
|
||||
// Count actual bytes copied when materializing fragments; completed
|
||||
// blocks move into parents without copying their text. This measures
|
||||
|
||||
@@ -32,4 +32,23 @@ pub trait Dom {
|
||||
fn control_value(&self, _node: Self::NodeId) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
|
||||
/// A caller with computed CSS can retain block boundaries introduced by
|
||||
/// stylesheets (for example, links rendered as separate action buttons).
|
||||
/// Structural-only adapters can leave this at its default.
|
||||
fn has_block_layout(&self, _node: Self::NodeId) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// A caller with computed CSS can retain boundaries between independent
|
||||
/// inline boxes such as tabs, badges, and compact action controls.
|
||||
fn has_text_boundary(&self, _node: Self::NodeId) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Adapters that already inventory elements can skip the anchor prepass
|
||||
/// when the document contains no local-fragment links.
|
||||
fn may_have_fragment_links(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::{Dom, NodeKind};
|
||||
|
||||
/// Markdown has no merged cells, nested tables, or block content inside cells.
|
||||
/// Detect those structures before conversion so a table is traversed at most
|
||||
/// twice, even when it contains thousands of nested tables.
|
||||
pub(crate) fn needed<D: Dom + ?Sized>(
|
||||
dom: &D,
|
||||
root: D::NodeId,
|
||||
depth: usize,
|
||||
limit: usize,
|
||||
) -> bool {
|
||||
let mut stack = vec![(dom.first_child(root), depth + 1)];
|
||||
while let Some((node, depth)) = stack.pop() {
|
||||
let Some(node) = node else {
|
||||
continue;
|
||||
};
|
||||
if depth >= limit {
|
||||
continue;
|
||||
}
|
||||
stack.push((dom.next_sibling(node), depth));
|
||||
if let NodeKind::Element(tag) = dom.node_kind(node) {
|
||||
if matches!(
|
||||
tag,
|
||||
"table"
|
||||
| "pre"
|
||||
| "blockquote"
|
||||
| "ul"
|
||||
| "ol"
|
||||
| "li"
|
||||
| "hr"
|
||||
| "h1"
|
||||
| "h2"
|
||||
| "h3"
|
||||
| "h4"
|
||||
| "h5"
|
||||
| "h6"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if matches!(tag, "td" | "th")
|
||||
&& ["rowspan", "colspan"].iter().any(|attr| {
|
||||
dom.attribute(node, attr)
|
||||
.is_some_and(|value| value.trim().parse::<u32>() != Ok(1))
|
||||
})
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
stack.push((dom.first_child(node), depth + 1));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn has_cells<D: Dom + ?Sized>(
|
||||
dom: &D,
|
||||
root: D::NodeId,
|
||||
depth: usize,
|
||||
limit: usize,
|
||||
) -> bool {
|
||||
let mut stack = vec![(dom.first_child(root), depth + 1)];
|
||||
while let Some((node, depth)) = stack.pop() {
|
||||
let Some(node) = node else {
|
||||
continue;
|
||||
};
|
||||
if depth >= limit {
|
||||
continue;
|
||||
}
|
||||
if matches!(dom.node_kind(node), NodeKind::Element("th" | "td")) {
|
||||
return true;
|
||||
}
|
||||
stack.push((dom.next_sibling(node), depth));
|
||||
stack.push((dom.first_child(node), depth + 1));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
enum Task<Id> {
|
||||
Node(Id, usize),
|
||||
Siblings(Option<Id>, usize),
|
||||
Close(String),
|
||||
Literal(&'static str),
|
||||
}
|
||||
|
||||
/// A compact HTML block is valid Markdown and preserves the original cell
|
||||
/// associations. Keep content markup only; scripts, styles, event handlers
|
||||
/// and unrelated presentation attributes are never copied.
|
||||
pub(crate) fn render<D: Dom + ?Sized>(
|
||||
dom: &D,
|
||||
root: D::NodeId,
|
||||
depth: usize,
|
||||
limit: usize,
|
||||
anchor_targets: &crate::anchors::Targets,
|
||||
) -> String {
|
||||
let mut output = String::new();
|
||||
let mut emitted_anchors = HashSet::new();
|
||||
let mut tasks = vec![Task::Node(root, depth)];
|
||||
while let Some(task) = tasks.pop() {
|
||||
match task {
|
||||
Task::Literal(value) => output.push_str(value),
|
||||
Task::Close(tag) => {
|
||||
output.push_str("</");
|
||||
output.push_str(&tag);
|
||||
output.push('>');
|
||||
}
|
||||
Task::Siblings(None, _) => {}
|
||||
Task::Siblings(Some(node), depth) => {
|
||||
tasks.push(Task::Siblings(dom.next_sibling(node), depth));
|
||||
tasks.push(Task::Node(node, depth));
|
||||
}
|
||||
Task::Node(node, depth) => {
|
||||
if depth >= limit {
|
||||
continue;
|
||||
}
|
||||
if matches!(dom.node_kind(node), NodeKind::Element(_)) {
|
||||
if dom.has_block_layout(node) {
|
||||
output.push_str("<div>");
|
||||
tasks.push(Task::Close("div".to_owned()));
|
||||
} else if dom.has_text_boundary(node) {
|
||||
output.push(' ');
|
||||
tasks.push(Task::Literal(" "));
|
||||
}
|
||||
}
|
||||
if let NodeKind::Element(tag) = dom.node_kind(node)
|
||||
&& !allowed(tag)
|
||||
{
|
||||
for id in anchor_targets.targets(dom, node) {
|
||||
if emitted_anchors.insert(id.to_owned()) {
|
||||
output.push_str(&crate::anchors::markup(id));
|
||||
}
|
||||
}
|
||||
}
|
||||
match dom.node_kind(node) {
|
||||
NodeKind::Text(text) => escape(text, &mut output),
|
||||
NodeKind::Element(
|
||||
"head" | "title" | "script" | "style" | "noscript" | "template",
|
||||
)
|
||||
| NodeKind::Other => {}
|
||||
NodeKind::Element("math") => {
|
||||
for id in crate::anchors::within(dom, node, limit - depth, anchor_targets) {
|
||||
if emitted_anchors.insert(id.clone()) {
|
||||
output.push_str(&crate::anchors::markup(&id));
|
||||
}
|
||||
}
|
||||
output.push_str(&crate::mathml::render(dom, node, limit - depth))
|
||||
}
|
||||
NodeKind::Element("input") => {
|
||||
if let Some(value) = crate::form::input_text(dom, node) {
|
||||
escape(&value, &mut output);
|
||||
}
|
||||
}
|
||||
NodeKind::Element("textarea") => {
|
||||
if let Some(value) = crate::form::textarea_text(dom, node) {
|
||||
escape(&value, &mut output);
|
||||
} else {
|
||||
tasks.push(Task::Siblings(dom.first_child(node), depth + 1));
|
||||
}
|
||||
}
|
||||
NodeKind::Element(tag) => {
|
||||
if crate::visibility::nonrendered_serialized_state(dom, node) {
|
||||
continue;
|
||||
}
|
||||
if matches!(tag, "iframe" | "video" | "audio") {
|
||||
let urls = crate::media::sources(dom, node, tag, depth, limit);
|
||||
if !urls.is_empty() {
|
||||
output.push_str("<div>");
|
||||
if tag == "video"
|
||||
&& let Some(poster) = dom
|
||||
.attribute(node, "poster")
|
||||
.filter(|src| crate::media::safe_url(src, true))
|
||||
{
|
||||
output.push_str("<img src=\"");
|
||||
escape(poster, &mut output);
|
||||
output.push_str("\" alt=\"\">");
|
||||
}
|
||||
for url in urls
|
||||
.into_iter()
|
||||
.filter(|url| crate::media::safe_url(url, false))
|
||||
{
|
||||
output.push_str("<a href=\"");
|
||||
escape(url, &mut output);
|
||||
output.push_str("\">");
|
||||
escape(crate::media::label(dom, node, tag), &mut output);
|
||||
output.push_str("</a><br>");
|
||||
}
|
||||
output.push_str("</div>");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let retained = if allowed(tag) { Some(tag) } else { None };
|
||||
if let Some(tag) = retained {
|
||||
output.push('<');
|
||||
output.push_str(tag);
|
||||
for name in [
|
||||
"rowspan", "colspan", "scope", "headers", "id", "href", "src",
|
||||
"alt", "title", "start", "reversed", "value", "label", "selected",
|
||||
"multiple", "disabled", "name",
|
||||
] {
|
||||
let image_source = if name == "src" && tag == "img" {
|
||||
crate::media::source(dom, node)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let value = if name == "src" && tag == "img" {
|
||||
image_source.as_deref()
|
||||
} else {
|
||||
dom.attribute(node, name)
|
||||
};
|
||||
if let Some(value) = value {
|
||||
if matches!(name, "href" | "src")
|
||||
&& !crate::media::safe_url(value, name == "src")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
output.push(' ');
|
||||
output.push_str(name);
|
||||
output.push_str("=\"");
|
||||
escape(value, &mut output);
|
||||
output.push('"');
|
||||
}
|
||||
}
|
||||
output.push('>');
|
||||
if !matches!(tag, "br" | "hr" | "img") {
|
||||
tasks.push(Task::Close(tag.to_owned()));
|
||||
}
|
||||
}
|
||||
if let Some((math, relative_depth)) =
|
||||
crate::mathml::primary_alternative(dom, node, limit - depth)
|
||||
{
|
||||
for id in
|
||||
crate::anchors::within(dom, node, limit - depth, anchor_targets)
|
||||
{
|
||||
if emitted_anchors.insert(id.clone()) {
|
||||
output.push_str(&crate::anchors::markup(&id));
|
||||
}
|
||||
}
|
||||
output.push_str(&crate::mathml::render(
|
||||
dom,
|
||||
math,
|
||||
limit - depth - relative_depth,
|
||||
));
|
||||
} else {
|
||||
if let Some(label) = crate::content::fallback_text(dom, node)
|
||||
&& !crate::content::has_readable_content(dom, node, limit - depth)
|
||||
{
|
||||
escape(&label, &mut output);
|
||||
} else {
|
||||
tasks.push(Task::Siblings(dom.first_child(node), depth + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
NodeKind::Document => {
|
||||
tasks.push(Task::Siblings(dom.first_child(node), depth + 1))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn allowed(tag: &str) -> bool {
|
||||
matches!(
|
||||
tag,
|
||||
"table"
|
||||
| "thead"
|
||||
| "tbody"
|
||||
| "tfoot"
|
||||
| "tr"
|
||||
| "td"
|
||||
| "th"
|
||||
| "caption"
|
||||
| "select"
|
||||
| "optgroup"
|
||||
| "option"
|
||||
| "button"
|
||||
| "textarea"
|
||||
| "p"
|
||||
| "div"
|
||||
| "span"
|
||||
| "a"
|
||||
| "img"
|
||||
| "br"
|
||||
| "hr"
|
||||
| "strong"
|
||||
| "b"
|
||||
| "em"
|
||||
| "i"
|
||||
| "s"
|
||||
| "del"
|
||||
| "sup"
|
||||
| "sub"
|
||||
| "code"
|
||||
| "pre"
|
||||
| "blockquote"
|
||||
| "ul"
|
||||
| "ol"
|
||||
| "li"
|
||||
| "h1"
|
||||
| "h2"
|
||||
| "h3"
|
||||
| "h4"
|
||||
| "h5"
|
||||
| "h6"
|
||||
| "dl"
|
||||
| "dt"
|
||||
| "dd"
|
||||
| "details"
|
||||
| "summary"
|
||||
)
|
||||
}
|
||||
|
||||
fn escape(text: &str, output: &mut String) {
|
||||
for ch in text.chars() {
|
||||
match ch {
|
||||
'&' => output.push_str("&"),
|
||||
'<' => output.push_str("<"),
|
||||
'>' => output.push_str(">"),
|
||||
'"' => output.push_str("""),
|
||||
// Raw HTML blocks end at an empty Markdown line. Entities preserve
|
||||
// preformatted line endings without prematurely ending the block.
|
||||
'\n' => output.push_str(" "),
|
||||
'\r' => output.push_str(" "),
|
||||
_ => output.push(ch),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
mod anchors;
|
||||
mod content;
|
||||
mod converter;
|
||||
mod dom;
|
||||
mod form;
|
||||
mod html_table;
|
||||
mod machine;
|
||||
mod math;
|
||||
mod mathml;
|
||||
@@ -16,10 +18,10 @@ pub use converter::{Converter, convert};
|
||||
pub use dom::{Dom, NodeKind};
|
||||
pub use options::Options;
|
||||
|
||||
pub use media::{SrcsetCandidate, parse_srcset};
|
||||
|
||||
#[cfg(test)]
|
||||
extern crate self as moli_html2md;
|
||||
|
||||
#[cfg(test)]
|
||||
mod copy_tests;
|
||||
|
||||
pub use media::{SrcsetCandidate, parse_srcset};
|
||||
|
||||
+189
-2
@@ -3,6 +3,7 @@ use crate::table::Table;
|
||||
use crate::visibility::nonrendered_serialized_state;
|
||||
use crate::writer::{Style, Writer, longest_run};
|
||||
use crate::{Dom, NodeKind, Options};
|
||||
use std::collections::HashSet;
|
||||
|
||||
enum Task<'a, Id> {
|
||||
Visit(Id, usize),
|
||||
@@ -41,6 +42,8 @@ struct Machine<'a, D: Dom + ?Sized> {
|
||||
tables: Vec<Table<D::NodeId>>,
|
||||
raw: String,
|
||||
serial: usize,
|
||||
anchor_targets: crate::anchors::Targets,
|
||||
emitted_anchors: HashSet<String>,
|
||||
}
|
||||
|
||||
pub(crate) fn convert<D: Dom + ?Sized>(dom: &D, root: D::NodeId, options: &Options) -> String {
|
||||
@@ -53,6 +56,8 @@ pub(crate) fn convert<D: Dom + ?Sized>(dom: &D, root: D::NodeId, options: &Optio
|
||||
tables: Vec::new(),
|
||||
raw: String::new(),
|
||||
serial: 0,
|
||||
anchor_targets: crate::anchors::referenced(dom, root, options.max_depth),
|
||||
emitted_anchors: HashSet::new(),
|
||||
};
|
||||
machine.run();
|
||||
machine.take_writer().into_string()
|
||||
@@ -210,6 +215,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
if depth >= self.options.max_depth {
|
||||
return;
|
||||
}
|
||||
if self.dom.has_block_layout(node) {
|
||||
self.writer().boundary(2);
|
||||
self.tasks.push(Task::Boundary);
|
||||
} else if self.dom.has_text_boundary(node) {
|
||||
self.writer().boundary(1);
|
||||
self.tasks.push(Task::ChoiceBoundary);
|
||||
}
|
||||
let tag = match self.dom.node_kind(node) {
|
||||
NodeKind::Document => {
|
||||
self.children(node, depth + 1);
|
||||
@@ -225,11 +237,21 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
if nonrendered_serialized_state(self.dom, node) {
|
||||
return;
|
||||
}
|
||||
if let Some(table) = self.tables.last_mut() {
|
||||
table.visit_element(tag);
|
||||
if tag != "table"
|
||||
&& !matches!(
|
||||
tag,
|
||||
"head" | "script" | "style" | "noscript" | "template" | "title"
|
||||
)
|
||||
{
|
||||
for id in self.anchor_targets.targets(self.dom, node) {
|
||||
if self.emitted_anchors.insert(id.to_owned()) {
|
||||
self.writer().inline_html(&crate::anchors::markup(id));
|
||||
}
|
||||
}
|
||||
}
|
||||
match tag {
|
||||
"math" => {
|
||||
self.emit_anchors_within(node, self.options.max_depth - depth);
|
||||
let markup = crate::mathml::render(self.dom, node, self.options.max_depth - depth);
|
||||
self.writer().inline_html(&markup);
|
||||
return;
|
||||
@@ -313,6 +335,39 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
}
|
||||
"iframe" | "video" | "audio" if self.media(node, tag, depth) => return,
|
||||
"code" | "pre" => {
|
||||
let remaining = self.options.max_depth - depth;
|
||||
if subtree_has_link(self.dom, node, remaining)
|
||||
|| (!self.anchor_targets.is_empty()
|
||||
&& crate::anchors::contains(
|
||||
self.dom,
|
||||
node,
|
||||
remaining,
|
||||
&self.anchor_targets,
|
||||
))
|
||||
{
|
||||
let html = crate::html_table::render(
|
||||
self.dom,
|
||||
node,
|
||||
depth,
|
||||
self.options.max_depth,
|
||||
&self.anchor_targets,
|
||||
);
|
||||
if tag == "pre" {
|
||||
self.writer().block(html.into(), 2, 2);
|
||||
} else {
|
||||
self.writer().inline_html(&html);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if tag == "pre" && !self.anchor_targets.is_empty() {
|
||||
for id in
|
||||
crate::anchors::within(self.dom, node, remaining, &self.anchor_targets)
|
||||
{
|
||||
if self.emitted_anchors.insert(id.clone()) {
|
||||
self.writer().inline_html(&crate::anchors::markup(&id));
|
||||
}
|
||||
}
|
||||
}
|
||||
self.raw.clear();
|
||||
let end = if tag == "pre" {
|
||||
Task::EndPre(self.language(node))
|
||||
@@ -330,6 +385,24 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
return;
|
||||
}
|
||||
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
|
||||
let remaining = self.options.max_depth - depth;
|
||||
if subtree_has_nested_heading(self.dom, node, remaining) {
|
||||
self.writer().boundary(2);
|
||||
self.tasks.push(Task::Boundary);
|
||||
self.children(node, depth + 1);
|
||||
return;
|
||||
}
|
||||
if subtree_has_block(self.dom, node, remaining) {
|
||||
let html = crate::html_table::render(
|
||||
self.dom,
|
||||
node,
|
||||
depth,
|
||||
self.options.max_depth,
|
||||
&self.anchor_targets,
|
||||
);
|
||||
self.writer().block(html.into(), 2, 2);
|
||||
return;
|
||||
}
|
||||
self.capture();
|
||||
self.writer().heading();
|
||||
self.tasks
|
||||
@@ -342,6 +415,32 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
"ul" | "ol" => self.start_list(node, tag == "ol"),
|
||||
"li" => self.start_item(),
|
||||
"table" => {
|
||||
let presentation = self.dom.attribute(node, "role").is_some_and(|role| {
|
||||
matches!(
|
||||
role.trim().to_ascii_lowercase().as_str(),
|
||||
"presentation" | "none"
|
||||
)
|
||||
});
|
||||
if (!presentation
|
||||
&& crate::html_table::needed(self.dom, node, depth, self.options.max_depth))
|
||||
|| (!self.anchor_targets.is_empty()
|
||||
&& crate::anchors::contains(
|
||||
self.dom,
|
||||
node,
|
||||
self.options.max_depth - depth,
|
||||
&self.anchor_targets,
|
||||
))
|
||||
{
|
||||
let html = crate::html_table::render(
|
||||
self.dom,
|
||||
node,
|
||||
depth,
|
||||
self.options.max_depth,
|
||||
&self.anchor_targets,
|
||||
);
|
||||
self.writer().block(html.into(), 2, 2);
|
||||
return;
|
||||
}
|
||||
if let Some(mut table) =
|
||||
Table::from_dom(self.dom, node, depth, self.options.max_depth)
|
||||
{
|
||||
@@ -353,6 +452,19 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
}
|
||||
return;
|
||||
}
|
||||
if !presentation
|
||||
&& crate::html_table::has_cells(self.dom, node, depth, self.options.max_depth)
|
||||
{
|
||||
let html = crate::html_table::render(
|
||||
self.dom,
|
||||
node,
|
||||
depth,
|
||||
self.options.max_depth,
|
||||
&self.anchor_targets,
|
||||
);
|
||||
self.writer().block(html.into(), 2, 2);
|
||||
return;
|
||||
}
|
||||
self.writer().boundary(2);
|
||||
self.tasks.push(Task::Boundary);
|
||||
}
|
||||
@@ -365,6 +477,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
if crate::mathml::primary_alternative(self.dom, node, self.options.max_depth - depth)
|
||||
.is_some()
|
||||
{
|
||||
self.emit_anchors_within(node, self.options.max_depth - depth);
|
||||
self.tasks
|
||||
.push(Task::MathChildren(self.dom.first_child(node), depth + 1));
|
||||
return;
|
||||
@@ -378,6 +491,14 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
self.children(node, depth + 1);
|
||||
}
|
||||
|
||||
fn emit_anchors_within(&mut self, node: D::NodeId, remaining: usize) {
|
||||
for id in crate::anchors::within(self.dom, node, remaining, &self.anchor_targets) {
|
||||
if self.emitted_anchors.insert(id.clone()) {
|
||||
self.writer().inline_html(&crate::anchors::markup(&id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool {
|
||||
let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth);
|
||||
let poster = (tag == "video")
|
||||
@@ -585,6 +706,72 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
|
||||
}
|
||||
}
|
||||
|
||||
fn subtree_has_nested_heading<D: Dom + ?Sized>(dom: &D, root: D::NodeId, limit: usize) -> bool {
|
||||
let mut pending = Vec::new();
|
||||
let mut child = dom.first_child(root);
|
||||
while let Some(id) = child {
|
||||
pending.push((id, 1));
|
||||
child = dom.next_sibling(id);
|
||||
}
|
||||
while let Some((node, depth)) = pending.pop() {
|
||||
if matches!(
|
||||
dom.node_kind(node),
|
||||
NodeKind::Element("h1" | "h2" | "h3" | "h4" | "h5" | "h6")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if depth + 1 >= limit {
|
||||
continue;
|
||||
}
|
||||
let mut child = dom.first_child(node);
|
||||
while let Some(id) = child {
|
||||
pending.push((id, depth + 1));
|
||||
child = dom.next_sibling(id);
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn subtree_has_link<D: Dom + ?Sized>(dom: &D, root: D::NodeId, limit: usize) -> bool {
|
||||
let mut pending = vec![(dom.first_child(root), 1)];
|
||||
while let Some((node, depth)) = pending.pop() {
|
||||
let Some(node) = node else {
|
||||
continue;
|
||||
};
|
||||
if depth >= limit {
|
||||
continue;
|
||||
}
|
||||
pending.push((dom.next_sibling(node), depth));
|
||||
if matches!(dom.node_kind(node), NodeKind::Element("a"))
|
||||
&& dom
|
||||
.attribute(node, "href")
|
||||
.is_some_and(|href| !href.is_empty() && crate::media::safe_url(href, false))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
pending.push((dom.first_child(node), depth + 1));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn subtree_has_block<D: Dom + ?Sized>(dom: &D, root: D::NodeId, limit: usize) -> bool {
|
||||
let mut pending = vec![(dom.first_child(root), 1)];
|
||||
while let Some((node, depth)) = pending.pop() {
|
||||
let Some(node) = node else {
|
||||
continue;
|
||||
};
|
||||
if depth >= limit {
|
||||
continue;
|
||||
}
|
||||
pending.push((dom.next_sibling(node), depth));
|
||||
if matches!(dom.node_kind(node), NodeKind::Element(tag) if is_structural_block(tag)) {
|
||||
return true;
|
||||
}
|
||||
pending.push((dom.first_child(node), depth + 1));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn class_language(class: Option<&str>) -> Option<&str> {
|
||||
class?
|
||||
.split_ascii_whitespace()
|
||||
|
||||
@@ -10,7 +10,6 @@ pub(crate) struct Table<Id> {
|
||||
cells: std::vec::IntoIter<Cell<Id>>,
|
||||
content: Vec<Output>,
|
||||
in_cell: bool,
|
||||
has_complex_content: bool,
|
||||
rows: Vec<usize>,
|
||||
separators: Vec<&'static str>,
|
||||
has_header: bool,
|
||||
@@ -37,7 +36,6 @@ impl<Id: Copy> Table<Id> {
|
||||
cells: Vec::new().into_iter(),
|
||||
content: Vec::new(),
|
||||
in_cell: false,
|
||||
has_complex_content: false,
|
||||
rows: Vec::new(),
|
||||
separators: Vec::new(),
|
||||
has_header: false,
|
||||
@@ -166,39 +164,9 @@ impl<Id: Copy> Table<Id> {
|
||||
self.in_cell = false;
|
||||
}
|
||||
|
||||
pub(crate) fn visit_element(&mut self, tag: &str) {
|
||||
if self.in_cell
|
||||
&& matches!(
|
||||
tag,
|
||||
"table"
|
||||
| "pre"
|
||||
| "blockquote"
|
||||
| "ul"
|
||||
| "ol"
|
||||
| "li"
|
||||
| "hr"
|
||||
| "h1"
|
||||
| "h2"
|
||||
| "h3"
|
||||
| "h4"
|
||||
| "h5"
|
||||
| "h6"
|
||||
)
|
||||
{
|
||||
self.has_complex_content = true;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn finish(self) -> Output {
|
||||
if self.has_complex_content {
|
||||
let mut output = Output::default();
|
||||
for cell in self.content.into_iter().filter(|cell| !cell.is_empty()) {
|
||||
if !output.is_empty() {
|
||||
output.push_text("\n\n".into());
|
||||
}
|
||||
output.append(cell);
|
||||
}
|
||||
return output;
|
||||
if self.content.iter().all(Output::is_empty) {
|
||||
return Output::default();
|
||||
}
|
||||
let mut output = String::new();
|
||||
if !self.has_header {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use moli_html2md::{Converter, Dom, NodeKind, Options, convert};
|
||||
mod support;
|
||||
|
||||
// A second DOM implementation, independent of NativeDom or an HTML parser.
|
||||
// Its stable arena IDs and borrowed strings are enough for the converter.
|
||||
@@ -304,7 +305,9 @@ fn table_spans_do_not_duplicate_cell_text() {
|
||||
let row = dom.element(table, "tr");
|
||||
let cell = dom.leaf(row, "td", "end");
|
||||
dom.attr(cell, "colspan", "999999999999");
|
||||
assert_eq!(convert(&dom, 0), "one\n\ntwo\n\nend");
|
||||
let result = convert(&dom, 0);
|
||||
assert!(result.contains("rowspan=\"2\">one</td><td>two</td>"));
|
||||
assert!(result.contains("colspan=\"999999999999\">end</td>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -381,7 +384,9 @@ fn converts_deep_tables_on_a_small_thread_stack() {
|
||||
max_depth: usize::MAX,
|
||||
..Options::default()
|
||||
});
|
||||
assert_eq!(converter.convert(&dom, 0), "| |\n| --- |\n| deep |");
|
||||
let result = converter.convert(&dom, 0);
|
||||
assert_eq!(result.matches("<table>").count(), 10_000);
|
||||
assert_eq!(result.matches("deep").count(), 1);
|
||||
})
|
||||
.expect("spawn small-stack test")
|
||||
.join()
|
||||
@@ -409,8 +414,8 @@ fn converts_nested_header_tables_on_a_small_thread_stack() {
|
||||
});
|
||||
let actual = converter.convert(&dom, 0);
|
||||
assert_eq!(actual.matches("Header").count(), 1_000);
|
||||
assert_eq!(actual.matches("| --- |").count(), 1);
|
||||
assert!(actual.ends_with("| deep |"));
|
||||
assert_eq!(actual.matches("<table>").count(), 1_000);
|
||||
assert_eq!(actual.matches("deep").count(), 1);
|
||||
})
|
||||
.expect("spawn small-stack test")
|
||||
.join()
|
||||
@@ -508,10 +513,15 @@ fn markdown_parser_recovers_pipes_and_preformatted_code_inside_tables() {
|
||||
let cell = dom.element(row, "td");
|
||||
dom.leaf(cell, "pre", " <a>|b\nnext\n");
|
||||
let markdown = convert(&dom, 0);
|
||||
let html = rendered_html(&markdown);
|
||||
assert_eq!(
|
||||
html,
|
||||
"<p>Code</p>\n<p><code>a|b</code></p>\n<pre><code> <a>|b\nnext\n</code></pre>\n",
|
||||
"{markdown}"
|
||||
let html = support::rendered_html(&markdown);
|
||||
assert!(html.contains("<code>a|b</code>"), "{html}");
|
||||
assert_eq!(html.matches("<td>").count(), 2);
|
||||
let parsed = support::Tree::parse(&html);
|
||||
assert!(
|
||||
parsed
|
||||
.nodes
|
||||
.iter()
|
||||
.any(|node| node.text.as_deref() == Some(" <a>|b\nnext\n")),
|
||||
"{html}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ fn inline_code_breaks_separate_words_without_formatting_markers() {
|
||||
for html in [
|
||||
"<code>git<br>status</code>",
|
||||
"<code><strong>git<br></strong><em>status</em></code>",
|
||||
"<code><a href='/'>git</a><span><br></span><b>status</b></code>",
|
||||
] {
|
||||
for preformatted_code in [false, true] {
|
||||
let result = markdown_with(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
mod support;
|
||||
|
||||
use moli_html2md::{Converter, Options, parse_srcset};
|
||||
use moli_html2md::{Converter, Dom, NodeKind, Options, parse_srcset};
|
||||
use std::cell::Cell;
|
||||
use support::{Tree, rendered_html};
|
||||
|
||||
fn markdown(html: &str, preformatted_code: bool) -> String {
|
||||
@@ -846,10 +847,7 @@ fn lazy_media_uses_declared_resources_when_standard_sources_are_unavailable() {
|
||||
"<table><tr><td colspan='2'><img src='' data-src='/actual.jpg' alt='Photo'></td></tr></table>",
|
||||
false,
|
||||
);
|
||||
assert!(
|
||||
rendered_html(&table).contains("src=\"/actual.jpg\""),
|
||||
"{table}"
|
||||
);
|
||||
assert!(table.contains("src=\"/actual.jpg\""), "{table}");
|
||||
assert!(!table.contains("spacer.gif"), "{table}");
|
||||
}
|
||||
|
||||
@@ -962,6 +960,7 @@ fn math_streams_across_transparent_wrappers_in_every_inline_container() {
|
||||
r"<a href='#equation'>Equation</a><p id='equation'>$x<span> + </span>y$</p><p>$z<span> + </span>1$</p>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("<a id=\"equation\"></a>"), "{result}");
|
||||
assert!(result.contains("$x + y$\n\n$z + 1$"), "{result}");
|
||||
}
|
||||
|
||||
@@ -1047,3 +1046,354 @@ fn code_excludes_embedded_copy_controls_and_scripts() {
|
||||
assert!(!result.contains("Copy code"), "{result}");
|
||||
assert!(!result.contains("advert"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_layout_tables_do_not_invent_data() {
|
||||
let result = markdown(
|
||||
"<p>Article</p><table><caption>Video unavailable</caption><tr><td><object data='video.swf'></object></td></tr></table><table><tr><th>Metric</th><th>Value</th></tr><tr><td>Rate</td><td></td></tr></table>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("Video unavailable"), "{result}");
|
||||
assert!(!result.contains("| |\n| --- |\n| |"), "{result}");
|
||||
assert!(result.contains("| Rate | |"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merged_cells_keep_calendar_and_group_associations() {
|
||||
let source = "<table><tr><th>Mon</th><th>Tue</th><th>Wed</th></tr><tr><td colspan='2'></td><td>1</td></tr></table><table><tr><td rowspan='2'>Group</td><td>A</td></tr><tr><td>B</td></tr></table>";
|
||||
let result = markdown(source, false);
|
||||
let html = rendered_html(&result);
|
||||
assert!(html.contains("<td colspan=\"2\"></td><td>1</td>"), "{html}");
|
||||
assert!(
|
||||
html.contains("<td rowspan=\"2\">Group</td><td>A</td>"),
|
||||
"{html}"
|
||||
);
|
||||
assert_eq!(html.matches(">Group<").count(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_tables_stay_with_their_column_labels() {
|
||||
let source = "<table><tr><th>Today</th><th>Month</th></tr><tr><td><table><tr><td>Alice</td><td>20</td></tr></table></td><td><table><tr><td>Bob</td><td>100</td></tr></table></td></tr></table>";
|
||||
let html = rendered_html(&markdown(source, false));
|
||||
assert!(html.contains("<th>Today</th><th>Month</th>"), "{html}");
|
||||
assert!(html.contains("<td><table>"), "{html}");
|
||||
assert!(html.contains("<td>Alice</td><td>20</td>"), "{html}");
|
||||
assert!(html.contains("<td>Bob</td><td>100</td>"), "{html}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complex_tables_keep_media_choices_and_literal_content_inert() {
|
||||
let source = "<table onclick='bad()'><tr><td colspan='2'><iframe src='/lecture' title='Lecture'></iframe><select><option label='Day'>long label</option><option>Night</option></select><pre><script>literal</script>\n\nline</pre><script>bad()</script><a href='javascript:bad()'>Bad link</a></td></tr></table>";
|
||||
let html = rendered_html(&markdown(source, false));
|
||||
assert!(html.contains("href=\"/lecture\">Lecture</a>"), "{html}");
|
||||
assert!(
|
||||
html.contains("<option label=\"Day\">long label</option><option>Night</option>"),
|
||||
"{html}"
|
||||
);
|
||||
assert!(
|
||||
html.contains("<script>literal</script>"),
|
||||
"{html}"
|
||||
);
|
||||
assert!(!html.contains("onclick"), "{html}");
|
||||
assert!(!html.contains("<script>"), "{html}");
|
||||
assert!(!html.contains("javascript:"), "{html}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fragment_links_keep_explicit_targets_without_copying_unreferenced_ids() {
|
||||
let source = "<p><a href='#chapter-7'>Chapter</a> <a href='#%E6%B3%A8%E9%87%8A'>Note</a></p><h2 id='chapter-7'>Translated chapter</h2><p id='unreferenced'>Text</p><a name='注释'></a><p>Note body</p>";
|
||||
let result = rendered_html(&markdown(source, false));
|
||||
assert!(result.contains("href=\"#chapter-7\""), "{result}");
|
||||
assert!(result.contains("id=\"chapter-7\""), "{result}");
|
||||
assert!(result.contains("id=\"注释\""), "{result}");
|
||||
assert!(result.contains("<h2>Translated chapter</h2>"), "{result}");
|
||||
assert!(!result.contains("id=\"unreferenced\""), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percent_encoded_named_anchor_matches_its_fragment_link() {
|
||||
let result = markdown(
|
||||
"<a href='#Run%20the%20program'>Run</a><h2><a name='Run the program'></a>Run the program</h2>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("[Run](#Run%20the%20program)"), "{result}");
|
||||
assert!(
|
||||
result.contains("## <a id=\"Run the program\"></a>Run the program"),
|
||||
"{result}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fragment_decoding_does_not_rewrite_dom_ids() {
|
||||
let result = markdown("<a href='#x%2520y'>Go</a><p id='x%20y'>Target</p>", false);
|
||||
assert!(result.contains("[Go](#x%2520y)"), "{result}");
|
||||
assert!(result.contains("<a id=\"x%20y\"></a>"), "{result}");
|
||||
assert!(result.contains("Target"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn literal_fragment_target_wins_before_percent_decoding() {
|
||||
let result = markdown(
|
||||
"<a href='#x%41'>Go</a><p id='x%41'>Original target</p><p id='xA'>Different target</p>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("<a id=\"x%41\"></a>"), "{result}");
|
||||
assert!(!result.contains("<a id=\"xA\"></a>"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fragment_targets_prefer_ids_and_only_accept_legacy_anchor_names() {
|
||||
let result = rendered_html(&markdown(
|
||||
"<a href='#target'>Go</a><p><a name='target'>Legacy</a></p><p id='target'>ID target</p>",
|
||||
false,
|
||||
));
|
||||
assert_eq!(result.matches("id=\"target\"").count(), 1, "{result}");
|
||||
assert!(result.find("ID target").unwrap() > result.find("id=\"target\"").unwrap());
|
||||
assert!(!result.contains("<a id=\"target\"></a>Legacy"), "{result}");
|
||||
|
||||
let result = rendered_html(&markdown(
|
||||
"<a href='#target'>Go</a><table><tr><td colspan='2'><mark name='target'>Wrong</mark><a name='target'>Right</a></td></tr></table>",
|
||||
false,
|
||||
));
|
||||
assert!(!result.contains("id=\"target\"></a>Wrong"), "{result}");
|
||||
assert!(result.contains("name=\"target\""), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_fragment_targets_preserve_rows_cells_and_legacy_named_anchors() {
|
||||
let source = "<a href='#row'>Row</a><a href='#cell'>Cell</a><a href='#legacy'>Legacy</a><table><tr id='row'><td id='cell'><a name='legacy'></a>Value</td><td>Other</td></tr></table>";
|
||||
let result = rendered_html(&markdown(source, false));
|
||||
for target in ["id=\"row\"", "id=\"cell\"", "name=\"legacy\""] {
|
||||
assert!(result.contains(target), "missing {target}: {result}");
|
||||
}
|
||||
assert!(result.contains("<td>Other</td>"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_fallback_preserves_referenced_targets_for_special_nodes() {
|
||||
let code = markdown(
|
||||
"<a href='#L1'>Line 1</a><pre><code><mark id='L1'>let x = 1;</mark></code></pre>",
|
||||
false,
|
||||
);
|
||||
assert!(code.contains("<a id=\"L1\"></a>"), "{code}");
|
||||
let table = markdown(
|
||||
"<a href='#email'>Email</a><a href='#eq1'>Equation</a><a href='#video1'>Video</a><table><tr><td><input id='email' value='a@example.test'></td><td><math id='eq1'><mi>x</mi></math></td><td><video id='video1' src='/clip.mp4'></video></td></tr></table>",
|
||||
false,
|
||||
);
|
||||
for id in ["email", "eq1", "video1"] {
|
||||
assert!(table.contains(&format!("<a id=\"{id}\"></a>")), "{table}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_nested_and_empty_headings_do_not_emit_literal_markers() {
|
||||
let result = markdown(
|
||||
"<h3><center><h3> </h3><h2>Department</h2><h2> </h2></center></h3><p>Rule text.</p>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("## Department"), "{result}");
|
||||
assert!(!result.contains("### ###"), "{result}");
|
||||
assert_eq!(result.matches("## Department").count(), 1, "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preformatted_navigation_retains_link_targets() {
|
||||
let result = markdown(
|
||||
"<pre>ALL GAMES: <a href='home.shtml'>HOME GAMES</a> : <a href='away.shtml'>AWAY GAMES</a>\nScores stay aligned</pre>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("<pre>"), "{result}");
|
||||
assert!(
|
||||
result.contains("<a href=\"home.shtml\">HOME GAMES</a>"),
|
||||
"{result}"
|
||||
);
|
||||
assert!(
|
||||
result.contains("<a href=\"away.shtml\">AWAY GAMES</a>"),
|
||||
"{result}"
|
||||
);
|
||||
assert!(result.contains("Scores stay aligned"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preformatted_code_preserves_externally_referenced_targets() {
|
||||
let result = markdown(
|
||||
"<a href='#L1'>Line 1</a><pre><code><span id='L1'>let x = 1;</span>\n<span id='L2'>let y = 2;</span></code></pre>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("[Line 1](#L1)"), "{result}");
|
||||
assert!(
|
||||
result.contains("<span id=\"L1\">let x = 1;</span>"),
|
||||
"{result}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_code_retains_term_link_targets() {
|
||||
let result = markdown(
|
||||
"<p>Returns <code><a href='/terms/null'>Null</a></code> when absent.</p>",
|
||||
false,
|
||||
);
|
||||
assert!(
|
||||
result.contains("<code><a href=\"/terms/null\">Null</a></code>"),
|
||||
"{result}"
|
||||
);
|
||||
assert!(result.contains("when absent."), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_content_inside_heading_keeps_its_own_boundary() {
|
||||
let result = markdown(
|
||||
"<h1>Concept title<div class='popover'><div>Resource Information</div><p>Long explanation.</p></div></h1>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("<h1>Concept title<div"), "{result}");
|
||||
assert!(
|
||||
result.contains("<div>Resource Information</div>"),
|
||||
"{result}"
|
||||
);
|
||||
assert!(!result.starts_with("# Concept title Resource"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embedded_media_uses_the_same_url_policy_in_all_output_paths() {
|
||||
let media = "<video src='/clip.mp4' poster='javascript:alert(1)'></video><iframe src='data:image/png;base64,AAAA' title='Frame'></iframe><audio src='data:image/png;base64,BBBB' title='Audio'></audio>";
|
||||
for source in [
|
||||
media.to_owned(),
|
||||
format!("<table><tr><td colspan='2'>{media}</td></tr></table>"),
|
||||
] {
|
||||
let result = markdown(&source, false);
|
||||
assert!(result.contains("/clip.mp4"), "{result}");
|
||||
assert!(!result.contains("javascript:"), "{result}");
|
||||
assert!(!result.contains("data:image"), "{result}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complex_table_preserves_checkbox_state() {
|
||||
let source = "<table><tr><th rowspan=2>Status</th><td><input type=radio checked></td></tr><tr><td><input type=radio></td></tr></table>";
|
||||
let result = rendered_html(&markdown(source, false));
|
||||
assert!(result.contains("rowspan=\"2\""), "{result}");
|
||||
assert!(result.contains("☑"), "{result}");
|
||||
assert!(result.contains("☐"), "{result}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fragment_targets_keep_all_aliases_and_delegated_math_descendants() {
|
||||
let result = markdown(
|
||||
"<a href='#modern'>Modern</a><a href='#legacy'>Legacy</a><a href='#eq'>Equation</a><a id='modern' name='legacy'>Target</a><math><mrow id='eq'><mi>x</mi></mrow></math>",
|
||||
false,
|
||||
);
|
||||
for id in ["modern", "legacy", "eq"] {
|
||||
assert!(result.contains(&format!("<a id=\"{id}\"></a>")), "{result}");
|
||||
}
|
||||
}
|
||||
|
||||
struct CountingDom<'a> {
|
||||
tree: &'a Tree,
|
||||
calls: Cell<usize>,
|
||||
}
|
||||
|
||||
impl Dom for CountingDom<'_> {
|
||||
type NodeId = usize;
|
||||
|
||||
fn node_kind(&self, node: usize) -> NodeKind<'_> {
|
||||
self.calls.set(self.calls.get() + 1);
|
||||
self.tree.node_kind(node)
|
||||
}
|
||||
fn first_child(&self, node: usize) -> Option<usize> {
|
||||
self.calls.set(self.calls.get() + 1);
|
||||
self.tree.first_child(node)
|
||||
}
|
||||
fn next_sibling(&self, node: usize) -> Option<usize> {
|
||||
self.calls.set(self.calls.get() + 1);
|
||||
self.tree.next_sibling(node)
|
||||
}
|
||||
fn attribute(&self, node: usize, name: &str) -> Option<&str> {
|
||||
self.calls.set(self.calls.get() + 1);
|
||||
self.tree.attribute(node, name)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transparent_depth_keeps_dom_queries_linear() {
|
||||
let measure = |depth: usize| {
|
||||
let html = format!(
|
||||
"{}content{}",
|
||||
"<span>".repeat(depth),
|
||||
"</span>".repeat(depth)
|
||||
);
|
||||
let tree = Tree::parse(&html);
|
||||
let dom = CountingDom {
|
||||
tree: &tree,
|
||||
calls: Cell::new(0),
|
||||
};
|
||||
let result = Converter::default().convert(&dom, tree.root);
|
||||
assert_eq!(result, "content");
|
||||
dom.calls.get()
|
||||
};
|
||||
let shallow = measure(64);
|
||||
let deep = measure(128);
|
||||
assert!(
|
||||
deep < shallow * 3,
|
||||
"query growth was not linear: {shallow} -> {deep}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readable_fallbacks_survive_all_table_representations() {
|
||||
for wrapper in [
|
||||
"<p>{}</p>",
|
||||
"<table><tr><td>{}</td></tr></table>",
|
||||
"<table><tr><td colspan='2'>{}</td></tr></table>",
|
||||
] {
|
||||
let content = "<a href='/download' aria-label='Download report'> </a><button aria-label='Open panel'><span aria-hidden='true'></span></button><svg><title>Chart title</title></svg>";
|
||||
let result = markdown(&wrapper.replace("{}", content), false);
|
||||
for expected in ["Download report", "Open panel", "Chart title"] {
|
||||
assert!(result.contains(expected), "missing {expected}: {result}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_textarea_value_replaces_or_clears_serialized_default_in_all_paths() {
|
||||
for wrapper in [
|
||||
"<div title='Group'>{}</div>",
|
||||
"<table><tr><td colspan='2'>{}</td></tr></table>",
|
||||
] {
|
||||
let current = markdown(
|
||||
&wrapper.replace(
|
||||
"{}",
|
||||
"<textarea value='Current user text'>Old default</textarea>",
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert!(current.contains("Current user text"), "{current}");
|
||||
assert!(!current.contains("Old default"), "{current}");
|
||||
|
||||
let cleared = markdown(
|
||||
&wrapper.replace("{}", "<textarea value=''>Old default</textarea>"),
|
||||
false,
|
||||
);
|
||||
assert!(!cleared.contains("Old default"), "{cleared}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_table_math_delegation_preserves_referenced_descendant_targets() {
|
||||
let result = markdown(
|
||||
"<a href='#L1'>Term</a><table><tr><td colspan='2'><math><mrow id='L1'><mi>x</mi></mrow></math></td></tr></table>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains("<a id=\"L1\"></a>"), "{result}");
|
||||
assert!(result.contains("<math>"), "{result}");
|
||||
|
||||
let alternative = markdown(
|
||||
"<a href='#wrapped'>Term</a><table><tr><td colspan='2'><span id='wrapped'><span><math><mi>x</mi></math></span><span aria-hidden='true'>visual</span></span></td></tr></table>",
|
||||
false,
|
||||
);
|
||||
assert!(
|
||||
alternative.contains("<a id=\"wrapped\"></a>"),
|
||||
"{alternative}"
|
||||
);
|
||||
assert!(alternative.contains("<math>"), "{alternative}");
|
||||
}
|
||||
|
||||
@@ -11,43 +11,81 @@ fn markdown(html: &str) -> String {
|
||||
result
|
||||
}
|
||||
|
||||
// Compare browser-parsed cell nesting, spans and text, independently of the
|
||||
// serializer's spelling, indentation or Markdown representation.
|
||||
fn table_signature(html: &str) -> Vec<String> {
|
||||
let tree = Tree::parse(html);
|
||||
let mut result = Vec::new();
|
||||
let mut stack = vec![(tree.root, false)];
|
||||
while let Some((id, close)) = stack.pop() {
|
||||
let node = &tree.nodes[id];
|
||||
let table_tag = node.tag.as_deref().filter(|tag| {
|
||||
matches!(
|
||||
*tag,
|
||||
"table" | "thead" | "tbody" | "tfoot" | "tr" | "td" | "th" | "caption"
|
||||
)
|
||||
});
|
||||
if close {
|
||||
if let Some(tag) = table_tag {
|
||||
result.push(format!("/{tag}"));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Some(tag) = table_tag {
|
||||
result.push(tag.to_owned());
|
||||
for (key, value) in &node.attrs {
|
||||
if matches!(key.as_str(), "rowspan" | "colspan" | "scope" | "headers") {
|
||||
result.push(format!("{key}={value}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(text) = &node.text {
|
||||
let text = text.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
if !text.is_empty() {
|
||||
result.push(text);
|
||||
}
|
||||
}
|
||||
stack.push((id, true));
|
||||
let mut children = Vec::new();
|
||||
let mut child = node.first;
|
||||
while let Some(id) = child {
|
||||
children.push(id);
|
||||
child = tree.nodes[id].next;
|
||||
}
|
||||
stack.extend(children.into_iter().rev().map(|id| (id, false)));
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn assert_preserves_table(source: &str) {
|
||||
assert_eq!(
|
||||
table_signature(&rendered_html(&markdown(source))),
|
||||
table_signature(source)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hacker_news_layout_tables_preserve_story_and_metadata_order() {
|
||||
let result = markdown(include_str!("fixtures/hacker-news-layout.html"));
|
||||
assert_eq!(
|
||||
result,
|
||||
include_str!("fixtures/hacker-news-layout.md").trim_end()
|
||||
);
|
||||
assert_preserves_table(include_str!("fixtures/hacker-news-layout.html"));
|
||||
let html = rendered_html(&result);
|
||||
// The simple navigation row becomes a table; nested/spanning story layout
|
||||
// still expands into blocks in the original content order.
|
||||
assert_eq!(html.matches("<table>").count(), 1);
|
||||
assert_eq!(html.matches("<table").count(), 3);
|
||||
assert!(!html.contains("<br>"));
|
||||
assert_eq!(html.matches("<a ").count(), 14);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_data_table_survives_an_expanded_layout_table() {
|
||||
fn nested_data_table_preserves_its_outer_cells() {
|
||||
let html = "<table><tr><td>before</td></tr><tr><td><table><tr><th>Key</th><th>Value</th></tr><tr><td>a|b</td><td><code>c|d</code></td></tr></table></td></tr><tr><td>after</td></tr></table>";
|
||||
let actual = markdown(html);
|
||||
assert_eq!(
|
||||
actual,
|
||||
"before\n\n| Key | Value |\n| --- | --- |\n| a\\|b | `c\\|d` |\n\nafter"
|
||||
);
|
||||
assert_eq!(
|
||||
rendered_html(&actual),
|
||||
"<p>before</p>\n<table><thead><tr><th>Key</th><th>Value</th></tr></thead><tbody>\n<tr><td>a|b</td><td><code>c|d</code></td></tr>\n</tbody></table>\n<p>after</p>\n"
|
||||
);
|
||||
assert_preserves_table(html);
|
||||
assert!(rendered_html(&markdown(html)).contains("<code>c|d</code>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrapping_a_table_preserves_its_content() {
|
||||
let data =
|
||||
"<table><tr><td>one</td><td>two</td></tr><tr><td>three</td><td>four</td></tr></table>";
|
||||
assert_eq!(
|
||||
markdown(&format!("<table><tr><td>{data}</td></tr></table>")),
|
||||
markdown(data)
|
||||
);
|
||||
assert_preserves_table(&format!("<table><tr><td>{data}</td></tr></table>"));
|
||||
assert_eq!(
|
||||
markdown(data),
|
||||
"| | |\n| --- | --- |\n| one | two |\n| three | four |"
|
||||
@@ -118,9 +156,12 @@ fn other_table_roles_do_not_change_conversion() {
|
||||
#[test]
|
||||
fn table_cells_preserve_block_syntax_and_code_whitespace() {
|
||||
let html = "<table><tr><td>intro</td><td><h2>Heading</h2><pre><code> a\n b\n</code></pre><blockquote>quote</blockquote><ul><li>item</li></ul></td><td>end</td></tr></table>";
|
||||
assert_eq!(
|
||||
markdown(html),
|
||||
"intro\n\n## Heading\n\n```\n a\n b\n```\n\n> quote\n\n- item\n\nend"
|
||||
assert_preserves_table(html);
|
||||
let tree = Tree::parse(&rendered_html(&markdown(html)));
|
||||
assert!(
|
||||
tree.nodes
|
||||
.iter()
|
||||
.any(|node| node.text.as_deref() == Some(" a\n b\n"))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -277,7 +318,7 @@ fn literal_pipes_survive_in_text_code_links_and_image_attributes() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unrepresentable_tables_expand_without_losing_or_duplicating_content() {
|
||||
fn complex_tables_keep_cell_associations_without_duplicate_content() {
|
||||
for row in [
|
||||
"<tr><td colspan='2'>one</td><td>two</td></tr>",
|
||||
"<tr><td rowspan='2'>one</td><td>two</td></tr>",
|
||||
@@ -285,15 +326,8 @@ fn unrepresentable_tables_expand_without_losing_or_duplicating_content() {
|
||||
"<tr><td colspan='999999999999999999999'>one</td><td>two</td></tr>",
|
||||
"<tr><td>one</td><td>two</td><td>extra</td></tr>",
|
||||
] {
|
||||
let actual = markdown(&format!(
|
||||
"<table><tr><th>A</th><th>B</th></tr>{row}</table>"
|
||||
));
|
||||
let extra = if row.contains("extra") {
|
||||
"\n\nextra"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
assert_eq!(actual, format!("A\n\nB\n\none\n\ntwo{extra}"));
|
||||
let source = format!("<table><tr><th>A</th><th>B</th></tr>{row}</table>");
|
||||
assert_preserves_table(&source);
|
||||
}
|
||||
for content in [
|
||||
"<pre> code\nnext\n</pre>",
|
||||
@@ -303,14 +337,10 @@ fn unrepresentable_tables_expand_without_losing_or_duplicating_content() {
|
||||
"<table><tr><td>nested</td></tr></table>",
|
||||
] {
|
||||
for cell in ["th", "td"] {
|
||||
let actual = markdown(&format!(
|
||||
let source = format!(
|
||||
"<table><tr><{cell}>A</{cell}><{cell}>B</{cell}></tr><tr><td>one</td><td>{content}</td></tr></table>"
|
||||
));
|
||||
assert_eq!(
|
||||
actual,
|
||||
format!("A\n\nB\n\none\n\n{}", markdown(content)),
|
||||
"{cell}: {content}"
|
||||
);
|
||||
assert_preserves_table(&source);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -349,12 +379,9 @@ fn table_blocks_preserve_surrounding_inline_styles() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_heading_rows_fall_back_to_blocks() {
|
||||
assert_eq!(
|
||||
markdown(
|
||||
"<table><thead><tr><th>A</th><th>B</th></tr><tr><th>one</th><th>two</th></tr></thead><tbody><tr><td>three</td><td>four</td></tr></tbody></table>"
|
||||
),
|
||||
"A\n\nB\n\none\n\ntwo\n\nthree\n\nfour"
|
||||
fn multiple_heading_rows_keep_their_hierarchy() {
|
||||
assert_preserves_table(
|
||||
"<table><thead><tr><th>A</th><th>B</th></tr><tr><th>one</th><th>two</th></tr></thead><tbody><tr><td>three</td><td>four</td></tr></tbody></table>",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -439,15 +466,16 @@ fn nested_tables_obey_the_conversion_depth_limit() {
|
||||
let dom = Tree::parse(
|
||||
"<table><tr><td>visible<div><table><tr><td>too deep</td></tr></table></div></td></tr></table>",
|
||||
);
|
||||
for (max_depth, expected) in [(6, "| |\n| --- |\n| visible |"), (7, "visible")] {
|
||||
for max_depth in [6, 7] {
|
||||
let converter = Converter::new(Options {
|
||||
max_depth,
|
||||
..Options::default()
|
||||
});
|
||||
assert_eq!(converter.convert(&dom, dom.root), expected);
|
||||
let result = converter.convert(&dom, dom.root);
|
||||
assert!(result.contains("visible"));
|
||||
assert!(!result.contains("too deep"));
|
||||
}
|
||||
assert_eq!(
|
||||
convert(&dom, dom.root),
|
||||
"visible\n\n| |\n| --- |\n| too deep |"
|
||||
assert_preserves_table(
|
||||
"<table><tr><td>visible<div><table><tr><td>too deep</td></tr></table></div></td></tr></table>",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -32,8 +32,9 @@ single-line dumps. They remain active assertions; no
|
||||
case is skipped. A reference change that makes a difference obsolete also fails
|
||||
the test. Separate converter tests cover GFM tables and strikethrough as Moli
|
||||
extensions. Explicit table headers follow Turndown's GFM plugin; simple headerless
|
||||
tables receive empty headings. Complex tables retain the core's block expansion
|
||||
rather than the plugin's raw HTML fallback.
|
||||
tables receive empty headings. Complex tables retain content-only HTML so merged
|
||||
cells and nested tables keep their associations. Explicitly presentational tables
|
||||
still expand into blocks.
|
||||
|
||||
## Findings tracked outside the reference corpus
|
||||
|
||||
|
||||
Reference in New Issue
Block a user