mirror of
https://github.com/lexmount/moli.git
synced 2026-11-13 16:10:01 +00:00
fix(html2md): preserve math payloads and literal dollar boundaries
This commit is contained in:
lanyue-llk
@@ -138,7 +138,65 @@ pub(crate) fn normalized_span(text: &str) -> String {
|
||||
let mut chars = text[delimiter..text.len() - delimiter].chars().peekable();
|
||||
let mut escaped = false;
|
||||
let mut space = false;
|
||||
let mut text_depth = 0usize;
|
||||
while let Some(ch) = chars.next() {
|
||||
if ch == '\\' && !escaped && chars.clone().take(4).eq("verb".chars()) {
|
||||
let mut probe = chars.clone();
|
||||
for _ in 0..4 {
|
||||
probe.next();
|
||||
}
|
||||
let star = probe.peek() == Some(&'*');
|
||||
if star {
|
||||
probe.next();
|
||||
}
|
||||
if let Some(verb_delimiter) = probe
|
||||
.next()
|
||||
.filter(|c| !c.is_whitespace() && !c.is_ascii_alphabetic())
|
||||
{
|
||||
if space && output.len() > delimiter {
|
||||
output.push(' ');
|
||||
}
|
||||
space = false;
|
||||
output.push_str(r"\verb");
|
||||
for _ in 0..4 {
|
||||
chars.next();
|
||||
}
|
||||
if star {
|
||||
output.push('*');
|
||||
chars.next();
|
||||
}
|
||||
output.push(chars.next().expect("verb delimiter"));
|
||||
for c in chars.by_ref() {
|
||||
output.push(c);
|
||||
if c == verb_delimiter {
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if ch == '\\' && !escaped {
|
||||
let command = if chars.clone().take(5).eq("text{".chars()) {
|
||||
Some("text{")
|
||||
} else if chars.clone().take(5).eq("mbox{".chars()) {
|
||||
Some("mbox{")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(command) = command {
|
||||
if space && output.len() > delimiter {
|
||||
output.push(' ');
|
||||
}
|
||||
space = false;
|
||||
output.push('\\');
|
||||
output.push_str(command);
|
||||
for _ in 0..command.len() {
|
||||
chars.next();
|
||||
}
|
||||
text_depth += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if ch == '%' && !escaped {
|
||||
for part in chars.by_ref() {
|
||||
if matches!(part, '\n' | '\r') {
|
||||
@@ -154,11 +212,21 @@ pub(crate) fn normalized_span(text: &str) -> String {
|
||||
continue;
|
||||
}
|
||||
if space {
|
||||
output.push(' ');
|
||||
if output.len() > delimiter {
|
||||
output.push(' ');
|
||||
}
|
||||
space = false;
|
||||
}
|
||||
output.push(ch);
|
||||
if !escaped && text_depth > 0 {
|
||||
if ch == '{' {
|
||||
text_depth += 1;
|
||||
} else if ch == '}' {
|
||||
text_depth -= 1;
|
||||
}
|
||||
}
|
||||
if ch == '<'
|
||||
&& text_depth == 0
|
||||
&& chars
|
||||
.peek()
|
||||
.is_some_and(|next| next.is_ascii_alphabetic() || matches!(next, '/' | '!' | '?'))
|
||||
@@ -167,9 +235,6 @@ pub(crate) fn normalized_span(text: &str) -> String {
|
||||
}
|
||||
escaped = ch == '\\' && !escaped;
|
||||
}
|
||||
if space {
|
||||
output.push(' ');
|
||||
}
|
||||
output.push_str(&text[text.len() - delimiter..]);
|
||||
output
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use crate::{Dom, NodeKind};
|
||||
use std::fmt::Write as _;
|
||||
|
||||
/// Math renderers often pair semantic MathML with an aria-hidden visual copy.
|
||||
/// Recognize that relationship from markup, independently of library classes.
|
||||
@@ -201,8 +202,8 @@ const ATTRIBUTES: &[&str] = &[
|
||||
fn escape(text: &str, output: &mut String, markdown_text: bool) {
|
||||
for ch in text.chars() {
|
||||
match ch {
|
||||
'\\' | '*' | '_' | '`' | '~' | '[' | ']' if markdown_text => {
|
||||
output.push_str(&format!("&#{};", ch as u32));
|
||||
'\\' | '*' | '_' | '`' | '~' | '[' | ']' | '$' if markdown_text => {
|
||||
write!(output, "&#{};", ch as u32).expect("writing to String");
|
||||
}
|
||||
'&' => output.push_str("&"),
|
||||
'<' => output.push_str("<"),
|
||||
|
||||
@@ -154,7 +154,7 @@ impl<'a> Writer<'a> {
|
||||
self.line_digits = Some(0);
|
||||
}
|
||||
match ch {
|
||||
'\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '~' => {
|
||||
'\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '~' | '$' => {
|
||||
self.output.push('\\');
|
||||
self.output.push(ch);
|
||||
}
|
||||
|
||||
@@ -257,7 +257,7 @@ fn separate_nested_spans_keep_html_when_delimiters_are_ambiguous() {
|
||||
#[test]
|
||||
fn multiline_image_alt_preserves_the_image() {
|
||||
let result = markdown("<img src='/x' alt='first\n# heading'>");
|
||||
// Turndown emits the same Markdown. Matching it does not preserve the image.
|
||||
// Alt newlines must not open a heading; the image and its text alternative survive.
|
||||
let actual_html = rendered_html(&result);
|
||||
let desired_html = "<p><img src=\"/x\" alt=\"first # heading\" /></p>\n";
|
||||
assert_eq!(
|
||||
@@ -280,8 +280,8 @@ fn multiline_link_title_preserves_the_link() {
|
||||
|
||||
#[test]
|
||||
fn multiple_nested_spans_keep_their_styles_at_paragraph_end() {
|
||||
// This predates the closing-edge fix: with no following character there
|
||||
// is no punctuation fallback, but sibling delimiters can still mispair.
|
||||
// This guards the closing-edge fix: with no following character there
|
||||
// is no punctuation fallback, but sibling delimiters must not mispair.
|
||||
let result = markdown("<em><strong>x</strong>b<strong>c</strong></em>");
|
||||
let actual_html = rendered_html(&result);
|
||||
let desired_html = "<p><em><strong>x</strong>b<strong>c</strong></em></p>\n";
|
||||
|
||||
@@ -825,9 +825,9 @@ fn math_recognition_does_not_reinterpret_prices_or_literal_code() {
|
||||
r"<p>Pay $5 for a_b and $10 for c_d; <code>$x_i$</code>; $unfinished_name</p>",
|
||||
false,
|
||||
);
|
||||
assert!(result.contains(r"$5 for a\_b and $10 for c\_d"), "{result}");
|
||||
assert!(result.contains(r"\$5 for a\_b and \$10 for c\_d"), "{result}");
|
||||
assert!(result.contains("`$x_i$`"), "{result}");
|
||||
assert!(result.contains(r"$unfinished\_name"), "{result}");
|
||||
assert!(result.contains(r"\$unfinished\_name"), "{result}");
|
||||
let result = markdown(r"<p>括号 \(x_i + \alpha\)；方括号 \[y_1=2\]。</p>", false);
|
||||
assert!(result.contains(r"\(x_i + \alpha\)"), "{result}");
|
||||
assert!(result.contains(r"\[y_1=2\]"), "{result}");
|
||||
|
||||
@@ -167,7 +167,7 @@ fn code_excludes_interactive_controls_without_guessing_class_names() {
|
||||
#[test]
|
||||
fn formulas_keep_their_order_across_output_boundaries() {
|
||||
let result = markdown(
|
||||
r"<p>$a_1$<img src='/plot.png' alt='Plot'>$b_2$<code>literal</code>\
|
||||
r"<p>$a_1$<img src='/plot.png' alt='Plot'>$b_2$<code>literal</code>
|
||||
$c_3$<a href='/target'>Link</a>$d_4$<br>$e_5$</p><blockquote>$f_6$</blockquote>",
|
||||
);
|
||||
let mut end = 0;
|
||||
@@ -573,3 +573,41 @@ fn mathml_operators_survive_commonmark_as_literal_math_text() {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn math_comments_literals_and_inline_markup_keep_distinct_boundaries() {
|
||||
use pulldown_cmark::{Event, Options as MarkdownOptions, Parser};
|
||||
for (source, expected) in [
|
||||
("<p>$a%b$</p>", "a"),
|
||||
("<p>$5%$</p>", "5"),
|
||||
("<p>$%c\nx$</p>", "x"),
|
||||
(r"<p>$\verb|50%|$</p>", r"\verb|50%|"),
|
||||
(r"<p>$\verb|a b|$</p>", r"\verb|a b|"),
|
||||
(r"<p>$\text{a<b}$</p>", r"\text{a<b}"),
|
||||
(r"<p>$\mbox{a<b}$</p>", r"\mbox{a<b}"),
|
||||
] {
|
||||
let result = markdown(source);
|
||||
let math: Vec<_> = Parser::new_ext(&result, MarkdownOptions::ENABLE_MATH)
|
||||
.filter_map(|e| match e {
|
||||
Event::InlineMath(x) => Some(x.into_string()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(math, vec![expected], "{source}: {result}");
|
||||
}
|
||||
for source in [
|
||||
"<p>$[a](b)$</p>",
|
||||
"<p>$x<a href='/d'>l</a>y$</p>",
|
||||
"<math><mi>$</mi><mn>5</mn><mo>-</mo><mi>$</mi><mn>3</mn></math>",
|
||||
] {
|
||||
let result = markdown(source);
|
||||
assert!(
|
||||
!Parser::new_ext(&result, MarkdownOptions::ENABLE_MATH)
|
||||
.any(|e| matches!(e, Event::InlineMath(_) | Event::DisplayMath(_))),
|
||||
"{result}"
|
||||
);
|
||||
if source.contains("href") {
|
||||
assert!(rendered_html(&result).contains("href=\"/d\""), "{result}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@
|
||||
"reason": "Escape list markers across text-node boundaries. Turndown escapes nodes individually and accidentally creates an ordered list."
|
||||
},
|
||||
"upstream_036": {
|
||||
"markdown": "![the\ntitle](logo.png \"the\ntitle\")",
|
||||
"markdown": "![the title](logo.png \"the title\")",
|
||||
"reason": "An image without alt uses its authored title as a text alternative. Turndown leaves that label empty."
|
||||
},
|
||||
"upstream_037": {
|
||||
|
||||
Reference in New Issue
Block a user