From bf487dcfff43cbca8bc88fc6e366a3aa061e0bdc Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:03:25 +0800 Subject: [PATCH] fix(html2md): preserve math payloads and literal dollar boundaries --- moli-html2md/src/math.rs | 73 ++++++++++++++++++- moli-html2md/src/mathml.rs | 5 +- moli-html2md/src/writer.rs | 2 +- moli-html2md/tests/inline_boundaries.rs | 6 +- moli-html2md/tests/regressions.rs | 4 +- moli-html2md/tests/review_semantics.rs | 40 +++++++++- moli-html2md/tests/turndown/expectations.json | 2 +- 7 files changed, 118 insertions(+), 14 deletions(-) diff --git a/moli-html2md/src/math.rs b/moli-html2md/src/math.rs index 3a97301c8d..975365ed80 100644 --- a/moli-html2md/src/math.rs +++ b/moli-html2md/src/math.rs @@ -138,7 +138,65 @@ pub(crate) fn normalized_span(text: &str) -> String { let mut chars = text[delimiter..text.len() - delimiter].chars().peekable(); let mut escaped = false; let mut space = false; + let mut text_depth = 0usize; while let Some(ch) = chars.next() { + if ch == '\\' && !escaped && chars.clone().take(4).eq("verb".chars()) { + let mut probe = chars.clone(); + for _ in 0..4 { + probe.next(); + } + let star = probe.peek() == Some(&'*'); + if star { + probe.next(); + } + if let Some(verb_delimiter) = probe + .next() + .filter(|c| !c.is_whitespace() && !c.is_ascii_alphabetic()) + { + if space && output.len() > delimiter { + output.push(' '); + } + space = false; + output.push_str(r"\verb"); + for _ in 0..4 { + chars.next(); + } + if star { + output.push('*'); + chars.next(); + } + output.push(chars.next().expect("verb delimiter")); + for c in chars.by_ref() { + output.push(c); + if c == verb_delimiter { + break; + } + } + continue; + } + } + if ch == '\\' && !escaped { + let command = if chars.clone().take(5).eq("text{".chars()) { + Some("text{") + } else if chars.clone().take(5).eq("mbox{".chars()) { + Some("mbox{") + } else { + None + }; + if let Some(command) = command { + if space && output.len() > delimiter { + output.push(' '); + } + space = false; + output.push('\\'); + output.push_str(command); + for _ in 0..command.len() { + chars.next(); + } + text_depth += 1; + continue; + } + } if ch == '%' && !escaped { for part in chars.by_ref() { if matches!(part, '\n' | '\r') { @@ -154,11 +212,21 @@ pub(crate) fn normalized_span(text: &str) -> String { continue; } if space { - output.push(' '); + if output.len() > delimiter { + output.push(' '); + } space = false; } output.push(ch); + if !escaped && text_depth > 0 { + if ch == '{' { + text_depth += 1; + } else if ch == '}' { + text_depth -= 1; + } + } if ch == '<' + && text_depth == 0 && chars .peek() .is_some_and(|next| next.is_ascii_alphabetic() || matches!(next, '/' | '!' | '?')) @@ -167,9 +235,6 @@ pub(crate) fn normalized_span(text: &str) -> String { } escaped = ch == '\\' && !escaped; } - if space { - output.push(' '); - } output.push_str(&text[text.len() - delimiter..]); output } diff --git a/moli-html2md/src/mathml.rs b/moli-html2md/src/mathml.rs index 3c4e88f5b2..0624a37fe9 100644 --- a/moli-html2md/src/mathml.rs +++ b/moli-html2md/src/mathml.rs @@ -1,4 +1,5 @@ use crate::{Dom, NodeKind}; +use std::fmt::Write as _; /// Math renderers often pair semantic MathML with an aria-hidden visual copy. /// Recognize that relationship from markup, independently of library classes. @@ -201,8 +202,8 @@ const ATTRIBUTES: &[&str] = &[ fn escape(text: &str, output: &mut String, markdown_text: bool) { for ch in text.chars() { match ch { - '\\' | '*' | '_' | '`' | '~' | '[' | ']' if markdown_text => { - output.push_str(&format!("&#{};", ch as u32)); + '\\' | '*' | '_' | '`' | '~' | '[' | ']' | '$' if markdown_text => { + write!(output, "&#{};", ch as u32).expect("writing to String"); } '&' => output.push_str("&"), '<' => output.push_str("<"), diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rs index 10bf66fdf4..e0ab958ad9 100644 --- a/moli-html2md/src/writer.rs +++ b/moli-html2md/src/writer.rs @@ -154,7 +154,7 @@ impl<'a> Writer<'a> { self.line_digits = Some(0); } match ch { - '\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '~' => { + '\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '~' | '$' => { self.output.push('\\'); self.output.push(ch); } diff --git a/moli-html2md/tests/inline_boundaries.rs b/moli-html2md/tests/inline_boundaries.rs index 07e5a9fb50..2ae28c7022 100644 --- a/moli-html2md/tests/inline_boundaries.rs +++ b/moli-html2md/tests/inline_boundaries.rs @@ -257,7 +257,7 @@ fn separate_nested_spans_keep_html_when_delimiters_are_ambiguous() { #[test] fn multiline_image_alt_preserves_the_image() { let result = markdown("first\n# heading"); - // Turndown emits the same Markdown. Matching it does not preserve the image. + // Alt newlines must not open a heading; the image and its text alternative survive. let actual_html = rendered_html(&result); let desired_html = "

\"first

\n"; assert_eq!( @@ -280,8 +280,8 @@ fn multiline_link_title_preserves_the_link() { #[test] fn multiple_nested_spans_keep_their_styles_at_paragraph_end() { - // This predates the closing-edge fix: with no following character there - // is no punctuation fallback, but sibling delimiters can still mispair. + // This guards the closing-edge fix: with no following character there + // is no punctuation fallback, but sibling delimiters must not mispair. let result = markdown("xbc"); let actual_html = rendered_html(&result); let desired_html = "

xbc

\n"; diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index 5cd3134bfb..17131b8b6a 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -825,9 +825,9 @@ fn math_recognition_does_not_reinterpret_prices_or_literal_code() { r"

Pay $5 for a_b and $10 for c_d; $x_i$; $unfinished_name

", false, ); - assert!(result.contains(r"$5 for a\_b and $10 for c\_d"), "{result}"); + assert!(result.contains(r"\$5 for a\_b and \$10 for c\_d"), "{result}"); assert!(result.contains("`$x_i$`"), "{result}"); - assert!(result.contains(r"$unfinished\_name"), "{result}"); + assert!(result.contains(r"\$unfinished\_name"), "{result}"); let result = markdown(r"

括号 \(x_i + \alpha\)；方括号 \[y_1=2\]。

", false); assert!(result.contains(r"\(x_i + \alpha\)"), "{result}"); assert!(result.contains(r"\[y_1=2\]"), "{result}"); diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs index 9f00f3e891..01c971c4f9 100644 --- a/moli-html2md/tests/review_semantics.rs +++ b/moli-html2md/tests/review_semantics.rs @@ -167,7 +167,7 @@ fn code_excludes_interactive_controls_without_guessing_class_names() { #[test] fn formulas_keep_their_order_across_output_boundaries() { let result = markdown( - r"

$a_1$Plot$b_2$literal\ + r"

$a_1$Plot$b_2$literal $c_3$Link$d_4$
$e_5$

$f_6$
", ); let mut end = 0; @@ -573,3 +573,41 @@ fn mathml_operators_survive_commonmark_as_literal_math_text() { } } + +#[test] +fn math_comments_literals_and_inline_markup_keep_distinct_boundaries() { + use pulldown_cmark::{Event, Options as MarkdownOptions, Parser}; + for (source, expected) in [ + ("

$a%b$

", "a"), + ("

$5%$

", "5"), + ("

$%c\nx$

", "x"), + (r"

$\verb|50%|$

", r"\verb|50%|"), + (r"

$\verb|a b|$

", r"\verb|a b|"), + (r"

$\text{a<b}$

", r"\text{a$\mbox{a<b}$

", r"\mbox{a = Parser::new_ext(&result, MarkdownOptions::ENABLE_MATH) + .filter_map(|e| match e { + Event::InlineMath(x) => Some(x.into_string()), + _ => None, + }) + .collect(); + assert_eq!(math, vec![expected], "{source}: {result}"); + } + for source in [ + "

$[a](b)$

", + "

$xly$

", + "$5-$3", + ] { + let result = markdown(source); + assert!( + !Parser::new_ext(&result, MarkdownOptions::ENABLE_MATH) + .any(|e| matches!(e, Event::InlineMath(_) | Event::DisplayMath(_))), + "{result}" + ); + if source.contains("href") { + assert!(rendered_html(&result).contains("href=\"/d\""), "{result}"); + } + } +} diff --git a/moli-html2md/tests/turndown/expectations.json b/moli-html2md/tests/turndown/expectations.json index 2944b9671b..fceba657a5 100644 --- a/moli-html2md/tests/turndown/expectations.json +++ b/moli-html2md/tests/turndown/expectations.json @@ -52,7 +52,7 @@ "reason": "Escape list markers across text-node boundaries. Turndown escapes nodes individually and accidentally creates an ordered list." }, "upstream_036": { - "markdown": "![the\ntitle](logo.png \"the\ntitle\")", + "markdown": "![the title](logo.png \"the title\")", "reason": "An image without alt uses its authored title as a text alternative. Turndown leaves that label empty." }, "upstream_037": {