use moli_html2md::{Converter, Dom, NodeKind, Options, convert}; // A second DOM implementation, independent of NativeDom or an HTML parser. // Its stable arena IDs and borrowed strings are enough for the converter. #[derive(Clone, Debug, PartialEq, Eq)] struct Node { kind: NodeKind<'static>, attrs: Vec<(&'static str, &'static str)>, first: Option, last: Option, next: Option, } #[derive(Clone, Debug, PartialEq, Eq)] struct Tree(Vec); impl Tree { fn new() -> Self { Self(vec![Node { kind: NodeKind::Document, attrs: Vec::new(), first: None, last: None, next: None, }]) } fn add(&mut self, parent: usize, kind: NodeKind<'static>) -> usize { let id = self.0.len(); self.0.push(Node { kind, attrs: Vec::new(), first: None, last: None, next: None, }); if let Some(previous) = self.0[parent].last { self.0[previous].next = Some(id); } else { self.0[parent].first = Some(id); } self.0[parent].last = Some(id); id } fn element(&mut self, parent: usize, tag: &'static str) -> usize { self.add(parent, NodeKind::Element(tag)) } fn text(&mut self, parent: usize, text: &'static str) { self.add(parent, NodeKind::Text(text)); } fn attr(&mut self, node: usize, name: &'static str, value: &'static str) { self.0[node].attrs.push((name, value)); } fn leaf(&mut self, parent: usize, tag: &'static str, text: &'static str) -> usize { let element = self.element(parent, tag); self.text(element, text); element } } impl Dom for Tree { type NodeId = usize; fn node_kind(&self, node: usize) -> NodeKind<'_> { self.0[node].kind } fn first_child(&self, node: usize) -> Option { self.0[node].first } fn next_sibling(&self, node: usize) -> Option { self.0[node].next } fn attribute(&self, node: usize, name: &str) -> Option<&str> { self.0[node] .attrs .iter() .find_map(|(key, value)| (*key == name).then_some(*value)) } } #[test] fn hidden_page_state_is_omitted_without_losing_inactive_tab_content() { let mut dom = Tree::new(); let page = dom.element(0, "main"); dom.leaf(page, "h1", "Article title"); let state: &'static str = Box::leak(format!("{{\"state\":\"{}\"}}", "x".repeat(512)).into_boxed_str()); let hidden = dom.leaf(page, "div", state); dom.attr(hidden, "style", "color:red; DISPLAY : none !important"); let hidden_attribute = dom.leaf(page, "p", "Collapsed details"); dom.attr(hidden_attribute, "hidden", ""); let visible_override = dom.leaf(page, "p", "Visible details"); dom.attr(visible_override, "style", "display:none; display:block"); let important_hidden = dom.leaf(page, "p", "Important hidden details"); dom.attr( important_hidden, "style", "display:none!important; display:block", ); let table = dom.element(page, "table"); let header = dom.element(table, "tr"); dom.leaf(header, "th", "Name"); let hidden_row = dom.element(table, "tr"); dom.attr(hidden_row, "hidden", ""); dom.leaf(hidden_row, "td", "Hidden row"); let row = dom.element(table, "tr"); dom.leaf(row, "td", "Visible row"); let text = Converter::default().convert(&dom, 0); assert!(text.contains("Article title")); assert!(text.contains("Visible details")); assert!(text.contains("Visible row")); assert!(!text.contains(state)); assert!(text.contains("Collapsed details")); assert!(text.contains("Important hidden details")); assert!(text.contains("Hidden row")); } #[test] fn coalesces_styles_without_changing_the_tree() { let mut dom = Tree::new(); dom.leaf(0, "em", "foo"); dom.leaf(0, "i", "bar"); dom.text(0, " "); let bold = dom.element(0, "strong"); dom.leaf(bold, "b", "nested"); dom.leaf(0, "strong", "!"); let before = dom.clone(); let converter = Converter::default(); for _ in 0..2 { assert_eq!(converter.convert(&dom, 0), "*foobar* **nested!**"); assert_eq!(dom, before); } } #[test] fn moves_whitespace_outside_formatting_but_keeps_nonbreaking_spaces() { let mut dom = Tree::new(); dom.text(0, "by"); dom.leaf(0, "strong", " \t"); dom.leaf(0, "small", "Albert Einstein"); dom.text(0, "\u{a0}"); let em = dom.element(0, "em"); dom.leaf(em, "strong", " next "); dom.text(0, "word"); assert_eq!(convert(&dom, 0), "by Albert Einstein\u{a0} ***next*** word"); } #[test] fn keeps_separate_links_and_escapes_destinations_and_titles() { let mut dom = Tree::new(); for label in ["one", "two"] { let link = dom.leaf(0, "a", label); dom.attr(link, "href", "/a(b) c"); dom.attr(link, "title", "a \"title\""); } assert_eq!( convert(&dom, 0), "[one](/a\\(b\\)%20c \"a \\\"title\\\"\")[two](/a\\(b\\)%20c \"a \\\"title\\\"\")" ); } #[test] fn escapes_literal_markdown_and_decoded_entities_across_text_nodes() { let mut dom = Tree::new(); let p = dom.element(0, "p"); dom.text(p, "12"); dom.text(p, ". [x] *y* _z_ `a` ©"); dom.leaf(0, "p", "# heading"); dom.leaf(0, "p", "- item"); dom.text(0, "!"); let link = dom.leaf(0, "a", "link"); dom.attr(link, "href", "/"); assert_eq!( convert(&dom, 0), "12\\. \\[x\\] \\*y\\* \\_z\\_ \\`a\\` \\ &copy;\n\n\\# heading\n\n\\- item\n\n\\![link](/)" ); } #[test] fn keeps_adjacent_code_elements_distinct_and_chooses_safe_delimiters() { let mut dom = Tree::new(); dom.leaf(0, "code", "a`"); dom.leaf(0, "code", "`b"); dom.text(0, " "); dom.leaf(0, "code", " ` "); dom.text(0, "after"); assert_eq!( rendered_html(&convert(&dom, 0)), "

a``b ` after

\n" ); } #[test] fn empty_text_nodes_do_not_separate_code_values() { let mut dom = Tree::new(); dom.leaf(0, "code", "name"); dom.text(0, ""); dom.leaf(0, "code", "string"); dom.text(0, ""); dom.text(0, ""); dom.leaf(0, "code", "third"); for preformatted_code in [false, true] { let result = Converter::new(Options { preformatted_code, ..Options::default() }) .convert(&dom, 0); assert_eq!( rendered_html(&result), "

namestringthird

\n", "preformatted={preformatted_code}: {result}" ); } } #[test] fn preserves_preformatted_text_including_blank_lines_and_nested_elements() { let mut dom = Tree::new(); let pre = dom.element(0, "pre"); let code = dom.element(pre, "code"); dom.attr(code, "class", "highlight language-rust"); dom.text(code, " first\r\n\r\n"); dom.leaf(code, "span", "```\n"); dom.text(code, "\n"); assert_eq!(convert(&dom, 0), "````rust\n first\n\n```\n\n````"); } #[test] fn prefixes_multiline_list_items_and_quotes() { let mut dom = Tree::new(); let quote = dom.element(0, "blockquote"); let list = dom.element(quote, "ol"); dom.attr(list, "start", "9"); let item = dom.element(list, "li"); dom.leaf(item, "p", "first"); dom.leaf(item, "p", "second"); dom.leaf(list, "li", "last"); assert_eq!( convert(&dom, 0), "> 9. first\n>\n> second\n>\n> 10. last" ); } #[test] fn supports_headings_images_and_non_content_elements() { let mut dom = Tree::new(); dom.leaf(0, "h2", "Heading"); let image = dom.element(0, "img"); dom.attr(image, "src", "/image.png"); dom.attr(image, "alt", "[image]"); dom.element(0, "hr"); for tag in ["head", "script", "style", "noscript", "template"] { dom.leaf(0, tag, "invisible"); } dom.add(0, NodeKind::Other); dom.leaf(0, "p", "visible"); assert_eq!( convert(&dom, 0), "## Heading\n\n![\\[image\\]](/image.png)\n\n---\n\nvisible" ); } #[test] fn adds_empty_headers_to_headerless_tables() { let mut dom = Tree::new(); let table = dom.element(0, "table"); let row = dom.element(table, "tr"); dom.leaf(row, "td", "one"); dom.leaf(row, "td", "two"); assert_eq!(convert(&dom, 0), "| | |\n| --- | --- |\n| one | two |"); } #[test] fn preserves_table_cell_boundaries_hard_breaks_and_literal_pipes() { let mut dom = Tree::new(); let table = dom.element(0, "table"); let header = dom.element(table, "tr"); let cell = dom.leaf(header, "th", "Key"); dom.attr(cell, "align", "right"); let row = dom.element(table, "tr"); let cell = dom.element(row, "td"); dom.text(cell, "a|b"); dom.element(cell, "br"); dom.text(cell, "c"); assert_eq!(convert(&dom, 0), "| Key |\n| ---: |\n| a\\|b
c |"); } #[test] fn table_spans_do_not_duplicate_cell_text() { let mut dom = Tree::new(); let table = dom.element(0, "table"); let row = dom.element(table, "tr"); let cell = dom.leaf(row, "td", "one"); dom.attr(cell, "rowspan", "2"); dom.leaf(row, "td", "two"); let row = dom.element(table, "tr"); let cell = dom.leaf(row, "td", "end"); dom.attr(cell, "colspan", "999999999999"); assert_eq!(convert(&dom, 0), "one\n\ntwo\n\nend"); } #[test] fn bounds_every_traversal_including_raw_code() { for tag in [ "div", "span", "strong", "blockquote", "table", "pre", "code", ] { let mut dom = Tree::new(); dom.text(0, "visible"); let mut node = 0; for _ in 0..30 { node = dom.element(node, tag); } dom.text(node, "too deep"); let converter = Converter::new(Options { max_depth: 10, ..Options::default() }); let result = converter.convert(&dom, 0); assert!(!result.contains("too deep"), "{tag}: {result}"); assert!(result.starts_with("visible"), "{tag}: {result}"); } } #[test] fn converts_deep_trees_on_a_small_thread_stack() { std::thread::Builder::new() .stack_size(64 * 1024) .spawn(|| { for tag in ["span", "strong", "pre"] { let mut dom = Tree::new(); let mut node = 0; for _ in 0..20_000 { node = dom.element(node, tag); } dom.text(node, "deep"); let converter = Converter::new(Options { max_depth: usize::MAX, ..Options::default() }); let expected = match tag { "strong" => "**deep**", "pre" => "```\ndeep\n```", _ => "deep", }; assert_eq!(converter.convert(&dom, 0), expected); } }) .expect("spawn small-stack test") .join() .expect("iterative conversion should complete"); } #[test] fn converts_deep_tables_on_a_small_thread_stack() { std::thread::Builder::new() .stack_size(64 * 1024) .spawn(|| { let mut dom = Tree::new(); let mut node = 0; for _ in 0..10_000 { node = dom.element(node, "table"); node = dom.element(node, "tr"); node = dom.element(node, "td"); } dom.text(node, "deep"); let converter = Converter::new(Options { max_depth: usize::MAX, ..Options::default() }); assert_eq!(converter.convert(&dom, 0), "| |\n| --- |\n| deep |"); }) .expect("spawn small-stack test") .join() .expect("nested tables should not recurse"); } #[test] fn converts_nested_header_tables_on_a_small_thread_stack() { std::thread::Builder::new() .stack_size(64 * 1024) .spawn(|| { let mut dom = Tree::new(); let mut node = 0; for _ in 0..1_000 { let table = dom.element(node, "table"); let row = dom.element(table, "tr"); dom.leaf(row, "th", "Header"); let row = dom.element(table, "tr"); node = dom.element(row, "td"); } dom.text(node, "deep"); let converter = Converter::new(Options { max_depth: usize::MAX, ..Options::default() }); let actual = converter.convert(&dom, 0); assert_eq!(actual.matches("Header").count(), 1_000); assert_eq!(actual.matches("| --- |").count(), 1); assert!(actual.ends_with("| deep |")); }) .expect("spawn small-stack test") .join() .expect("header table conversion should not recurse"); } #[test] fn conversion_is_limited_to_the_supplied_subtree() { let mut dom = Tree::new(); let first = dom.leaf(0, "p", "first"); dom.leaf(0, "p", "second"); assert_eq!(convert(&dom, first), "first"); let converter = Converter::new(Options { max_depth: 0, ..Options::default() }); assert_eq!(converter.convert(&dom, first), ""); } fn rendered_html(markdown: &str) -> String { let parser = pulldown_cmark::Parser::new_ext( markdown, pulldown_cmark::Options::ENABLE_TABLES | pulldown_cmark::Options::ENABLE_STRIKETHROUGH, ); let mut html = String::new(); pulldown_cmark::html::push_html(&mut html, parser); html } #[test] fn markdown_parser_recovers_adjacent_and_nested_formatting() { let mut dom = Tree::new(); dom.leaf(0, "em", "foo"); dom.leaf(0, "em", "bar"); dom.text(0, " "); let strong = dom.element(0, "strong"); dom.leaf(strong, "em", "both"); dom.text(0, " "); dom.leaf(0, "del", "deleted"); assert_eq!( rendered_html(&convert(&dom, 0)), "

foobar both deleted

\n" ); } #[test] fn markdown_parser_recovers_unicode_spaces_at_emphasis_boundaries() { let mut dom = Tree::new(); dom.leaf(0, "em", "\u{a0} a \u{2003}"); dom.text(0, "b"); assert_eq!( rendered_html(&convert(&dom, 0)), "

\u{a0} a \u{2003}b

\n" ); let mut dom = Tree::new(); dom.leaf(0, "p", "\u{a0} a"); assert_eq!(convert(&dom, 0), "\u{a0} a"); } #[test] fn markdown_parser_recovers_intraword_emphasis_next_to_punctuation() { for (tag, html_tag) in [("strong", "strong"), ("em", "em"), ("del", "del")] { for text in ["(word)", "word!", "!word", "日本語。"] { let mut dom = Tree::new(); dom.text(0, "before"); dom.leaf(0, tag, text); dom.text(0, "after"); let markdown = convert(&dom, 0); assert_eq!( rendered_html(&markdown), format!("

before<{html_tag}>{text}after

\n"), "{markdown}" ); } } } #[test] fn markdown_parser_recovers_literal_hashes_at_the_end_of_headings() { let mut dom = Tree::new(); dom.leaf(0, "h1", "title #"); assert_eq!(rendered_html(&convert(&dom, 0)), "

title #

\n"); } #[test] fn markdown_parser_recovers_pipes_and_preformatted_code_inside_tables() { let mut dom = Tree::new(); let table = dom.element(0, "table"); let header = dom.element(table, "tr"); dom.leaf(header, "th", "Code"); let row = dom.element(table, "tr"); let cell = dom.element(row, "td"); dom.leaf(cell, "code", "a|b"); let row = dom.element(table, "tr"); let cell = dom.element(row, "td"); dom.leaf(cell, "pre", " |b\nnext\n"); let markdown = convert(&dom, 0); let html = rendered_html(&markdown); assert_eq!( html, "

Code

\n

a|b

\n
  <a>|b\nnext\n
\n", "{markdown}" ); }