diff --git a/moli-html2md/src/form.rs b/moli-html2md/src/form.rs index 418e5fc572..857b56d0d7 100644 --- a/moli-html2md/src/form.rs +++ b/moli-html2md/src/form.rs @@ -41,3 +41,12 @@ pub(crate) fn input_text<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Op }); value.map(Cow::Borrowed) } + +pub(crate) fn textarea_text<'a, D: Dom + ?Sized>( + dom: &'a D, + node: D::NodeId, +) -> Option> { + dom.attribute(node, "value") + .filter(|value| !value.is_empty()) + .map(Cow::Borrowed) +} diff --git a/moli-html2md/src/html_table.rs b/moli-html2md/src/html_table.rs index dbcc9ecb1b..f08f59ec3b 100644 --- a/moli-html2md/src/html_table.rs +++ b/moli-html2md/src/html_table.rs @@ -145,6 +145,13 @@ pub(crate) fn render( escape(&value, &mut output); } } + NodeKind::Element("textarea") => { + if let Some(value) = crate::form::textarea_text(dom, node) { + escape(&value, &mut output); + } else { + tasks.push(Task::Siblings(dom.first_child(node), depth + 1)); + } + } NodeKind::Element(tag) => { if crate::visibility::nonrendered_serialized_state(dom, node) { continue; @@ -256,6 +263,7 @@ fn allowed(tag: &str) -> bool { | "optgroup" | "option" | "button" + | "textarea" | "p" | "div" | "span" diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index d1873b819c..9131672da1 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -339,6 +339,12 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } return; } + "textarea" => { + if let Some(value) = crate::form::textarea_text(self.dom, node) { + self.writer().text(&value); + return; + } + } "iframe" | "video" | "audio" => { if self.media(node, tag, depth) { return; diff --git a/moli-html2md/src/math.rs b/moli-html2md/src/math.rs index 668a29a26c..c6bd6ad137 100644 --- a/moli-html2md/src/math.rs +++ b/moli-html2md/src/math.rs @@ -2,15 +2,22 @@ /// Dollar boundaries follow the usual non-space/non-digit rules, so prices do /// not consume the prose between two currency amounts. pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> { + let mut steps = 0; + next_span_counted(text, &mut steps) +} + +fn next_span_counted(text: &str, steps: &mut usize) -> Option<(usize, usize)> { let bytes = text.as_bytes(); let mut index = 0; + let mut backslashes = 0; let mut dollar = None; let mut double_dollar = None; let mut paren = None; let mut bracket = None; let mut candidate = None; while index < bytes.len() { - if bytes[index] == b'$' && !escaped(bytes, index) { + *steps += 1; + if bytes[index] == b'$' && backslashes % 2 == 0 { if bytes.get(index + 1) == Some(&b'$') { if let Some(start) = double_dollar.take() { accept_candidate(text, start, index + 2, 2, false, &mut candidate); @@ -18,6 +25,7 @@ pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> { double_dollar = Some(index); } index += 2; + backslashes = 0; if let Some(ready) = ready_candidate(candidate, [dollar, double_dollar, paren, bracket]) { @@ -35,13 +43,14 @@ pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> { dollar = Some(index); } index += 1; + backslashes = 0; if let Some(ready) = ready_candidate(candidate, [dollar, double_dollar, paren, bracket]) { return Some(ready); } continue; } - if bytes[index] == b'\\' && !escaped(bytes, index) { + if bytes[index] == b'\\' && backslashes % 2 == 0 { match bytes.get(index + 1) { Some(b'(') => { paren.get_or_insert(index); @@ -63,6 +72,7 @@ pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> { } if matches!(bytes.get(index + 1), Some(b'(' | b'[' | b')' | b']')) { index += 2; + backslashes = 0; if let Some(ready) = ready_candidate(candidate, [dollar, double_dollar, paren, bracket]) { @@ -71,6 +81,11 @@ pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> { continue; } } + if bytes[index] == b'\\' { + backslashes += 1; + } else { + backslashes = 0; + } index += 1; } candidate @@ -124,12 +139,20 @@ fn contains_markdown_resource(text: &str) -> bool { text.contains("](") || text.contains("][") || text.contains("![[") } -fn escaped(bytes: &[u8], index: usize) -> bool { - bytes[..index] - .iter() - .rev() - .take_while(|&&ch| ch == b'\\') - .count() - % 2 - == 1 +#[cfg(test)] +mod tests { + use super::next_span_counted; + + #[test] + fn math_scan_work_is_linear_for_backslashes_and_unclosed_delimiters() { + for text in ["\\\\".repeat(8_192), "\\(x ".repeat(4_096)] { + let mut steps = 0; + let _ = next_span_counted(&text, &mut steps); + assert!( + steps <= text.len(), + "{steps} steps for {} bytes", + text.len() + ); + } + } } diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index e485a2d74f..7de3d571df 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -1064,7 +1064,7 @@ fn block_content_inside_heading_keeps_its_own_boundary() { #[test] fn form_values_preserve_current_readable_state_without_secrets() { - let source = "
PeriodMonSun
Morning

"; + let source = "
PeriodMonSun
Morning

"; let result = rendered_html(&markdown(source, false)); for value in [ "Morning", @@ -1075,6 +1075,7 @@ fn form_values_preserve_current_readable_state_without_secrets() { "Search", "Map button", "Password", + "Notes", ] { assert!(result.contains(value), "missing {value}: {result}"); } @@ -1096,6 +1097,20 @@ fn unsafe_resource_urls_keep_labels_without_emitting_active_links() { assert!(!result.contains("javascript:"), "{result}"); } +#[test] +fn embedded_media_uses_the_same_url_policy_in_all_output_paths() { + let media = ""; + for source in [ + media.to_owned(), + format!("
{media}
"), + ] { + let result = markdown(&source, false); + assert!(result.contains("/clip.mp4"), "{result}"); + assert!(!result.contains("javascript:"), "{result}"); + assert!(!result.contains("data:image"), "{result}"); + } +} + #[test] fn complex_table_preserves_checkbox_state() { let source = "
Status
"; diff --git a/moli-renderer-v8/src/runtime/page_dump/visibility.rs b/moli-renderer-v8/src/runtime/page_dump/visibility.rs index 41e67f752d..881b951d85 100644 --- a/moli-renderer-v8/src/runtime/page_dump/visibility.rs +++ b/moli-renderer-v8/src/runtime/page_dump/visibility.rs @@ -85,6 +85,7 @@ impl<'a> MarkdownDom<'a> { .flatten() }) .collect(); + let visible_handler_functions = visible_handler_functions(dom); for (node, _) in &styles { if (Dom::attribute(dom, *node, "aria-expanded").is_some() || Dom::attribute(dom, *node, "role") @@ -97,6 +98,19 @@ impl<'a> MarkdownDom<'a> { .filter(|id| target_is_disclosure(dom, &targets, id)), ); } + for attribute_name in dom.get_attribute_names(*node).unwrap_or_default() { + if !attribute_name.starts_with("on") { + continue; + } + let Some(handler) = Dom::attribute(dom, *node, &attribute_name) else { + continue; + }; + for target in inline_disclosure_targets(handler, &visible_handler_functions) { + if target_is_disclosure(dom, &targets, target) { + disclosures.insert(target); + } + } + } // Some pages pair a shortened paragraph with an explicitly linked // hidden full-text copy. Keep the complete copy once; an unrelated // dialog or a matching paragraph elsewhere is not such a pair. @@ -181,19 +195,23 @@ impl<'a> MarkdownDom<'a> { let mut visibility_restored_regions = HashSet::new(); for (node, values) in styles { let role = Dom::attribute(dom, node, "role").unwrap_or_default(); - if matches!(Dom::node_kind(dom, node), NodeKind::Element("input")) - && let Some(element) = dom.node(node).and_then(|node| node.as_element()) + if matches!( + Dom::node_kind(dom, node), + NodeKind::Element("input" | "textarea") + ) && let Some(element) = dom.node(node).and_then(|node| node.as_element()) { resolved_urls .entry(node) .or_insert_with(HashMap::new) .insert("value".to_owned(), element.input_value()); - if element.checked() { + if matches!(Dom::node_kind(dom, node), NodeKind::Element("input")) + && element.checked() + { resolved_urls .entry(node) .or_insert_with(HashMap::new) .insert("checked".to_owned(), String::new()); - } else { + } else if matches!(Dom::node_kind(dom, node), NodeKind::Element("input")) { unchecked_controls.insert(node); } } @@ -404,6 +422,343 @@ fn target_is_disclosure( }) } +fn inline_disclosure_targets<'a>( + value: &'a str, + visible_functions: &HashSet, +) -> Vec<&'a str> { + let mut result = Vec::new(); + for (marker, fragment) in [ + ("document.getElementById", false), + ("document.querySelector", true), + ] { + for offset in code_marker_offsets(value, marker) { + let remaining = &value[offset + marker.len()..]; + let Some((literal, end)) = quoted_call_argument(remaining) else { + continue; + }; + let Some(target) = literal + .strip_prefix('#') + .or_else(|| (!fragment).then_some(literal)) + .filter(|target| !target.is_empty()) + else { + continue; + }; + let direct_mutation = sets_display_visible(&remaining[end..], None); + let named_mutation = enclosing_function_name(&value[..offset]) + .is_some_and(|name| visible_functions.contains(name)); + if direct_mutation || named_mutation { + result.push(target); + } + } + } + result +} + +fn visible_handler_functions(dom: &NativeDom) -> HashSet { + let mut result = HashSet::new(); + let mut pending = vec![dom.document_node_id()]; + while let Some(node) = pending.pop() { + if !matches!(Dom::node_kind(dom, node), NodeKind::Element("script")) { + pending.extend(dom.child_ids(node)); + continue; + } + let source = descendant_text(dom, node); + for offset in code_marker_offsets(&source, "function") { + let signature = &source[offset + "function".len()..]; + let Some((name, parameter, body)) = function_parts(signature) else { + continue; + }; + if function_sets_parameter_visible(body, parameter) { + result.insert(name.to_owned()); + } + } + } + result +} + +fn function_parts(source: &str) -> Option<(&str, &str, &str)> { + let source = source.trim_start(); + let name_end = source.find(|character: char| !is_identifier(character))?; + let name = &source[..name_end]; + if name.is_empty() { + return None; + } + let parameters = source[name_end..].trim_start().strip_prefix('(')?; + let close = parameters.find(')')?; + let parameter = parameters[..close].trim(); + if parameter.is_empty() || parameter.contains(',') || !parameter.chars().all(is_identifier) { + return None; + } + let rest = parameters[close + 1..].trim_start(); + let body_start = rest.find('{')?; + let body_end = matching_delimiter(rest, body_start, '{', '}')?; + Some((name, parameter, &rest[body_start + 1..body_end])) +} + +fn function_sets_parameter_visible(body: &str, parameter: &str) -> bool { + for offset in code_marker_offsets(body, ".style.display") { + let owner = body[..offset] + .trim_end() + .rsplit_once(|character: char| !is_identifier(character)) + .map_or(body[..offset].trim_end(), |(_, owner)| owner); + if owner.is_empty() + || !sets_display_visible(&body[offset + ".style.display".len()..], Some("")) + { + continue; + } + if owner == parameter || alias_depends_on_parameter(&body[..offset], owner, parameter) { + return true; + } + } + false +} + +fn alias_depends_on_parameter(prefix: &str, owner: &str, parameter: &str) -> bool { + let Some(offset) = code_marker_offsets(prefix, owner) + .into_iter() + .rev() + .find(|offset| { + let before = prefix[..*offset].chars().next_back(); + let after = prefix[*offset + owner.len()..].chars().next(); + before.is_none_or(|character| !is_identifier(character)) + && after.is_none_or(|character| !is_identifier(character)) + && prefix[*offset + owner.len()..] + .trim_start() + .starts_with('=') + }) + else { + return false; + }; + let assignment = prefix[offset + owner.len()..].trim_start(); + let Some(right) = assignment.strip_prefix('=') else { + return false; + }; + let right = right.split_once(';').map_or(right, |(right, _)| right); + contains_identifier(right, parameter) +} + +fn contains_identifier(value: &str, identifier: &str) -> bool { + code_marker_offsets(value, identifier) + .into_iter() + .any(|offset| { + let before = value[..offset].chars().next_back(); + let after = value[offset + identifier.len()..].chars().next(); + before.is_none_or(|character| !is_identifier(character)) + && after.is_none_or(|character| !is_identifier(character)) + }) +} + +fn enclosing_function_name(value: &str) -> Option<&str> { + let value = value.trim_end(); + let value = value.strip_suffix('(')?.trim_end(); + let start = value + .rfind(|character: char| !is_identifier(character)) + .map_or(0, |offset| offset + 1); + let name = &value[start..]; + (!name.is_empty()).then_some(name) +} + +fn quoted_call_argument(value: &str) -> Option<(&str, usize)> { + let leading = value.len() - value.trim_start().len(); + let value = value.trim_start().strip_prefix('(')?; + let after_open = value.len() - value.trim_start().len(); + let value = value.trim_start(); + let quote = value + .chars() + .next() + .filter(|quote| matches!(quote, '\'' | '"'))?; + let literal = &value[quote.len_utf8()..]; + let end = literal.find(quote)?; + let after_literal = literal[end + quote.len_utf8()..].trim_start(); + if !after_literal.starts_with(')') { + return None; + } + let consumed = leading + + 1 + + after_open + + quote.len_utf8() + + end + + quote.len_utf8() + + (literal[end + quote.len_utf8()..].len() - after_literal.len()) + + 1; + Some((&literal[..end], consumed)) +} + +fn sets_display_visible(value: &str, display_already_consumed: Option<&str>) -> bool { + let value = if display_already_consumed.is_some() { + value + } else { + let Some(value) = value.trim_start().strip_prefix(".style.display") else { + return false; + }; + value + }; + let Some(value) = value.trim_start().strip_prefix('=') else { + return false; + }; + let value = value.trim_start(); + let Some(quote) = value + .chars() + .next() + .filter(|quote| matches!(quote, '\'' | '"')) + else { + return false; + }; + let literal = &value[quote.len_utf8()..]; + let Some(end) = literal.find(quote) else { + return false; + }; + matches!( + literal[..end].trim(), + "" | "block" + | "inline" + | "inline-block" + | "flex" + | "inline-flex" + | "grid" + | "inline-grid" + | "list-item" + | "table" + | "table-row" + | "table-cell" + ) +} + +fn code_marker_offsets(value: &str, marker: &str) -> Vec { + let mut result = Vec::new(); + let mut offset = 0; + let mut quote = None; + let mut escaped = false; + let mut line_comment = false; + let mut block_comment = false; + while offset < value.len() { + let tail = &value[offset..]; + if line_comment { + if tail.starts_with('\n') { + line_comment = false; + } + } else if block_comment { + if tail.starts_with("*/") { + block_comment = false; + offset += 2; + continue; + } + } else if let Some(delimiter) = quote { + let character = tail.chars().next().expect("offset is within value"); + if escaped { + escaped = false; + } else if character == '\\' { + escaped = true; + } else if character == delimiter { + quote = None; + } + } else if tail.starts_with("//") { + line_comment = true; + offset += 2; + continue; + } else if tail.starts_with("/*") { + block_comment = true; + offset += 2; + continue; + } else { + let character = tail.chars().next().expect("offset is within value"); + if matches!(character, '\'' | '"' | '`') { + quote = Some(character); + } else if tail.starts_with(marker) { + result.push(offset); + offset += marker.len(); + continue; + } + } + offset += tail + .chars() + .next() + .expect("offset is within value") + .len_utf8(); + } + result +} + +fn matching_delimiter(value: &str, start: usize, open: char, close: char) -> Option { + let mut depth = 0; + let mut quote = None; + let mut escaped = false; + let mut line_comment = false; + let mut block_comment = false; + let mut offset = start; + while offset < value.len() { + let tail = &value[offset..]; + if line_comment { + if tail.starts_with('\n') { + line_comment = false; + } + } else if block_comment { + if tail.starts_with("*/") { + block_comment = false; + offset += 2; + continue; + } + } else if let Some(delimiter) = quote { + let character = tail.chars().next().expect("offset is within value"); + if escaped { + escaped = false; + } else if character == '\\' { + escaped = true; + } else if character == delimiter { + quote = None; + } + } else if tail.starts_with("//") { + line_comment = true; + offset += 2; + continue; + } else if tail.starts_with("/*") { + block_comment = true; + offset += 2; + continue; + } else { + let character = tail.chars().next().expect("offset is within value"); + if matches!(character, '\'' | '"' | '`') { + quote = Some(character); + } else if character == open { + depth += 1; + } else if character == close { + depth -= 1; + if depth == 0 { + return Some(offset); + } + } + } + offset += tail + .chars() + .next() + .expect("offset is within value") + .len_utf8(); + } + None +} + +fn is_identifier(character: char) -> bool { + character == '_' || character == '$' || character.is_ascii_alphanumeric() +} + +fn descendant_text(dom: &NativeDom, root: NativeNodeId) -> String { + let mut pending = vec![root]; + let mut result = String::new(); + while let Some(node) = pending.pop() { + if let NodeKind::Text(value) = Dom::node_kind(dom, node) { + result.push_str(value); + } + let mut children = Vec::new(); + let mut child = dom.first_child(node); + while let Some(id) = child { + children.push(id); + child = dom.next_sibling(id); + } + pending.extend(children.into_iter().rev()); + } + result +} + fn px(value: &str) -> Option { value.strip_suffix("px")?.trim().parse().ok() } diff --git a/moli-renderer-v8/src/runtime/page_vm/tests/markdown_dump.rs b/moli-renderer-v8/src/runtime/page_vm/tests/markdown_dump.rs index 15500cf383..4de83bb0a2 100644 --- a/moli-renderer-v8/src/runtime/page_vm/tests/markdown_dump.rs +++ b/moli-renderer-v8/src/runtime/page_vm/tests/markdown_dump.rs @@ -100,6 +100,14 @@ async fn markdown_uses_live_visibility_and_preserves_disclosure_content() { let actions = page.render_page_dump(options(false)); assert!(actions.contains("[Accept](https://example.test/accept)\n\n[Reject](https://example.test/reject)"), "{actions}"); page.vm_mut().eval(r##" + document.head.innerHTML = ``; document.body.innerHTML = `

A detailed review starts... Read more

@@ -147,14 +155,23 @@ async fn markdown_uses_live_visibility_and_preserves_disclosure_content() { - `; + + + + + + + `; "##).unwrap(); let related = page.render_page_dump(options(false)); assert!(!related.contains("Hidden telemetry state"), "{related}"); assert!(!related.contains("Hidden diagnostic state"), "{related}"); assert!(!related.contains("Hidden read-only state"), "{related}"); assert!(!related.contains("Hidden commented state"), "{related}"); - assert!(!related.contains("Query-selected history"), "{related}"); + assert!(related.contains("Query-selected history"), "{related}"); + assert!(related.contains("Function-revealed records"), "{related}"); + assert!(!related.contains("Function-read hidden state"), "{related}"); + assert!(!related.contains("Unrelated mutation hidden state"), "{related}"); page.vm_mut().eval(r#" document.body.innerHTML = `

Visible gradient text

@@ -179,14 +196,17 @@ async fn markdown_uses_live_visibility_and_preserves_disclosure_content() { assert!(composite.contains("full conclusion"), "{composite}"); assert!(!composite.contains("A detailed review..."), "{composite}"); page.vm_mut().eval(r#" - document.body.innerHTML = ''; + document.body.innerHTML = ''; document.getElementById('field').value = 'Edited'; document.getElementById('check').checked = true; + document.getElementById('notes').value = 'Edited notes'; "#).unwrap(); let controls = page.render_page_dump(options(false)); assert!(controls.contains("Edited"), "{controls}"); assert!(controls.contains("☑"), "{controls}"); + assert!(controls.contains("Edited notes"), "{controls}"); assert!(!controls.contains("Initial"), "{controls}"); + assert!(!controls.contains("Original notes"), "{controls}"); page.vm_mut().eval(r#" document.body.innerHTML = ` Responsive