fix(content): preserve proven interactive disclosures

moli-html2md/src/form.rs

@@ -41,3 +41,12 @@ pub(crate) fn input_text<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Op
         });
     value.map(Cow::Borrowed)
 }
+
+pub(crate) fn textarea_text<'a, D: Dom + ?Sized>(
+    dom: &'a D,
+    node: D::NodeId,
+) -> Option<Cow<'a, str>> {
+    dom.attribute(node, "value")
+        .filter(|value| !value.is_empty())
+        .map(Cow::Borrowed)
+}

moli-html2md/src/html_table.rs

@@ -145,6 +145,13 @@ pub(crate) fn render<D: Dom + ?Sized>(
                             escape(&value, &mut output);
                         }
                     }
+                    NodeKind::Element("textarea") => {
+                        if let Some(value) = crate::form::textarea_text(dom, node) {
+                            escape(&value, &mut output);
+                        } else {
+                            tasks.push(Task::Siblings(dom.first_child(node), depth + 1));
+                        }
+                    }
                     NodeKind::Element(tag) => {
                         if crate::visibility::nonrendered_serialized_state(dom, node) {
                             continue;
@@ -256,6 +263,7 @@ fn allowed(tag: &str) -> bool {
             | "optgroup"
             | "option"
             | "button"
+            | "textarea"
             | "p"
             | "div"
             | "span"

moli-html2md/src/machine.rs

@@ -339,6 +339,12 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> {
                 }
                 return;
             }
+            "textarea" => {
+                if let Some(value) = crate::form::textarea_text(self.dom, node) {
+                    self.writer().text(&value);
+                    return;
+                }
+            }
             "iframe" | "video" | "audio" => {
                 if self.media(node, tag, depth) {
                     return;

moli-html2md/src/math.rs

@@ -2,15 +2,22 @@
 /// Dollar boundaries follow the usual non-space/non-digit rules, so prices do
 /// not consume the prose between two currency amounts.
 pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> {
+    let mut steps = 0;
+    next_span_counted(text, &mut steps)
+}
+
+fn next_span_counted(text: &str, steps: &mut usize) -> Option<(usize, usize)> {
     let bytes = text.as_bytes();
     let mut index = 0;
+    let mut backslashes = 0;
     let mut dollar = None;
     let mut double_dollar = None;
     let mut paren = None;
     let mut bracket = None;
     let mut candidate = None;
     while index < bytes.len() {
-        if bytes[index] == b'$' && !escaped(bytes, index) {
+        *steps += 1;
+        if bytes[index] == b'$' && backslashes % 2 == 0 {
             if bytes.get(index + 1) == Some(&b'$') {
                 if let Some(start) = double_dollar.take() {
                     accept_candidate(text, start, index + 2, 2, false, &mut candidate);
@@ -18,6 +25,7 @@ pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> {
                     double_dollar = Some(index);
                 }
                 index += 2;
+                backslashes = 0;
                 if let Some(ready) =
                     ready_candidate(candidate, [dollar, double_dollar, paren, bracket])
                 {
@@ -35,13 +43,14 @@ pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> {
                 dollar = Some(index);
             }
             index += 1;
+            backslashes = 0;
             if let Some(ready) = ready_candidate(candidate, [dollar, double_dollar, paren, bracket])
             {
                 return Some(ready);
             }
             continue;
         }
-        if bytes[index] == b'\\' && !escaped(bytes, index) {
+        if bytes[index] == b'\\' && backslashes % 2 == 0 {
             match bytes.get(index + 1) {
                 Some(b'(') => {
                     paren.get_or_insert(index);
@@ -63,6 +72,7 @@ pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> {
             }
             if matches!(bytes.get(index + 1), Some(b'(' | b'[' | b')' | b']')) {
                 index += 2;
+                backslashes = 0;
                 if let Some(ready) =
                     ready_candidate(candidate, [dollar, double_dollar, paren, bracket])
                 {
@@ -71,6 +81,11 @@ pub(crate) fn next_span(text: &str) -> Option<(usize, usize)> {
                 continue;
             }
         }
+        if bytes[index] == b'\\' {
+            backslashes += 1;
+        } else {
+            backslashes = 0;
+        }
         index += 1;
     }
     candidate
@@ -124,12 +139,20 @@ fn contains_markdown_resource(text: &str) -> bool {
     text.contains("](") || text.contains("][") || text.contains("![[")
 }
 
-fn escaped(bytes: &[u8], index: usize) -> bool {
-    bytes[..index]
-        .iter()
-        .rev()
-        .take_while(|&&ch| ch == b'\\')
-        .count()
-        % 2
-        == 1
+#[cfg(test)]
+mod tests {
+    use super::next_span_counted;
+
+    #[test]
+    fn math_scan_work_is_linear_for_backslashes_and_unclosed_delimiters() {
+        for text in ["\\\\".repeat(8_192), "\\(x ".repeat(4_096)] {
+            let mut steps = 0;
+            let _ = next_span_counted(&text, &mut steps);
+            assert!(
+                steps <= text.len(),
+                "{steps} steps for {} bytes",
+                text.len()
+            );
+        }
+    }
 }

moli-html2md/tests/regressions.rs

@@ -1064,7 +1064,7 @@ fn block_content_inside_heading_keeps_its_own_boundary() {
 
 #[test]
 fn form_values_preserve_current_readable_state_without_secrets() {
-    let source = "<table><tr><th>Period</th><th>Mon</th><th>Sun</th></tr><tr><td>Morning</td><td><input type=checkbox checked disabled></td><td><input type=checkbox disabled></td></tr></table><p><input value='Search term'><input type=email value='' placeholder='Your Email'><input type=submit value='Search'><input type=image alt='Map button'><input type=password value='secret' aria-label='Password'><input type=hidden value='token'><input type=file value='private.pdf'></p>";
+    let source = "<table><tr><th>Period</th><th>Mon</th><th>Sun</th></tr><tr><td>Morning</td><td><input type=checkbox checked disabled></td><td><input type=checkbox disabled></td></tr></table><p><input value='Search term'><input type=email value='' placeholder='Your Email'><input type=submit value='Search'><input type=image alt='Map button'><input type=password value='secret' aria-label='Password'><input type=hidden value='token'><input type=file value='private.pdf'><textarea>Notes</textarea></p>";
     let result = rendered_html(&markdown(source, false));
     for value in [
         "Morning",
@@ -1075,6 +1075,7 @@ fn form_values_preserve_current_readable_state_without_secrets() {
         "Search",
         "Map button",
         "Password",
+        "Notes",
     ] {
         assert!(result.contains(value), "missing {value}: {result}");
     }
@@ -1096,6 +1097,20 @@ fn unsafe_resource_urls_keep_labels_without_emitting_active_links() {
     assert!(!result.contains("javascript:"), "{result}");
 }
 
+#[test]
+fn embedded_media_uses_the_same_url_policy_in_all_output_paths() {
+    let media = "<video src='/clip.mp4' poster='javascript:alert(1)'></video><iframe src='data:image/png;base64,AAAA' title='Frame'></iframe><audio src='data:image/png;base64,BBBB' title='Audio'></audio>";
+    for source in [
+        media.to_owned(),
+        format!("<table><tr><td colspan='2'>{media}</td></tr></table>"),
+    ] {
+        let result = markdown(&source, false);
+        assert!(result.contains("/clip.mp4"), "{result}");
+        assert!(!result.contains("javascript:"), "{result}");
+        assert!(!result.contains("data:image"), "{result}");
+    }
+}
+
 #[test]
 fn complex_table_preserves_checkbox_state() {
     let source = "<table><tr><th rowspan=2>Status</th><td><input type=radio checked></td></tr><tr><td><input type=radio></td></tr></table>";

moli-renderer-v8/src/runtime/page_dump/visibility.rs

@@ -85,6 +85,7 @@ impl<'a> MarkdownDom<'a> {
                     .flatten()
             })
             .collect();
+        let visible_handler_functions = visible_handler_functions(dom);
         for (node, _) in &styles {
             if (Dom::attribute(dom, *node, "aria-expanded").is_some()
                 || Dom::attribute(dom, *node, "role")
@@ -97,6 +98,19 @@ impl<'a> MarkdownDom<'a> {
                         .filter(|id| target_is_disclosure(dom, &targets, id)),
                 );
             }
+            for attribute_name in dom.get_attribute_names(*node).unwrap_or_default() {
+                if !attribute_name.starts_with("on") {
+                    continue;
+                }
+                let Some(handler) = Dom::attribute(dom, *node, &attribute_name) else {
+                    continue;
+                };
+                for target in inline_disclosure_targets(handler, &visible_handler_functions) {
+                    if target_is_disclosure(dom, &targets, target) {
+                        disclosures.insert(target);
+                    }
+                }
+            }
             // Some pages pair a shortened paragraph with an explicitly linked
             // hidden full-text copy. Keep the complete copy once; an unrelated
             // dialog or a matching paragraph elsewhere is not such a pair.
@@ -181,19 +195,23 @@ impl<'a> MarkdownDom<'a> {
         let mut visibility_restored_regions = HashSet::new();
         for (node, values) in styles {
             let role = Dom::attribute(dom, node, "role").unwrap_or_default();
-            if matches!(Dom::node_kind(dom, node), NodeKind::Element("input"))
-                && let Some(element) = dom.node(node).and_then(|node| node.as_element())
+            if matches!(
+                Dom::node_kind(dom, node),
+                NodeKind::Element("input" | "textarea")
+            ) && let Some(element) = dom.node(node).and_then(|node| node.as_element())
             {
                 resolved_urls
                     .entry(node)
                     .or_insert_with(HashMap::new)
                     .insert("value".to_owned(), element.input_value());
-                if element.checked() {
+                if matches!(Dom::node_kind(dom, node), NodeKind::Element("input"))
+                    && element.checked()
+                {
                     resolved_urls
                         .entry(node)
                         .or_insert_with(HashMap::new)
                         .insert("checked".to_owned(), String::new());
-                } else {
+                } else if matches!(Dom::node_kind(dom, node), NodeKind::Element("input")) {
                     unchecked_controls.insert(node);
                 }
             }
@@ -404,6 +422,343 @@ fn target_is_disclosure<D: Dom + ?Sized>(
     })
 }
 
+fn inline_disclosure_targets<'a>(
+    value: &'a str,
+    visible_functions: &HashSet<String>,
+) -> Vec<&'a str> {
+    let mut result = Vec::new();
+    for (marker, fragment) in [
+        ("document.getElementById", false),
+        ("document.querySelector", true),
+    ] {
+        for offset in code_marker_offsets(value, marker) {
+            let remaining = &value[offset + marker.len()..];
+            let Some((literal, end)) = quoted_call_argument(remaining) else {
+                continue;
+            };
+            let Some(target) = literal
+                .strip_prefix('#')
+                .or_else(|| (!fragment).then_some(literal))
+                .filter(|target| !target.is_empty())
+            else {
+                continue;
+            };
+            let direct_mutation = sets_display_visible(&remaining[end..], None);
+            let named_mutation = enclosing_function_name(&value[..offset])
+                .is_some_and(|name| visible_functions.contains(name));
+            if direct_mutation || named_mutation {
+                result.push(target);
+            }
+        }
+    }
+    result
+}
+
+fn visible_handler_functions(dom: &NativeDom) -> HashSet<String> {
+    let mut result = HashSet::new();
+    let mut pending = vec![dom.document_node_id()];
+    while let Some(node) = pending.pop() {
+        if !matches!(Dom::node_kind(dom, node), NodeKind::Element("script")) {
+            pending.extend(dom.child_ids(node));
+            continue;
+        }
+        let source = descendant_text(dom, node);
+        for offset in code_marker_offsets(&source, "function") {
+            let signature = &source[offset + "function".len()..];
+            let Some((name, parameter, body)) = function_parts(signature) else {
+                continue;
+            };
+            if function_sets_parameter_visible(body, parameter) {
+                result.insert(name.to_owned());
+            }
+        }
+    }
+    result
+}
+
+fn function_parts(source: &str) -> Option<(&str, &str, &str)> {
+    let source = source.trim_start();
+    let name_end = source.find(|character: char| !is_identifier(character))?;
+    let name = &source[..name_end];
+    if name.is_empty() {
+        return None;
+    }
+    let parameters = source[name_end..].trim_start().strip_prefix('(')?;
+    let close = parameters.find(')')?;
+    let parameter = parameters[..close].trim();
+    if parameter.is_empty() || parameter.contains(',') || !parameter.chars().all(is_identifier) {
+        return None;
+    }
+    let rest = parameters[close + 1..].trim_start();
+    let body_start = rest.find('{')?;
+    let body_end = matching_delimiter(rest, body_start, '{', '}')?;
+    Some((name, parameter, &rest[body_start + 1..body_end]))
+}
+
+fn function_sets_parameter_visible(body: &str, parameter: &str) -> bool {
+    for offset in code_marker_offsets(body, ".style.display") {
+        let owner = body[..offset]
+            .trim_end()
+            .rsplit_once(|character: char| !is_identifier(character))
+            .map_or(body[..offset].trim_end(), |(_, owner)| owner);
+        if owner.is_empty()
+            || !sets_display_visible(&body[offset + ".style.display".len()..], Some(""))
+        {
+            continue;
+        }
+        if owner == parameter || alias_depends_on_parameter(&body[..offset], owner, parameter) {
+            return true;
+        }
+    }
+    false
+}
+
+fn alias_depends_on_parameter(prefix: &str, owner: &str, parameter: &str) -> bool {
+    let Some(offset) = code_marker_offsets(prefix, owner)
+        .into_iter()
+        .rev()
+        .find(|offset| {
+            let before = prefix[..*offset].chars().next_back();
+            let after = prefix[*offset + owner.len()..].chars().next();
+            before.is_none_or(|character| !is_identifier(character))
+                && after.is_none_or(|character| !is_identifier(character))
+                && prefix[*offset + owner.len()..]
+                    .trim_start()
+                    .starts_with('=')
+        })
+    else {
+        return false;
+    };
+    let assignment = prefix[offset + owner.len()..].trim_start();
+    let Some(right) = assignment.strip_prefix('=') else {
+        return false;
+    };
+    let right = right.split_once(';').map_or(right, |(right, _)| right);
+    contains_identifier(right, parameter)
+}
+
+fn contains_identifier(value: &str, identifier: &str) -> bool {
+    code_marker_offsets(value, identifier)
+        .into_iter()
+        .any(|offset| {
+            let before = value[..offset].chars().next_back();
+            let after = value[offset + identifier.len()..].chars().next();
+            before.is_none_or(|character| !is_identifier(character))
+                && after.is_none_or(|character| !is_identifier(character))
+        })
+}
+
+fn enclosing_function_name(value: &str) -> Option<&str> {
+    let value = value.trim_end();
+    let value = value.strip_suffix('(')?.trim_end();
+    let start = value
+        .rfind(|character: char| !is_identifier(character))
+        .map_or(0, |offset| offset + 1);
+    let name = &value[start..];
+    (!name.is_empty()).then_some(name)
+}
+
+fn quoted_call_argument(value: &str) -> Option<(&str, usize)> {
+    let leading = value.len() - value.trim_start().len();
+    let value = value.trim_start().strip_prefix('(')?;
+    let after_open = value.len() - value.trim_start().len();
+    let value = value.trim_start();
+    let quote = value
+        .chars()
+        .next()
+        .filter(|quote| matches!(quote, '\'' | '"'))?;
+    let literal = &value[quote.len_utf8()..];
+    let end = literal.find(quote)?;
+    let after_literal = literal[end + quote.len_utf8()..].trim_start();
+    if !after_literal.starts_with(')') {
+        return None;
+    }
+    let consumed = leading
+        + 1
+        + after_open
+        + quote.len_utf8()
+        + end
+        + quote.len_utf8()
+        + (literal[end + quote.len_utf8()..].len() - after_literal.len())
+        + 1;
+    Some((&literal[..end], consumed))
+}
+
+fn sets_display_visible(value: &str, display_already_consumed: Option<&str>) -> bool {
+    let value = if display_already_consumed.is_some() {
+        value
+    } else {
+        let Some(value) = value.trim_start().strip_prefix(".style.display") else {
+            return false;
+        };
+        value
+    };
+    let Some(value) = value.trim_start().strip_prefix('=') else {
+        return false;
+    };
+    let value = value.trim_start();
+    let Some(quote) = value
+        .chars()
+        .next()
+        .filter(|quote| matches!(quote, '\'' | '"'))
+    else {
+        return false;
+    };
+    let literal = &value[quote.len_utf8()..];
+    let Some(end) = literal.find(quote) else {
+        return false;
+    };
+    matches!(
+        literal[..end].trim(),
+        "" | "block"
+            | "inline"
+            | "inline-block"
+            | "flex"
+            | "inline-flex"
+            | "grid"
+            | "inline-grid"
+            | "list-item"
+            | "table"
+            | "table-row"
+            | "table-cell"
+    )
+}
+
+fn code_marker_offsets(value: &str, marker: &str) -> Vec<usize> {
+    let mut result = Vec::new();
+    let mut offset = 0;
+    let mut quote = None;
+    let mut escaped = false;
+    let mut line_comment = false;
+    let mut block_comment = false;
+    while offset < value.len() {
+        let tail = &value[offset..];
+        if line_comment {
+            if tail.starts_with('\n') {
+                line_comment = false;
+            }
+        } else if block_comment {
+            if tail.starts_with("*/") {
+                block_comment = false;
+                offset += 2;
+                continue;
+            }
+        } else if let Some(delimiter) = quote {
+            let character = tail.chars().next().expect("offset is within value");
+            if escaped {
+                escaped = false;
+            } else if character == '\\' {
+                escaped = true;
+            } else if character == delimiter {
+                quote = None;
+            }
+        } else if tail.starts_with("//") {
+            line_comment = true;
+            offset += 2;
+            continue;
+        } else if tail.starts_with("/*") {
+            block_comment = true;
+            offset += 2;
+            continue;
+        } else {
+            let character = tail.chars().next().expect("offset is within value");
+            if matches!(character, '\'' | '"' | '`') {
+                quote = Some(character);
+            } else if tail.starts_with(marker) {
+                result.push(offset);
+                offset += marker.len();
+                continue;
+            }
+        }
+        offset += tail
+            .chars()
+            .next()
+            .expect("offset is within value")
+            .len_utf8();
+    }
+    result
+}
+
+fn matching_delimiter(value: &str, start: usize, open: char, close: char) -> Option<usize> {
+    let mut depth = 0;
+    let mut quote = None;
+    let mut escaped = false;
+    let mut line_comment = false;
+    let mut block_comment = false;
+    let mut offset = start;
+    while offset < value.len() {
+        let tail = &value[offset..];
+        if line_comment {
+            if tail.starts_with('\n') {
+                line_comment = false;
+            }
+        } else if block_comment {
+            if tail.starts_with("*/") {
+                block_comment = false;
+                offset += 2;
+                continue;
+            }
+        } else if let Some(delimiter) = quote {
+            let character = tail.chars().next().expect("offset is within value");
+            if escaped {
+                escaped = false;
+            } else if character == '\\' {
+                escaped = true;
+            } else if character == delimiter {
+                quote = None;
+            }
+        } else if tail.starts_with("//") {
+            line_comment = true;
+            offset += 2;
+            continue;
+        } else if tail.starts_with("/*") {
+            block_comment = true;
+            offset += 2;
+            continue;
+        } else {
+            let character = tail.chars().next().expect("offset is within value");
+            if matches!(character, '\'' | '"' | '`') {
+                quote = Some(character);
+            } else if character == open {
+                depth += 1;
+            } else if character == close {
+                depth -= 1;
+                if depth == 0 {
+                    return Some(offset);
+                }
+            }
+        }
+        offset += tail
+            .chars()
+            .next()
+            .expect("offset is within value")
+            .len_utf8();
+    }
+    None
+}
+
+fn is_identifier(character: char) -> bool {
+    character == '_' || character == '$' || character.is_ascii_alphanumeric()
+}
+
+fn descendant_text(dom: &NativeDom, root: NativeNodeId) -> String {
+    let mut pending = vec![root];
+    let mut result = String::new();
+    while let Some(node) = pending.pop() {
+        if let NodeKind::Text(value) = Dom::node_kind(dom, node) {
+            result.push_str(value);
+        }
+        let mut children = Vec::new();
+        let mut child = dom.first_child(node);
+        while let Some(id) = child {
+            children.push(id);
+            child = dom.next_sibling(id);
+        }
+        pending.extend(children.into_iter().rev());
+    }
+    result
+}
+
 fn px(value: &str) -> Option<f32> {
     value.strip_suffix("px")?.trim().parse().ok()
 }

moli-renderer-v8/src/runtime/page_vm/tests/markdown_dump.rs

@@ -100,6 +100,14 @@ async fn markdown_uses_live_visibility_and_preserves_disclosure_content() {
         let actions = page.render_page_dump(options(false));
         assert!(actions.contains("[Accept](https://example.test/accept)\n\n[Reject](https://example.test/reject)"), "{actions}");
         page.vm_mut().eval(r##"
+            document.head.innerHTML = `<script>
+                function revealElement(what) {
+                    const target = typeof what === 'object' ? what : document.getElementById(what);
+                    target.style.display = 'block';
+                }
+                function inspectElement(what) { console.log(what.textContent); }
+                function revealUnrelated(what) { sidebar.style.display = 'block'; console.log(what); }
+            </script>`;
             document.body.innerHTML = `
                 <article><p>A detailed review starts... <a data-src="#complete" href="javascript:;">Read more</a></p>
                 <div id="complete" style="display:none">A detailed review starts here and retains the final conclusion.</div></article>
@@ -147,14 +155,23 @@ async fn markdown_uses_live_visibility_and_preserves_disclosure_content() {
                 <button onclick="/* document.getElementById('commented') */ console.log('clicked')">Comment</button>
                 <section id="commented" style="display:none">Hidden commented state</section>
                 <button onclick="document.querySelector('#history').style.display='block'">History</button>
-                <section id="history" style="display:none">Query-selected history</section>`;
+                <section id="history" style="display:none">Query-selected history</section>
+                <button onclick="return revealElement(document.getElementById('records'))">Records</button>
+                <section id="records" style="display:none">Function-revealed records</section>
+                <button onclick="inspectElement(document.getElementById('inspected'))">Inspect</button>
+                <section id="inspected" style="display:none">Function-read hidden state</section>
+                <button onclick="revealUnrelated(document.getElementById('unrelated-target'))">Other</button>
+                <section id="unrelated-target" style="display:none">Unrelated mutation hidden state</section>`;
         "##).unwrap();
         let related = page.render_page_dump(options(false));
         assert!(!related.contains("Hidden telemetry state"), "{related}");
         assert!(!related.contains("Hidden diagnostic state"), "{related}");
         assert!(!related.contains("Hidden read-only state"), "{related}");
         assert!(!related.contains("Hidden commented state"), "{related}");
-        assert!(!related.contains("Query-selected history"), "{related}");
+        assert!(related.contains("Query-selected history"), "{related}");
+        assert!(related.contains("Function-revealed records"), "{related}");
+        assert!(!related.contains("Function-read hidden state"), "{related}");
+        assert!(!related.contains("Unrelated mutation hidden state"), "{related}");
         page.vm_mut().eval(r#"
             document.body.innerHTML = `
                 <p style="color:white;background-image:linear-gradient(black,black)">Visible gradient text</p>
@@ -179,14 +196,17 @@ async fn markdown_uses_live_visibility_and_preserves_disclosure_content() {
         assert!(composite.contains("full conclusion"), "{composite}");
         assert!(!composite.contains("A detailed review..."), "{composite}");
         page.vm_mut().eval(r#"
-            document.body.innerHTML = '<input id="field" value="Initial"><input id="check" type="checkbox">';
+            document.body.innerHTML = '<input id="field" value="Initial"><input id="check" type="checkbox"><textarea id="notes">Original notes</textarea>';
             document.getElementById('field').value = 'Edited';
             document.getElementById('check').checked = true;
+            document.getElementById('notes').value = 'Edited notes';
         "#).unwrap();
         let controls = page.render_page_dump(options(false));
         assert!(controls.contains("Edited"), "{controls}");
         assert!(controls.contains("☑"), "{controls}");
+        assert!(controls.contains("Edited notes"), "{controls}");
         assert!(!controls.contains("Initial"), "{controls}");
+        assert!(!controls.contains("Original notes"), "{controls}");
         page.vm_mut().eval(r#"
             document.body.innerHTML = `
                 <img srcset="small.png 1x, large.png 2x" alt="Responsive">