From fc42d8ddd9db7e14d5cc2f6ebfe111d56d9dfb82 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:09:25 +0800 Subject: [PATCH] fix(parser): preserve text document MIME and encoding semantics --- moli-encoding/src/document.rs | 25 +++ moli-encoding/src/tests.rs | 96 +++++++++++ moli-parser/src/html.rs | 93 +++++++++- moli-parser/src/session.rs | 14 +- moli-parser/src/stream.rs | 16 +- .../src/protocol_server/tests/classic.rs | 93 ++++++++++ moli-protocol/src/domains/emulation/tests.rs | 18 +- .../src/domains/page/tests/lifecycle.rs | 16 +- .../src/domains/page/tests/runtime.rs | 10 +- .../target/tests/tests_background_staging.rs | 8 +- moli-renderer-v8/src/dom_parser.rs | 6 +- moli-renderer-v8/src/live_document_parser.rs | 23 +++ .../context_host/child_documents/commit.rs | 1 + .../child_documents/live_parser.rs | 8 + .../context_host/child_documents/loads.rs | 23 ++- .../src/runtime/phase_one/bootstrap.rs | 2 + .../src/runtime/phase_one/state.rs | 20 +++ .../src/runtime/phase_one/streaming.rs | 162 ++++++++++++++++-- moli-web-mime/src/classification.rs | 8 + moli-web-mime/src/lib.rs | 6 +- moli-web-mime/src/tests.rs | 22 +++ moli/src/fetch_dump/tests.rs | 4 +- 22 files changed, 625 insertions(+), 49 deletions(-) diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs index 88df60445..3c14f5109 100644 --- a/moli-encoding/src/document.rs +++ b/moli-encoding/src/document.rs @@ -47,9 +47,26 @@ pub struct HtmlDocumentStreamingDecoder { url_hint: Option, decoder: Option, selected_encoding: Option<&'static Encoding>, + sniff_html_declarations: bool, } impl HtmlDocumentStreamingDecoder { + /// Decode a text document without interpreting literal HTML/XML declarations. + /// JSON defaults to UTF-8; other text retains the configured legacy detector. + pub fn new_text_document( + headers: &[(String, String)], + url_hint: &str, + detector: LegacyEncodingDetector, + json: bool, + ) -> Self { + let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector)); + decoder.sniff_html_declarations = false; + if json && decoder.transport_encoding.is_none() { + decoder.transport_encoding = Some(encoding_rs::UTF_8); + } + decoder + } + pub fn new(headers: &[(String, String)]) -> Self { Self::new_with_options(headers, None, None, None) } @@ -88,6 +105,7 @@ impl HtmlDocumentStreamingDecoder { url_hint: url_hint.map(str::to_owned), decoder: None, selected_encoding: None, + sniff_html_declarations: true, } } @@ -97,6 +115,7 @@ impl HtmlDocumentStreamingDecoder { pub fn document_encoding_name(&self) -> &'static str { self.selected_encoding_name() + .or_else(|| self.transport_encoding.map(Encoding::name)) .unwrap_or(self.fallback_encoding.name()) } @@ -149,6 +168,12 @@ impl HtmlDocumentStreamingDecoder { if let Some(encoding) = self.transport_encoding { return Some(encoding); } + if !self.sniff_html_declarations { + return (finishing || self.sniff_buffer.len() >= 1024).then(|| { + self.detected_legacy_content_encoding() + .unwrap_or(self.fallback_encoding) + }); + } if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) { return Some(encoding); } diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs index 2b4863a84..d9717f32d 100644 --- a/moli-encoding/src/tests.rs +++ b/moli-encoding/src/tests.rs @@ -24,6 +24,102 @@ fn unexpected_legacy_encoding_detector( panic!("declared encoding must take precedence over heuristic detection") } +#[test] +fn json_text_document_defaults_to_utf8_across_every_chunk_split() { + let input = "{\"name\":\"Gülçek\"}"; + for split in 0..=input.len() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + unexpected_legacy_encoding_detector, + true, + ); + let mut output = decoder.push(&input.as_bytes()[..split]).join(""); + output.push_str(&decoder.push(&input.as_bytes()[split..]).join("")); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, input, "split {split}"); + assert_eq!(decoder.selected_encoding_name(), Some("UTF-8")); + } +} + +#[test] +fn plain_text_document_honors_transport_charset() { + let headers = [( + "Content-Type".to_owned(), + "text/plain; charset=gbk".to_owned(), + )]; + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &headers, + "https://example.test/", + unexpected_legacy_encoding_detector, + false, + ); + let mut output = decoder.push(&gbk_bytes("家居")).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "家居"); + assert_eq!(decoder.selected_encoding_name(), Some("GBK")); +} + +#[test] +fn text_document_bom_precedes_transport_charset() { + let headers = [( + "Content-Type".to_owned(), + "text/plain; charset=gbk".to_owned(), + )]; + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &headers, + "https://example.test/", + unexpected_legacy_encoding_detector, + false, + ); + assert_eq!(decoder.document_encoding_name(), "GBK"); + assert_eq!(decoder.selected_encoding_name(), None); + assert!(decoder.push(&[0xef]).is_empty()); + assert!(decoder.push(&[0xbb]).is_empty()); + let mut tail = vec![0xbf]; + tail.extend_from_slice("Gülçek".as_bytes()); + let mut output = decoder.push(&tail).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "Gülçek"); + assert_eq!(decoder.selected_encoding_name(), Some("UTF-8")); + assert_eq!(decoder.document_encoding_name(), "UTF-8"); +} + +#[test] +fn plain_text_document_retains_legacy_detection_without_a_charset() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://legacy.example/", + test_legacy_encoding_detector, + false, + ); + let mut output = decoder.push(&[0x80]).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "А"); + assert_eq!(decoder.selected_encoding_name(), Some("IBM866")); +} + +#[test] +fn plain_text_document_ignores_literal_encoding_declarations() { + for prefix in [ + "", + "", + ] { + let mut input = prefix.as_bytes().to_vec(); + input.push(0xe9); + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + test_legacy_encoding_detector, + false, + ); + let mut output = decoder.push(&input).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, format!("{prefix}é")); + assert_eq!(decoder.selected_encoding_name(), Some("windows-1252")); + } +} + #[test] fn content_type_charset_is_selected() { let headers = vec![( diff --git a/moli-parser/src/html.rs b/moli-parser/src/html.rs index ce351cf5e..985707f55 100644 --- a/moli-parser/src/html.rs +++ b/moli-parser/src/html.rs @@ -27,7 +27,7 @@ use super::live_target::{ ParserDomMutationConsumer, ParserDomReadConsumer, ParserElementCreationConsumer, ParserMutationEffectConsumer, ParserMutationEffectDelivery, ParserRuntimeDomSinks, ParserStreamHtmlTreeSinkTarget, new_live_document_root_html_tree_sink_stream, - new_parser_stream_html_tree_sink_stream, + new_parser_stream_html_tree_sink_stream, new_parser_stream_html_tree_sink_target, }; use super::{ ParserSourcePosition, html_chunks, @@ -438,6 +438,47 @@ impl HtmlParser { DocumentStream::new_parser_stream(final_url, self.scripting_enabled) } + /// Start a text document using the HTML tokenizer's plaintext state. + /// Subsequent chunks remain literal text inside the browser-owned `pre`. + pub fn start_text_document(&self, final_url: Url, content_type: &str) -> DocumentStream { + let stream = + DocumentStream::new_text_document(new_parser_stream_html_tree_sink_target(final_url)); + stream.inner.initialize_text_document(); + let mut host = stream.inner.take_parser_stream_dom_host(); + let document_handle = host.document_handle(); + host.set_document_content_type_for_handle(document_handle, content_type); + stream.inner.restore_parser_stream_dom_host(host); + stream + } + + /// Initialize the same literal-text shell in an existing document. The + /// embedder owns that document's MIME metadata and live DOM callbacks. + pub fn start_live_text_document_root( + &self, + final_url: Url, + document_handle: NativeNodeId, + consumer: &mut T, + ) -> DocumentStream + where + T: ParserDomReadConsumer + + ParserDomMutationConsumer + + ParserMutationEffectConsumer + + ParserElementCreationConsumer, + { + let stream = DocumentStream::new_text_document( + ParserStreamHtmlTreeSinkTarget::new_live_document_root(final_url, document_handle), + ); + // SAFETY: the consumer is exclusively borrowed until the step guard + // removes the erased callbacks, including when initialization unwinds. + let sinks = unsafe { ParserRuntimeDomSinks::from_consumer(consumer) }; + stream.inner.enter_runtime_dom_sinks_parse_step(sinks); + { + let step = RuntimeDomSinksParserStep { stream: &stream }; + step.stream.inner.initialize_text_document(); + } + stream + } + pub fn start_live_document_root( &self, final_url: Url, @@ -551,6 +592,17 @@ impl Default for ParserInputQueue { } impl DocumentStream { + fn new_text_document(target: ParserStreamHtmlTreeSinkTarget) -> Self { + let mut options = html_parse_opts_with_scripting(false); + // The byte decoder has already consumed the transport BOM. In plaintext + // every remaining U+FEFF is content, including at a chunk boundary. + options.tokenizer.discard_bom = false; + Self { + inner: HtmlTreeSinkStream::from_target_with_options(target, options), + input: RefCell::default(), + } + } + fn new_parser_stream(final_url: Url, scripting_enabled: bool) -> Self { Self { inner: new_parser_stream_html_tree_sink_stream(final_url, scripting_enabled), @@ -1843,6 +1895,45 @@ mod tests { const HTML_NS: &str = "http://www.w3.org/1999/xhtml"; const MATHML_NS: &str = "http://www.w3.org/1998/Math/MathML"; + #[test] + fn text_document_stream_preserves_literal_markup_and_leading_newline() { + let payload = + "\u{feff}\nGülçek&\u{feff}"; + for content_type in ["text/plain", "application/json", "application/problem+json"] { + let stream = HtmlParser::SCRIPTING_ENABLED.start_text_document( + Url::parse("https://example.test/data").unwrap(), + content_type, + ); + for character in payload.chars() { + stream.feed(&character.to_string()); + } + let document = stream.finish(); + let pre = first_element_by_ns(&document, HTML_NS, "pre"); + assert_eq!(document.text_content(pre).as_deref(), Some(payload)); + assert_eq!(document.document().unwrap().content_type(), content_type); + for tag in ["b", "script"] { + assert!( + document + .elements_by_tag_name_ns( + document.document_node_id(), + Some(HTML_NS), + tag, + true + ) + .is_empty() + ); + } + } + } + + #[test] + fn html_document_still_parses_markup_and_entities() { + let document = parse_test_document("Gülçek&"); + let bold = first_element_by_ns(&document, HTML_NS, "b"); + assert_eq!(document.text_content(bold).as_deref(), Some("Gülçek&")); + assert_eq!(document.document().unwrap().content_type(), "text/html"); + } + fn parse_test_document(html: &str) -> NativeDom { HtmlParser::SCRIPTING_ENABLED.parse( Url::parse("https://example.test/").expect("test url"), diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rs index 3025906a4..64de80047 100644 --- a/moli-parser/src/session.rs +++ b/moli-parser/src/session.rs @@ -167,6 +167,16 @@ impl TokenSink for EmbedderPausingTreeBuilder { } impl HtmlParserSession { + pub(super) fn initialize_text_document(&self) { + // Parse only the browser-owned shell. Response bytes enter the tokenizer + // after it has switched to plaintext, so tags and entities stay literal. + self.process(StrTendril::from(concat!( + "", + "
\n"
+        )));
+        self.tokenizer.set_plaintext_state();
+    }
+
     fn new(sink: DocumentSink, opts: ParseOpts) -> Self {
         let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);
         Self {
@@ -352,10 +362,10 @@ fn feed_with_definitive_encoding(
 
 pub(super) fn new_html_tree_sink_session(
     target: ParserStreamHtmlTreeSinkTarget,
-    scripting_enabled: bool,
+    options: ParseOpts,
 ) -> HtmlTreeSinkSession {
     let sink = DocumentSink::new(target);
-    let parser = HtmlParserSession::new(sink, html_parse_opts_with_scripting(scripting_enabled));
+    let parser = HtmlParserSession::new(sink, options);
     let script_input = ParserInputQueue::default();
 
     HtmlTreeSinkSession {
diff --git a/moli-parser/src/stream.rs b/moli-parser/src/stream.rs
index fa14f3b82..70d3a31d3 100644
--- a/moli-parser/src/stream.rs
+++ b/moli-parser/src/stream.rs
@@ -429,11 +429,25 @@ pub(crate) fn prepare_parser_script_handoff_for_static_document(
 }
 
 impl HtmlTreeSinkStream {
+    pub(super) fn initialize_text_document(&self) {
+        self.parser.initialize_text_document();
+    }
+
     pub(super) fn from_target_with_scripting(
         target: ParserStreamHtmlTreeSinkTarget,
         scripting_enabled: bool,
     ) -> Self {
-        let session = new_html_tree_sink_session(target, scripting_enabled);
+        Self::from_target_with_options(
+            target,
+            super::session::html_parse_opts_with_scripting(scripting_enabled),
+        )
+    }
+
+    pub(super) fn from_target_with_options(
+        target: ParserStreamHtmlTreeSinkTarget,
+        options: html5ever::ParseOpts,
+    ) -> Self {
+        let session = new_html_tree_sink_session(target, options);
         Self {
             parser: session.parser,
             script_input: session.script_input,
diff --git a/moli-protocol-server/src/protocol_server/tests/classic.rs b/moli-protocol-server/src/protocol_server/tests/classic.rs
index 84a3ea0be..ce08f5ff6 100644
--- a/moli-protocol-server/src/protocol_server/tests/classic.rs
+++ b/moli-protocol-server/src/protocol_server/tests/classic.rs
@@ -3,6 +3,99 @@ use axum::http::{HeaderMap, header};
 use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
 use serde_json::{Map, Value};
 
+#[tokio::test]
+async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {
+    const PAYLOAD: &str =
+        "literal&Gülçek";
+    let cases = [
+        (
+            "plain",
+            "text/plain; charset=utf-8",
+            "text/plain",
+            true,
+            PAYLOAD,
+        ),
+        (
+            "plain-bom",
+            "text/plain; charset=utf-8",
+            "text/plain",
+            true,
+            "\u{feff}\u{feff}",
+        ),
+        (
+            "json",
+            "application/json",
+            "application/json",
+            true,
+            PAYLOAD,
+        ),
+        (
+            "javascript",
+            "text/javascript; charset=utf-8",
+            "text/javascript",
+            true,
+            PAYLOAD,
+        ),
+        (
+            "html",
+            "text/html; charset=utf-8",
+            "text/html",
+            false,
+            PAYLOAD,
+        ),
+    ];
+    let mut fixture = axum::Router::new();
+    for (name, mime, _, _, payload) in cases {
+        fixture = fixture.route(
+            &format!("/{name}"),
+            axum::routing::get(move || async move { ([(header::CONTENT_TYPE, mime)], payload) }),
+        );
+    }
+    fixture = fixture.route(
+        "/frame/{kind}",
+        axum::routing::get(
+            |axum::extract::Path(kind): axum::extract::Path| async move {
+                (
+                    [(header::CONTENT_TYPE, "text/html")],
+                    format!(""),
+                )
+            },
+        ),
+    );
+    let (addr, _server) = spawn_dedicated_fixture_server(fixture, "document-mime");
+    let app = build_router(test_state());
+    let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+    let session_id = session["value"]["sessionId"].as_str().unwrap();
+    for (name, _, content_type, literal, payload) in cases {
+        for prefix in ["", "frame/"] {
+            let navigated = classic_request_json_with_body(
+                app.clone(),
+                Method::POST,
+                &format!("/session/{session_id}/url"),
+                json!({"url":format!("http://{addr}/{prefix}{name}")}),
+            )
+            .await;
+            assert_eq!(navigated, json!({"value":null}), "{prefix}{name}");
+            let observed = classic_request_json_with_body(
+                app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"),
+                json!({"script": "const w=document.querySelector('iframe')?.contentWindow ?? window; return [w.document.body.textContent,w.executed??null,w.document.querySelectorAll('script').length,w.document.contentType];", "args": []}),
+            ).await;
+            let expected = if literal {
+                json!([
+                    payload.strip_prefix('\u{feff}').unwrap_or(payload),
+                    null,
+                    0,
+                    content_type
+                ])
+            } else {
+                json!(["literal&Gülçek", 42, 1, content_type])
+            };
+            assert_eq!(observed["value"], expected, "{prefix}{name}");
+        }
+    }
+    classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
 #[tokio::test]
 async fn webdriver_classic_status_session_and_delete_routes_use_value_envelope() {
     let app = build_router(test_state());
diff --git a/moli-protocol/src/domains/emulation/tests.rs b/moli-protocol/src/domains/emulation/tests.rs
index d8a83d17c..76ac1936a 100644
--- a/moli-protocol/src/domains/emulation/tests.rs
+++ b/moli-protocol/src/domains/emulation/tests.rs
@@ -573,7 +573,9 @@ async fn admitted_idle_override_is_visible_to_concurrent_same_site_navigation()
             listener,
             Router::new().route(
                 "/",
-                get(|| async { "idle navigation" }),
+                get(|| async {
+                    axum::response::Html("idle navigation")
+                }),
             ),
         )
         .await
@@ -644,7 +646,9 @@ async fn idle_override_updates_idle_detector_and_clear_restores_actual_state() {
             listener,
             Router::new().route(
                 "/",
-                get(|| async { "idle detector" }),
+                get(|| async {
+                    axum::response::Html("idle detector")
+                }),
             ),
         )
         .await
@@ -2219,7 +2223,7 @@ async fn emulation_async_dispatch_updates_live_page_user_agent_and_xhr_header()
 #[tokio::test(flavor = "multi_thread")]
 async fn emulation_async_dispatch_updates_live_page_surface_without_mutating_accept_language() {
     async fn page_handler() -> impl IntoResponse {
-        "ok"
+        axum::response::Html("ok")
     }
 
     async fn xhr_handler(
@@ -2711,9 +2715,9 @@ async fn locale_override_updates_intl_without_mutating_language_surfaces() {
             .get(axum::http::header::ACCEPT_LANGUAGE)
             .and_then(|value| value.to_str().ok())
             .unwrap_or("");
-        format!(
+        axum::response::Html(format!(
             ""
-        )
+        ))
     }
 
     let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -2767,9 +2771,9 @@ async fn bidi_user_context_locale_composes_with_user_agent_on_all_identity_surfa
             .get(axum::http::header::ACCEPT_LANGUAGE)
             .and_then(|value| value.to_str().ok())
             .unwrap_or("");
-        format!(
+        axum::response::Html(format!(
             ""
-        )
+        ))
     }
 
     let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
diff --git a/moli-protocol/src/domains/page/tests/lifecycle.rs b/moli-protocol/src/domains/page/tests/lifecycle.rs
index 450912689..ea7c311b9 100644
--- a/moli-protocol/src/domains/page/tests/lifecycle.rs
+++ b/moli-protocol/src/domains/page/tests/lifecycle.rs
@@ -2391,11 +2391,15 @@ async fn network_navigations_use_unique_document_loader_ids() {
             axum::Router::new()
                 .route(
                     "/first",
-                    axum::routing::get(|| async { "
first
" }), + axum::routing::get(|| async { + axum::response::Html("
first
") + }), ) .route( "/second", - axum::routing::get(|| async { "
second
" }), + axum::routing::get(|| async { + axum::response::Html("
second
") + }), ), ) .await @@ -2474,11 +2478,15 @@ async fn navigations_without_network_domain_still_use_unique_loader_ids() { axum::Router::new() .route( "/first", - axum::routing::get(|| async { "
first
" }), + axum::routing::get(|| async { + axum::response::Html("
first
") + }), ) .route( "/second", - axum::routing::get(|| async { "
second
" }), + axum::routing::get(|| async { + axum::response::Html("
second
") + }), ), ) .await diff --git a/moli-protocol/src/domains/page/tests/runtime.rs b/moli-protocol/src/domains/page/tests/runtime.rs index ab651a209..1a1e48497 100644 --- a/moli-protocol/src/domains/page/tests/runtime.rs +++ b/moli-protocol/src/domains/page/tests/runtime.rs @@ -1269,20 +1269,22 @@ async fn runtime_evaluate_child_history_back_emits_child_frame_navigation_and_li .route( "/main", axum::routing::get(|| async { - "" + axum::response::Html( + "", + ) }), ) .route( "/initial", - axum::routing::get(|| async { "initial" }), + axum::routing::get(|| async { axum::response::Html("initial") }), ) .route( "/history-a", - axum::routing::get(|| async { "history-a" }), + axum::routing::get(|| async { axum::response::Html("history-a") }), ) .route( "/history-b", - axum::routing::get(|| async { "history-b" }), + axum::routing::get(|| async { axum::response::Html("history-b") }), ), ) .await diff --git a/moli-protocol/src/domains/target/tests/tests_background_staging.rs b/moli-protocol/src/domains/target/tests/tests_background_staging.rs index e52b86f54..49bebc255 100644 --- a/moli-protocol/src/domains/target/tests/tests_background_staging.rs +++ b/moli-protocol/src/domains/target/tests/tests_background_staging.rs @@ -3037,9 +3037,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat .get(axum::http::header::ACCEPT_LANGUAGE) .and_then(|value| value.to_str().ok()) .unwrap_or(""); - format!( + axum::response::Html(format!( "" - ) + )) } let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -3204,7 +3204,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat #[tokio::test(flavor = "multi_thread")] async fn same_context_background_session_can_clear_its_own_timezone_before_activation() { async fn handler() -> impl IntoResponse { - "" + axum::response::Html( + "", + ) } let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs index f1703887c..e06055eb4 100644 --- a/moli-renderer-v8/src/dom_parser.rs +++ b/moli-renderer-v8/src/dom_parser.rs @@ -367,7 +367,11 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>( source: &'a str, content_type: Option<&str>, ) -> std::borrow::Cow<'a, str> { - if source == "\u{feff}" && !content_type.is_some_and(is_dom_parser_xml_mime) { + if source == "\u{feff}" + && !content_type.is_some_and(|mime| { + is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime) + }) + { std::borrow::Cow::Borrowed("\u{feff}") } else { std::borrow::Cow::Borrowed(source) diff --git a/moli-renderer-v8/src/live_document_parser.rs b/moli-renderer-v8/src/live_document_parser.rs index 9eacbd703..db64e1ff8 100644 --- a/moli-renderer-v8/src/live_document_parser.rs +++ b/moli-renderer-v8/src/live_document_parser.rs @@ -716,6 +716,14 @@ impl DocumentParserSession { ) } + pub(crate) fn start_main_text_document(document_url: Url, content_type: &str) -> Self { + Self::new_html( + HtmlParser::with_scripting_enabled(false) + .start_text_document(document_url, content_type), + DocumentParserLifetime::Finite, + ) + } + pub(crate) fn start_finite_live_document( document_url: Url, document_handle: NativeNodeId, @@ -728,6 +736,21 @@ impl DocumentParserSession { ) } + pub(crate) fn start_finite_live_text_document( + document_url: Url, + document_handle: NativeNodeId, + owner: &mut impl LiveDocumentParserOwner, + ) -> Self { + Self::new_html( + HtmlParser::with_scripting_enabled(false).start_live_text_document_root( + document_url, + document_handle, + owner, + ), + DocumentParserLifetime::Finite, + ) + } + pub(crate) fn start_finite_live_xml_document( document_url: Url, document_handle: NativeNodeId, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs index 67f2387ab..6841526e6 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs @@ -239,6 +239,7 @@ impl JsContextHost { parser_base_url, source.as_ref(), is_xml_document, + snapshot.content_type.as_deref(), ); Some(ChildDocumentInstallResult { initial_classic_ready_work: parser_start.initial_classic_ready_work, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs index 42ad8071a..e2d57bdc7 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs @@ -1695,6 +1695,7 @@ impl JsContextHost { document_base_url: Url, markup: &str, is_xml_document: bool, + content_type: Option<&str>, ) -> ChildLiveDocumentParserStartResult { let owner = FrameDocumentOwner::new(owner_local_window_id, owner_document_id); self.child_document_parsers.clear(owner); @@ -1703,6 +1704,13 @@ impl JsContextHost { document_base_url, document_handle, ) + } else if content_type.is_some_and(moli_web_mime::is_text_document_mime) { + let mut parser_owner = ChildFrameLiveParserOwner::new(self, scope, document_handle); + DocumentParserSession::start_finite_live_text_document( + document_base_url, + document_handle, + &mut parser_owner, + ) } else { DocumentParserSession::start_finite_live_document( document_base_url, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs index bd9ce2a8a..6af252c1b 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs @@ -21,7 +21,8 @@ use crate::{ LoadedChildDocument, SubresourceResponseBody, }, }; -use moli_encoding::decode_html_document_with_fallback; +use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback}; +use moli_encoding_detector::detect_legacy_html_encoding; pub(crate) struct AppliedChildDocumentLoadCompletion { /// Initial parser-classic work produced by the committed child document. @@ -757,8 +758,24 @@ fn child_document_load_outcome_from_response( let body_bytes = response_body .try_bytes() .map_err(|error| format!("failed to read child document response body: {error}"))?; - let (markup, character_set) = - decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback)); + let (markup, character_set) = if let Some(mime) = content_type + .as_deref() + .filter(|mime| moli_web_mime::is_text_document_mime(mime)) + { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &head.headers, + head.final_url.as_str(), + detect_legacy_html_encoding, + moli_web_mime::is_json_module_mime(mime) || mime == "text/json", + ); + let mut markup = decoder.push(&body_bytes).concat(); + if let Some(tail) = decoder.finish() { + markup.push_str(&tail); + } + (markup, decoder.document_encoding_name()) + } else { + decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback)) + }; (markup, character_set.to_owned()) }; let policy_container = diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs index 5af62b821..454433e6c 100644 --- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs +++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs @@ -25,6 +25,7 @@ impl ConcurrentParseTimeRuntime { let (page_vm, triggered_navigation) = { let buffered_document_preloads = &mut state.buffered_document_preloads; let service_worker_preload_context = state.service_worker_preload_context.as_ref(); + let document_character_set = &state.document_character_set; PageVm::new_from_parser_stream_and_run_document_start( page_id, local_executor, @@ -34,6 +35,7 @@ impl ConcurrentParseTimeRuntime { &mut state.parser_session, started, |page_vm| { + page_vm.set_document_character_set(document_character_set.clone()); admit_pending_preloads( page_vm, buffered_document_preloads, diff --git a/moli-renderer-v8/src/runtime/phase_one/state.rs b/moli-renderer-v8/src/runtime/phase_one/state.rs index fa8ecbc35..e3c8a78c6 100644 --- a/moli-renderer-v8/src/runtime/phase_one/state.rs +++ b/moli-renderer-v8/src/runtime/phase_one/state.rs @@ -11,6 +11,7 @@ pub(super) struct ParseTimeDriverState { pub(super) buffered_document_preloads: Box, pub(super) service_worker_preload_context: Option, pub(super) input_closed: bool, + pub(super) is_text_document: bool, } impl ParseTimeDriverState { @@ -32,6 +33,24 @@ impl ParseTimeDriverState { buffered_document_preloads: Box::default(), service_worker_preload_context: None, input_closed: false, + is_text_document: false, + } + } + + pub(super) fn new_text(final_url: Url, mime: &str) -> Self { + Self { + parser_session: DocumentParserSession::start_main_text_document( + final_url.clone(), + mime, + ), + final_url, + document_character_set: "UTF-8".to_owned(), + scheduler: DocumentScriptScheduler::new(), + pending_parsing_blocking_script: PendingParsingBlockingClassicScriptRunner::empty(), + buffered_document_preloads: Box::default(), + service_worker_preload_context: None, + input_closed: false, + is_text_document: true, } } @@ -45,6 +64,7 @@ impl ParseTimeDriverState { buffered_document_preloads: Box::default(), service_worker_preload_context: None, input_closed: false, + is_text_document: false, } } diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs index d3862df63..dca248cdb 100644 --- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs +++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs @@ -94,10 +94,8 @@ impl ConcurrentParseTimeRuntime { env.apply_navigation_response_headers(&response_final_url, &response_headers); let mut body_source = RawDocumentBodySource::fetch_response(response); - let mut state = ParseTimeDriverState::new_with_scripting_enabled( - response_final_url, - main_document_parser_scripting_enabled(&env), - ); + let (mut state, mut decoder) = + response_document_parser(response_final_url, &response_headers, &env); state .buffered_document_preloads .set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission( @@ -124,11 +122,6 @@ impl ConcurrentParseTimeRuntime { ) }); state.service_worker_preload_context = service_worker_preload_context.clone(); - let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( - &response_headers, - state.final_url.as_str(), - detect_legacy_html_encoding, - ); // Raw navigation bodies are decoded during prebootstrap scan. The decoder // is carried forward so split multibyte sequences are not decoded twice or // lost between the scan and parser handoff. @@ -332,10 +325,7 @@ impl ConcurrentParseTimeRuntime { }, ))); } - let mut state = ParseTimeDriverState::new_with_scripting_enabled( - final_url, - main_document_parser_scripting_enabled(&env), - ); + let (mut state, mut decoder) = response_document_parser(final_url, &response_headers, &env); state .buffered_document_preloads .set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission( @@ -362,11 +352,6 @@ impl ConcurrentParseTimeRuntime { ) }); state.service_worker_preload_context = service_worker_preload_context.clone(); - let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( - &response_headers, - state.final_url.as_str(), - detect_legacy_html_encoding, - ); // External raw bodies replay captured chunks. Pre-scan only what is already // buffered before bootstrap so the producer's backpressure boundary still // controls how far ahead the parser can get. @@ -451,6 +436,10 @@ impl ConcurrentParseTimeRuntime { chunk: String, service_worker_context: Option<&ServiceWorkerScriptPreloadContext>, ) { + if self.state.is_text_document { + self.state.parser_session.queue_arrived_chunk(chunk); + return; + } self.state .buffered_document_preloads .append_to_main_document_scan_with_service_worker_context( @@ -607,6 +596,9 @@ fn scan_prebootstrap_html_chunk_into_state( chunk: &str, service_worker_context: Option<&ServiceWorkerScriptPreloadContext>, ) { + if state.is_text_document { + return; + } state .buffered_document_preloads .append_to_main_document_prebootstrap_scan_with_service_worker_context( @@ -617,6 +609,41 @@ fn scan_prebootstrap_html_chunk_into_state( ); } +fn response_document_parser( + final_url: Url, + headers: &[(String, String)], + env: &PageVmEnvConfig, +) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) { + let content_type = moli_web_mime::response_document_content_type(headers); + let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime)); + let mut state = if let Some(mime) = &text_type { + ParseTimeDriverState::new_text(final_url, mime) + } else { + ParseTimeDriverState::new_with_scripting_enabled( + final_url, + main_document_parser_scripting_enabled(env), + ) + }; + let decoder = if let Some(mime) = text_type { + HtmlDocumentStreamingDecoder::new_text_document( + headers, + state.final_url.as_str(), + detect_legacy_html_encoding, + moli_web_mime::is_json_module_mime(&mime) || mime == "text/json", + ) + } else { + HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( + headers, + state.final_url.as_str(), + detect_legacy_html_encoding, + ) + }; + // Headers/defaults are observable even when no body chunk is ready yet. + // A later BOM or decoder decision can still replace this tentative value. + sync_state_document_character_set_from_decoder(&mut state, &decoder); + (state, decoder) +} + pub(super) fn enqueue_streaming_raw_chunk( runtime: &mut ConcurrentParseTimeRuntime, decoder: &mut HtmlDocumentStreamingDecoder, @@ -908,6 +935,105 @@ mod tests { use crate::parser::ScriptSource; use tokio::io::{AsyncReadExt, AsyncWriteExt}; + #[tokio::test] + async fn text_document_charset_is_visible_to_document_start_before_body_arrives() { + let _js_runtime_owner = crate::JsRuntime::initialize(); + tokio::task::LocalSet::new().run_until(async { + for (mime, chunk, expected) in [ + ("text/plain; charset=gbk", None, "GBK"), + ("application/json", None, "UTF-8"), + ("text/plain; charset=gbk", Some(b"\xef\xbb\xbfready".as_slice()), "UTF-8"), + ] { + let loader_owner = ResourceRequestClient::new(&FetchConfig::default()).unwrap(); + let loader = loader_owner.handle(); + let mut env = default_test_page_vm_env_config(); + env.document_start_scripts.push(crate::DocumentStartScript { + registry_key: None, + devtools_session: None, + source: "document.title = document.characterSet".to_owned(), + world_name: None, + has_bidi_channel_argument: false, + bidi_channel_handoffs: Vec::new(), + }); + let (mut state, mut decoder) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.to_owned())], &env, + ); + if let Some(chunk) = chunk { + let chunks = decoder.push(chunk); + sync_state_document_character_set_from_decoder(&mut state, &decoder); + for chunk in chunks { + state.parser_session.queue_arrived_chunk(chunk); + } + } + assert_eq!(state.document_character_set, expected); + let (_, page_vm, navigated) = ConcurrentParseTimeRuntime::bootstrap_page_vm_from_state_on_fresh_local_task( + PageId::new_for_testing(1), JsLocalExecutor::new(), loader, env, + PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(), state, + Instant::now(), "charset test bootstrap closed", + ).await.unwrap(); + assert!(!navigated); + assert_eq!(page_vm.vm().snapshot_live_document().document_title(), expected, "{mime}"); + } + }).await; + } + + #[test] + fn response_document_dispatch_distinguishes_text_from_html_and_xml() { + for (mime, text) in [ + ("application/json", true), + ("application/problem+json", true), + ("text/plain; charset=gbk", true), + ("text/javascript", true), + ("text/html", false), + ("application/xhtml+xml", false), + ("application/xml", false), + ("text/xml", false), + ] { + let (state, _) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.to_owned())], + &default_test_page_vm_env_config(), + ); + assert_eq!(state.is_text_document, text, "{mime}"); + } + } + + #[tokio::test] + async fn text_document_does_not_preload_literal_script_tags() { + let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()) + .expect("default loader"); + let runtime_hooks = PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(); + for (mime, expected_preload) in [ + ("text/plain", false), + ("application/json", false), + ("text/html", true), + ] { + let (mut state, _) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.to_owned())], + &default_test_page_vm_env_config(), + ); + state.buffered_document_preloads.bind_resource_runtime( + runtime_hooks.owner_wake(), + runtime_hooks.resource_task_runner(), + ); + scan_prebootstrap_html_chunk_into_state( + &mut state, + &loader, + "", + None, + ); + assert_eq!( + state.buffered_document_preloads.entries.contains_key( + &classic_preload_key_for_streaming_test("https://example.test/literal.js") + ), + expected_preload, + "{mime}", + ); + } + } + #[tokio::test] async fn external_raw_document_body_source_collects_chunks_and_completion() { let (completion_tx, completion_rx) = oneshot::channel(); diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rs index 3ff3a0dd5..35fca10b5 100644 --- a/moli-web-mime/src/classification.rs +++ b/moli-web-mime/src/classification.rs @@ -51,6 +51,14 @@ pub fn is_text_mime_essence(input: &str) -> bool { input.starts_with("text/") } +/// Navigation responses rendered as literal text rather than HTML or XML. +/// This also includes JSON and JavaScript resources opened as documents. +pub fn is_text_document_mime(input: &str) -> bool { + !is_html_document_mime(input) + && !is_dom_parser_xml_mime(input) + && (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input)) +} + /// Whether a `style` element's raw `type` attribute selects classic CSS. /// /// HTML's update-a-style-block algorithm compares the untrimmed attribute diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs index 615e882d2..ec09da8be 100644 --- a/moli-web-mime/src/lib.rs +++ b/moli-web-mime/src/lib.rs @@ -14,9 +14,9 @@ pub use classification::{ is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime, is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence, is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence, - is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, is_text_mime, - is_text_mime_essence, is_video_mime, is_video_mime_essence, is_webassembly_mime, - multipart_form_data_boundary, + is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, + is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime, + is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary, }; pub use data_url::{ data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type, diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rs index e475520a9..a21ab2a60 100644 --- a/moli-web-mime/src/tests.rs +++ b/moli-web-mime/src/tests.rs @@ -648,3 +648,25 @@ fn script_like_mime_type_block_matches_fetch_response_rule() { )); assert!(!should_script_like_response_be_blocked_due_to_mime_type(&[])); } +#[test] +fn document_text_classification_excludes_html_and_xml() { + for mime in [ + "text/plain", + "Text/Css; charset=utf-8", + "application/json", + "application/ld+json", + "application/javascript", + ] { + assert!(crate::is_text_document_mime(mime), "{mime}"); + } + for mime in [ + "text/html", + "text/xml", + "application/xhtml+xml", + "image/svg+xml", + "image/png", + "application/octet-stream", + ] { + assert!(!crate::is_text_document_mime(mime), "{mime}"); + } +} diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rs index a1599c989..e9ed136c6 100644 --- a/moli/src/fetch_dump/tests.rs +++ b/moli/src/fetch_dump/tests.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use anyhow::Result; -use axum::{Router, routing::get}; +use axum::{Router, response::Html, routing::get}; use moli_core::{ LayoutPolicy, page::Page, @@ -36,7 +36,7 @@ async fn load_page_with_config( "/", get(move || { let body = Arc::clone(&server_body); - async move { (*body).clone() } + async move { Html((*body).clone()) } }), ); axum::serve(listener, app).await.unwrap();