From f9231bead70a756b33b16f4f728daaaf9ba714f2 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Sat, 12 Sep 2026 00:30:31 +0800 Subject: [PATCH] fix(xhr): decode response bytes using the response encoding Honor response and override charsets, BOMs, and XML or HTML encoding declarations across buffered, queued, and streaming delivery. Reuse the HTML document decoder with UTF-8 fallback and expose its selected charset. Keep JSON decoding fixed to UTF-8 and binary responses byte-exact. Select the decoder after HEADERS_RECEIVED handlers can change the MIME override. Keep queued response bytes intact and handle split BOMs and multibyte characters in streaming text. --- moli-encoding/src/tests.rs | 132 +++++++++++ moli-renderer-v8/src/dom_parser.rs | 33 +-- .../context_host/resource_loading.rs | 6 +- moli-renderer-v8/src/network_host.rs | 5 +- moli-renderer-v8/src/network_host/xhr.rs | 5 +- .../src/network_host/xhr/delivery/pending.rs | 44 +--- .../src/network_host/xhr/delivery/response.rs | 221 +++--------------- .../src/network_host/xhr/encoding.rs | 37 +++ .../src/network_host/xhr/response_type.rs | 11 + .../src/script_vm/tests/dom_xhr/xhr.rs | 140 +++++++++++ moli-renderer-v8/src/types.rs | 42 +--- 11 files changed, 400 insertions(+), 276 deletions(-) create mode 100644 moli-renderer-v8/src/network_host/xhr/encoding.rs diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs index c7ff1003a0..fd0e471aea 100644 --- a/moli-encoding/src/tests.rs +++ b/moli-encoding/src/tests.rs @@ -5,6 +5,138 @@ use encoding_rs::Encoding; use super::*; use moli_charset_parser::HTML_META_CHARSET_PRESCAN_LIMIT; +#[test] +fn xhr_response_decoder_preserves_encoding_across_every_chunk_boundary() { + use XhrResponseTextKind::{Html, Text, Xml}; + let xml = b"\xff"; + let xml_utf16 = "é"; + let cases = [ + ( + Text, + None, + b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(), + "機", + "UTF-8", + ), + ( + Text, + Some(encoding_rs::WINDOWS_1252), + b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(), + "機", + "UTF-8", + ), + ( + Text, + None, + b"\xef\xbb\xbf\xef\xbb\xbf".to_vec(), + "\u{feff}", + "UTF-8", + ), + (Text, None, b"\xff\xfeA\0\xe9\0".to_vec(), "Aé", "UTF-16LE"), + ( + Text, + Some(encoding_rs::UTF_8), + b"\xfe\xff\0A\0\xe9".to_vec(), + "Aé", + "UTF-16BE", + ), + ( + Text, + Some(encoding_rs::UTF_16LE), + b"\xe9\0".to_vec(), + "é", + "UTF-16LE", + ), + ( + Text, + Some(encoding_rs::SHIFT_JIS), + b"\x83e\x83X\x83g".to_vec(), + "テスト", + "Shift_JIS", + ), + (Text, None, b"a\xc2".to_vec(), "a\u{fffd}", "UTF-8"), + ( + Xml, + None, + xml.to_vec(), + "ÿ", + "windows-1252", + ), + ( + Xml, + Some(encoding_rs::UTF_8), + xml.to_vec(), + "\u{fffd}", + "UTF-8", + ), + ( + Text, + None, + xml.to_vec(), + "\u{fffd}", + "UTF-8", + ), + ( + Xml, + None, + xml_utf16 + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(), + xml_utf16, + "UTF-16LE", + ), + ( + Html, + None, + b"\xff".to_vec(), + "ÿ", + "windows-1252", + ), + ( + Html, + Some(encoding_rs::UTF_8), + b"\xff".to_vec(), + "\u{fffd}", + "UTF-8", + ), + ( + Html, + None, + b"\xe6\xa9\x9f".to_vec(), + "機", + "UTF-8", + ), + ]; + for (kind, transport, bytes, expected, encoding) in cases { + for boundary in 0..=bytes.len() { + let mut decoder = XhrResponseDecoder::new(kind, transport); + let mut text = decoder.push(&bytes[..boundary]); + text.push_str(&decoder.push(&bytes[boundary..])); + text.push_str(&decoder.finish()); + assert_eq!(text, expected, "boundary={boundary}, bytes={bytes:?}"); + assert_eq!(decoder.encoding_name(), encoding); + } + } +} + +#[test] +fn xhr_response_decoder_defers_partial_boms_and_flushes_incomplete_sequences() { + let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None); + assert_eq!(decoder.push(b"\xef"), ""); + assert_eq!(decoder.push(b"\xbb"), ""); + assert_eq!(decoder.push(b"\xbf"), ""); + assert_eq!(decoder.push(b"a\xc2"), "a"); + assert_eq!(decoder.finish(), "\u{fffd}"); + + let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None); + assert_eq!(decoder.push(b"\xff"), ""); + assert_eq!(decoder.push(b"\xfeA"), ""); + assert_eq!(decoder.push(b"\0\xe9"), "A"); + assert_eq!(decoder.push(b"\0"), "é"); + assert_eq!(decoder.finish(), ""); +} + fn gbk_bytes(input: &str) -> Vec { encoding_rs::GBK.encode(input).0.into_owned() } diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs index d99aeaec31..d4579d73ad 100644 --- a/moli-renderer-v8/src/dom_parser.rs +++ b/moli-renderer-v8/src/dom_parser.rs @@ -422,10 +422,26 @@ pub(crate) fn parse_detached_html_document_from_source<'s>( scope: &mut v8::PinScope<'s, '_>, document_url: Url, source: &str, +) -> Option> { + parse_detached_html_document_from_source_with_encoding(scope, document_url, source, None) +} + +pub(crate) fn parse_detached_html_document_from_source_with_encoding<'s>( + scope: &mut v8::PinScope<'s, '_>, + document_url: Url, + source: &str, + character_set: Option<&str>, ) -> Option> { let parsed = HtmlParser::with_scripting_enabled(false) .parse_without_declarative_shadow_roots(document_url, source.to_owned()); - build_detached_document(scope, parsed, DetachedDocumentKind::Html, false) + build_detached_document_from_dom_host_with_content_type( + scope, + DomHost::from_dom(parsed), + DetachedDocumentKind::Html, + false, + Some("text/html"), + character_set, + ) } pub(crate) fn parse_detached_html_document_from_source_with_declarative_shadow_roots<'s>( @@ -513,21 +529,6 @@ fn child_document_url_is_xml_like(url: &Url) -> bool { path.ends_with(".xml") || path.ends_with(".xhtml") || path.ends_with(".svg") } -fn build_detached_document<'s>( - scope: &mut v8::PinScope<'s, '_>, - parsed: NativeDom, - kind: DetachedDocumentKind, - expose_declarative_shadow_roots: bool, -) -> Option> { - build_detached_document_with_content_type( - scope, - parsed, - kind, - expose_declarative_shadow_roots, - None, - ) -} - fn build_detached_document_with_content_type<'s>( scope: &mut v8::PinScope<'s, '_>, parsed: NativeDom, diff --git a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs index e4eb1a2cc4..7d92c0859b 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs @@ -1884,7 +1884,11 @@ impl JsContextHost { let context = v8::Local::new(scope, state.pending.execution_context.context_global()?); let xhr = v8::Local::new(scope, xhr); state.body_writer.append(bytes); - let (decoded_text, loaded, total) = state.xhr_response.as_mut()?.append(bytes); + let headers = &state.head.headers; + let (decoded_text, loaded, total) = state.xhr_response.as_mut()?.append(bytes, || { + let scope = &mut v8::ContextScope::new(scope, context); + crate::network_host::xhr_response_text_decoder(scope, xhr, headers) + }); crate::types::XhrStreamingChunkDelivery { context, xhr, diff --git a/moli-renderer-v8/src/network_host.rs b/moli-renderer-v8/src/network_host.rs index a8b51bc281..9afb4002a2 100644 --- a/moli-renderer-v8/src/network_host.rs +++ b/moli-renderer-v8/src/network_host.rs @@ -200,8 +200,9 @@ pub(crate) use self::xhr::{ install_window_xml_http_request_template_bindings, install_xml_http_request_bindings, install_xml_http_request_event_target_bindings, progress_event_constructor_callback, set_xhr_state_number, throw_synchronous_xhr_failure, xhr_author_request_headers, - xhr_constructor_callback, xhr_ensure_send_allowed, xhr_state_bool_property, - xhr_state_number_property, xhr_state_string_property, + xhr_constructor_callback, xhr_dispatch_progress_event, xhr_ensure_send_allowed, + xhr_response_text_decoder, + xhr_state_bool_property, xhr_state_number_property, xhr_state_string_property, }; pub(in crate::network_host) const NETWORK_BODY_SLOT: &str = "__lmBody"; pub(in crate::network_host) const NETWORK_BODY_BYTES_SLOT: &str = "__lmBodyBytes"; diff --git a/moli-renderer-v8/src/network_host/xhr.rs b/moli-renderer-v8/src/network_host/xhr.rs index c4e47614f5..177ecedb24 100644 --- a/moli-renderer-v8/src/network_host/xhr.rs +++ b/moli-renderer-v8/src/network_host/xhr.rs @@ -1,5 +1,6 @@ mod bindings; mod delivery; +mod encoding; mod events; mod header_surface; mod instance_state; @@ -58,6 +59,8 @@ pub(crate) use self::delivery::{ apply_xhr_streaming_response_chunk, apply_xhr_streaming_response_head, apply_xhr_timeout, finish_xhr_abort, throw_synchronous_xhr_failure, }; +pub(crate) use self::encoding::xhr_response_text_decoder; +pub(crate) use self::events::xhr_dispatch_progress_event; pub(crate) use self::instance_state::xhr_has_upload_listeners; pub(crate) use self::instance_state::{ XHR_METHOD_SLOT, XHR_OPEN_GENERATION_SLOT, XHR_REQUEST_HEADERS_SLOT, XHR_RESPONSE_URL_SLOT, @@ -77,7 +80,7 @@ pub(crate) use self::request_state::begin_xhr_send; use self::request_state::{ XhrReadyState, clear_xhr_active_request, set_xhr_ready_state, xhr_ready_state, }; -use self::response_type::XmlHttpRequestResponseType; +use self::response_type::{XmlHttpRequestResponseType, xhr_response_type}; #[cfg(test)] pub(crate) use self::send::prepare_xhr_send_body; pub(crate) use self::send::{ diff --git a/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs b/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs index c40e03f2e9..1a680da356 100644 --- a/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs +++ b/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs @@ -11,8 +11,6 @@ const RESPONSE_STATUS_TEXT: &str = "statusText"; const RESPONSE_URL: &str = "url"; const RESPONSE_HEADERS: &str = "headers"; const RESPONSE_BODY: &str = "body"; -const RESPONSE_BODY_TEXT: &str = "bodyText"; -const RESPONSE_BODY_LENGTH: &str = "bodyLength"; /// Each queued job owns its complete outcome. A later open()/send() cannot /// overwrite a shared pending payload or let an old job consume the new one. @@ -45,10 +43,6 @@ struct XhrPendingResponse<'s> { headers: String, #[webapi(slot = RESPONSE_BODY)] body: Option>, - #[webapi(slot = RESPONSE_BODY_TEXT)] - body_text: String, - #[webapi(slot = RESPONSE_BODY_LENGTH)] - body_length: usize, } #[derive(WebApiObject)] @@ -70,38 +64,16 @@ pub(in crate::network_host::xhr) fn queue_xhr_response_delivery<'s>( response: Response, ) { let (head, body) = response.into_body(); - let response_type = xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT) - .as_deref() - .and_then(XmlHttpRequestResponseType::parse) - .unwrap_or(XmlHttpRequestResponseType::Default); - let (body, body_text, body_length) = match response_type { - XmlHttpRequestResponseType::ArrayBuffer | XmlHttpRequestResponseType::Blob => { - let bytes = body - .try_into_materialized_bytes() - .expect("XHR response body should remain materialized at the delivery boundary"); - let length = bytes.len(); - let body = - crate::network_host::network_body_source_object_from_bytes(scope, None, bytes); - (Some(body.into()), String::new(), length) - } - XmlHttpRequestResponseType::Json - | XmlHttpRequestResponseType::Document - | XmlHttpRequestResponseType::Default - | XmlHttpRequestResponseType::Text => { - let (text, bytes) = body - .try_into_lossy_materialized_text() - .expect("XHR response body should remain materialized at the delivery boundary"); - (None, text, bytes.len()) - } - }; + let bytes = body + .try_into_materialized_bytes() + .expect("XHR response body should remain materialized at the delivery boundary"); + let body = crate::network_host::network_body_source_object_from_bytes(scope, None, bytes); let response = XhrPendingResponse::new( head.status, head.status_text().to_owned(), head.final_url.to_string(), serde_json::to_string(&head.headers).unwrap_or_else(|_| "[]".to_owned()), - body, - body_text, - body_length, + Some(body.into()), ); queue_xhr_delivery(scope, xhr, XhrPendingDelivery::Response(response)); } @@ -171,10 +143,6 @@ fn xhr_complete_callback<'s>( .unwrap_or_default(); let body = get_private_value(scope, response, RESPONSE_BODY) .filter(|value| !value.is_null_or_undefined()); - let body_text = response_string(scope, response, RESPONSE_BODY_TEXT); - let body_length = get_private_value(scope, response, RESPONSE_BODY_LENGTH) - .and_then(|value| value.number_value(scope)) - .expect("queued XHR body length") as usize; apply_xhr_response_pending_body( scope, xhr, @@ -192,8 +160,6 @@ fn xhr_complete_callback<'s>( negotiated_http_version: None, }, body, - body_text, - Some(body_length), ); } diff --git a/moli-renderer-v8/src/network_host/xhr/delivery/response.rs b/moli-renderer-v8/src/network_host/xhr/delivery/response.rs index 3e470ecd85..06c111bca4 100644 --- a/moli-renderer-v8/src/network_host/xhr/delivery/response.rs +++ b/moli-renderer-v8/src/network_host/xhr/delivery/response.rs @@ -70,35 +70,12 @@ fn apply_xhr_response_body_source_with_mode( status_text: Option<&str>, mode: XhrResponseDeliveryMode, ) { - // XHR's Web-visible surface is still fully materialized. Branch by the - // requested responseType so byte consumers do not pay for an inaccessible - // lossy text view. - match xhr_response_type(scope, xhr) { - XmlHttpRequestResponseType::Json - | XmlHttpRequestResponseType::ArrayBuffer - | XmlHttpRequestResponseType::Blob => { - let body_bytes = body - .try_into_materialized_bytes() - .expect("XHR response body should remain materialized at the V8 boundary"); - apply_xhr_response_bytes(scope, xhr, head, body_bytes, status_text, mode); - } - XmlHttpRequestResponseType::Document - | XmlHttpRequestResponseType::Default - | XmlHttpRequestResponseType::Text => { - let (body_text, body_bytes) = body - .try_into_lossy_materialized_text() - .expect("XHR response body should remain materialized at the V8 boundary"); - apply_xhr_response_body( - scope, - xhr, - head, - Some(body_text), - body_bytes, - status_text, - mode, - ); - } - } + // Choose the response decoder only after HEADERS_RECEIVED handlers have + // had a chance to override the MIME type or change responseType. + let bytes = body + .try_into_materialized_bytes() + .expect("XHR response body should remain materialized at the V8 boundary"); + apply_xhr_response_body(scope, xhr, head, bytes, status_text, mode); } #[derive(Clone, Copy)] @@ -119,50 +96,8 @@ pub(in crate::network_host::xhr) fn apply_xhr_response_pending_body<'s>( xhr: v8::Local<'s, v8::Object>, head: moli_fetch::ResponseHead, body_value: Option>, - fallback_body_text: String, - fallback_body_len: Option, ) { - let response_type = xhr_response_type(scope, xhr); - if response_type == XmlHttpRequestResponseType::ArrayBuffer - && let Some(body_value) = body_value - && let Some(loaded) = buffer_value_byte_length(body_value) - { - apply_xhr_response_prebuilt_value( - scope, - xhr, - head, - response_type, - body_value, - loaded, - None, - XhrResponseDeliveryMode::Buffered, - ); - return; - } - - if body_value.is_none() - && matches!( - response_type, - XmlHttpRequestResponseType::Json - | XmlHttpRequestResponseType::Document - | XmlHttpRequestResponseType::Default - | XmlHttpRequestResponseType::Text - ) - { - let loaded = fallback_body_len.unwrap_or(fallback_body_text.len()); - apply_xhr_response_text( - scope, - xhr, - head, - fallback_body_text, - loaded, - None, - XhrResponseDeliveryMode::Buffered, - ); - return; - } - - let body_bytes = body_value + let bytes = body_value .and_then(|value| { v8::Local::::try_from(value) .ok() @@ -171,33 +106,21 @@ pub(in crate::network_host::xhr) fn apply_xhr_response_pending_body<'s>( }) .or_else(|| blob::buffer_source_bytes_from_value(scope, value)) }) - .unwrap_or_else(|| fallback_body_text.into_bytes()); - apply_xhr_response_bytes( + .unwrap_or_default(); + apply_xhr_response_body( scope, xhr, head, - body_bytes, + bytes, None, XhrResponseDeliveryMode::Buffered, ); } -fn apply_xhr_response_bytes( - scope: &mut v8::PinScope<'_, '_>, - xhr: v8::Local<'_, v8::Object>, - head: moli_fetch::ResponseHead, - body_bytes: Vec, - status_text: Option<&str>, - mode: XhrResponseDeliveryMode, -) { - apply_xhr_response_body(scope, xhr, head, None, body_bytes, status_text, mode); -} - fn apply_xhr_response_body( scope: &mut v8::PinScope<'_, '_>, xhr: v8::Local<'_, v8::Object>, head: moli_fetch::ResponseHead, - body_text: Option, body_bytes: Vec, status_text: Option<&str>, mode: XhrResponseDeliveryMode, @@ -205,15 +128,18 @@ fn apply_xhr_response_body( let Some(response_type) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else { return; }; - let body_text = match response_type { + let (body_text, character_set) = match response_type { XmlHttpRequestResponseType::Document | XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Text => { - Some(body_text.unwrap_or_else(|| String::from_utf8_lossy(&body_bytes).into_owned())) + let mut decoder = xhr_response_text_decoder(scope, xhr, &head.headers); + let mut text = decoder.push(&body_bytes); + text.push_str(&decoder.finish()); + (Some(text), decoder.encoding_name()) } XmlHttpRequestResponseType::Json | XmlHttpRequestResponseType::ArrayBuffer - | XmlHttpRequestResponseType::Blob => None, + | XmlHttpRequestResponseType::Blob => (None, "UTF-8"), }; let loaded = body_bytes.len() as f64; let progress = xhr_response_progress(&head, loaded); @@ -226,6 +152,7 @@ fn apply_xhr_response_body( &head, body_text.as_deref().unwrap_or(""), response_type, + character_set, ); set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document); document @@ -247,6 +174,7 @@ fn apply_xhr_response_body( &head, body_text.as_deref().unwrap_or(""), response_type, + character_set, ); set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document); } @@ -266,52 +194,6 @@ fn apply_xhr_response_body( ); } -fn apply_xhr_response_text( - scope: &mut v8::PinScope<'_, '_>, - xhr: v8::Local<'_, v8::Object>, - head: moli_fetch::ResponseHead, - body_text: String, - loaded: usize, - status_text: Option<&str>, - mode: XhrResponseDeliveryMode, -) { - let Some(response_type) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else { - return; - }; - let progress = xhr_response_progress(&head, loaded as f64); - let response_val: v8::Local<'_, v8::Value> = match response_type { - XmlHttpRequestResponseType::Json => parse_xhr_json_response(scope, body_text.as_bytes()), - XmlHttpRequestResponseType::Document => { - let document = - parse_xhr_response_document(scope, xhr, &head, &body_text, response_type); - set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document); - document - } - XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Text => { - if response_type == XmlHttpRequestResponseType::Default { - let document = - parse_xhr_response_document(scope, xhr, &head, &body_text, response_type); - set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document); - } - v8_string(scope, &body_text) - .map(|s| s.into()) - .unwrap_or_else(|| v8::undefined(scope).into()) - } - XmlHttpRequestResponseType::ArrayBuffer | XmlHttpRequestResponseType::Blob => { - v8::undefined(scope).into() - } - }; - finish_xhr_response( - scope, - xhr, - response_type, - response_val, - &body_text, - progress, - mode, - ); -} - fn parse_xhr_json_response<'s>( scope: &mut v8::PinScope<'s, '_>, bytes: &[u8], @@ -325,23 +207,6 @@ fn parse_xhr_json_response<'s>( .unwrap_or_else(|| v8::null(scope).into()) } -fn apply_xhr_response_prebuilt_value( - scope: &mut v8::PinScope<'_, '_>, - xhr: v8::Local<'_, v8::Object>, - head: moli_fetch::ResponseHead, - response_type: XmlHttpRequestResponseType, - response_val: v8::Local<'_, v8::Value>, - loaded: usize, - status_text: Option<&str>, - mode: XhrResponseDeliveryMode, -) { - let Some(_) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else { - return; - }; - let progress = xhr_response_progress(&head, loaded as f64); - finish_xhr_response(scope, xhr, response_type, response_val, "", progress, mode); -} - pub(crate) fn apply_xhr_streaming_response_head( scope: &mut v8::PinScope<'_, '_>, xhr: v8::Local<'_, v8::Object>, @@ -411,6 +276,16 @@ fn prepare_xhr_response( } if matches!(mode, XhrResponseDeliveryMode::Buffered) { set_xhr_response_head(scope, xhr, head, status_text); + if xhr_is_async(scope, xhr) { + let generation = xhr_state_number_property(scope, xhr, XHR_OPEN_GENERATION_SLOT); + super::super::events::xhr_fire_readystatechange(scope, xhr, XhrReadyState::HeadersReceived); + if scope.is_execution_terminating() + || xhr_is_aborted(scope, xhr) + || xhr_state_number_property(scope, xhr, XHR_OPEN_GENERATION_SLOT) != generation + { + return None; + } + } } Some(xhr_response_type(scope, xhr)) } @@ -457,15 +332,6 @@ fn finish_xhr_response( let dispatch_intermediate_events = matches!(mode, XhrResponseDeliveryMode::Buffered) && xhr_is_async(scope, xhr); - if dispatch_intermediate_events { - super::super::events::xhr_fire_readystatechange(scope, xhr, XhrReadyState::HeadersReceived); - if scope.is_execution_terminating() { - return; - } - if xhr_is_aborted(scope, xhr) { - return; - } - } set_xhr_state_string(scope, xhr, XHR_RESPONSE_TEXT_SLOT, response_text); set_xhr_state_value(scope, xhr, XHR_RESPONSE_SLOT, response_val); if dispatch_intermediate_events { @@ -561,26 +427,6 @@ fn identity_encoded_content_length(headers: &[(String, Vec)]) -> Option .then_some(first) } -fn xhr_response_type( - scope: &mut v8::PinScope<'_, '_>, - xhr: v8::Local<'_, v8::Object>, -) -> XmlHttpRequestResponseType { - xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT) - .as_deref() - .and_then(XmlHttpRequestResponseType::parse) - .unwrap_or(XmlHttpRequestResponseType::Default) -} - -fn buffer_value_byte_length(value: v8::Local<'_, v8::Value>) -> Option { - if let Ok(buffer) = v8::Local::::try_from(value) { - return Some(buffer.byte_length()); - } - if let Ok(view) = v8::Local::::try_from(value) { - return Some(view.byte_length()); - } - None -} - fn xhr_response_mime_essence( scope: &mut v8::PinScope<'_, '_>, xhr: v8::Local<'_, v8::Object>, @@ -603,6 +449,7 @@ fn parse_xhr_response_document<'s>( head: &moli_fetch::ResponseHead, body_text: &str, response_type: XmlHttpRequestResponseType, + character_set: &str, ) -> v8::Local<'s, v8::Value> { // XHR defaults a missing or invalid response MIME type to text/xml. // Its XML MIME types include any +xml subtype, unlike DOMParser's enum. @@ -613,7 +460,7 @@ fn parse_xhr_response_document<'s>( if (!is_xml && !is_html) || (is_html && response_type != XmlHttpRequestResponseType::Document) { return v8::null(scope).into(); } - let Some(document) = build_xhr_response_document(scope, xhr, xhr_response_url(head).clone(), body_text, &mime) + let Some(document) = build_xhr_response_document(scope, xhr, xhr_response_url(head).clone(), body_text, &mime, character_set) else { return v8::null(scope).into(); }; @@ -639,6 +486,7 @@ fn build_xhr_response_document<'s>( response_url: url::Url, body_text: &str, mime: &str, + character_set: &str, ) -> Option> { let xhr = v8::Global::new(scope, xhr); let xhr = v8::Local::new(scope, xhr); @@ -662,7 +510,12 @@ fn build_xhr_response_document<'s>( origin_document_handle, )?; let document = if is_html_document_mime(mime) { - dom_parser::parse_detached_html_document_from_source(scope, response_url, body_text) + dom_parser::parse_detached_html_document_from_source_with_encoding( + scope, + response_url, + body_text, + Some(character_set), + ) } else { dom_parser::parse_detached_xml_document_from_source( scope, diff --git a/moli-renderer-v8/src/network_host/xhr/encoding.rs b/moli-renderer-v8/src/network_host/xhr/encoding.rs new file mode 100644 index 0000000000..ab48879c34 --- /dev/null +++ b/moli-renderer-v8/src/network_host/xhr/encoding.rs @@ -0,0 +1,37 @@ +use moli_encoding::{XhrResponseDecoder, XhrResponseTextKind, encoding_for_label}; +use moli_web_mime::{effective_response_mime_essence, mime_charset, response_content_type}; + +use super::*; + +pub(crate) fn xhr_response_text_decoder( + scope: &mut v8::PinScope<'_, '_>, + xhr: v8::Local<'_, v8::Object>, + headers: &[(String, String)], +) -> XhrResponseDecoder { + let override_mime = xhr_state_string_property(scope, xhr, XHR_OVERRIDE_MIME_TYPE_SLOT); + // A MIME-only override must retain the response charset. Blink also + // ignores an empty override charset, but an unknown nonempty label + // replaces the transport label and lets content/default detection run. + let charset = override_mime + .as_deref() + .and_then(mime_charset) + .filter(|label| !label.is_empty()) + .or_else(|| { + response_content_type(headers) + .as_deref() + .and_then(mime_charset) + }); + let mime = effective_response_mime_essence(headers, override_mime.as_deref()) + .unwrap_or_else(|| "text/xml".to_owned()); + let kind = match xhr_response_type(scope, xhr) { + XmlHttpRequestResponseType::Document if mime == "text/html" => XhrResponseTextKind::Html, + XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Document + if matches!(mime.as_str(), "text/xml" | "application/xml") + || mime.ends_with("+xml") => + { + XhrResponseTextKind::Xml + } + _ => XhrResponseTextKind::Text, + }; + XhrResponseDecoder::new(kind, charset.as_deref().and_then(encoding_for_label)) +} diff --git a/moli-renderer-v8/src/network_host/xhr/response_type.rs b/moli-renderer-v8/src/network_host/xhr/response_type.rs index e9de63563c..2b321f123d 100644 --- a/moli-renderer-v8/src/network_host/xhr/response_type.rs +++ b/moli-renderer-v8/src/network_host/xhr/response_type.rs @@ -1,3 +1,4 @@ +use super::{XHR_RESPONSE_TYPE_SLOT, xhr_state_string_property}; use std::str::FromStr; #[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::IntoStaticStr)] @@ -22,6 +23,16 @@ impl XmlHttpRequestResponseType { } } +pub(super) fn xhr_response_type( + scope: &mut v8::PinScope<'_, '_>, + xhr: v8::Local<'_, v8::Object>, +) -> XmlHttpRequestResponseType { + xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT) + .as_deref() + .and_then(XmlHttpRequestResponseType::parse) + .unwrap_or(XmlHttpRequestResponseType::Default) +} + #[cfg(test)] mod tests { use super::XmlHttpRequestResponseType; diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs index e95e485ea5..5b8c01a5c4 100644 --- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs +++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs @@ -1454,6 +1454,146 @@ fn xml_http_request_document_response_requires_an_eligible_mime_and_well_formed_ } } +#[test] +fn xhr_response_decoding_uses_headers_received_overrides_for_buffered_and_streamed_bytes() { + for streaming in [false, true] { + for (chunks, partial, expected) in [ + ( + vec![vec![0x83], vec![0x65, 0x83], vec![0x58, 0x83, 0x67]], + vec!["", "テ", "テスト"], + "テスト", + ), + ( + vec![vec![0xff], vec![0xfe, b'A'], vec![0, 0xe9], vec![0]], + vec!["", "", "A", "Aé"], + "Aé", + ), + ] { + let mut vm = new_storage_test_vm("https://xhr-response-decoding.test/"); + vm.set_fetch_subresource_interception( + true, + Some(crate::types::SubresourceResourceType::Xhr), + ); + vm.eval( + r#" + globalThis.__decodingXhr = new XMLHttpRequest(); + globalThis.__decodingHeadersText = null; + __decodingXhr.open('GET', '/response'); + __decodingXhr.onreadystatechange = () => { + if (__decodingXhr.readyState === 2) { + __decodingHeadersText = __decodingXhr.responseText; + __decodingXhr.overrideMimeType('text/plain;charset=Shift_JIS'); + } + }; + __decodingXhr.send(); + "#, + ) + .unwrap(); + let requests = vm.take_pending_subresource_fetch_infos(); + assert_eq!(requests.len(), 1); + let request = &requests[0]; + let bytes = chunks.concat(); + let head = moli_fetch::ResponseHead { + final_url: request.url.clone(), + status: 200, + status_text: None, + headers: vec![ + ( + "Content-Type".to_owned(), + "text/plain;charset=UTF-8".to_owned(), + ), + ("Content-Length".to_owned(), bytes.len().to_string()), + ], + request_cookie_report: None, + cookie_set_reports: Vec::new(), + redirected: false, + redirect_chain: Vec::new(), + from_cache: false, + negotiated_http_version: None, + }; + if streaming { + let body_source_id = crate::network_host::new_network_body_source_id(); + vm.start_streaming_async_subresource_fetch( + crate::types::AsyncSubresourceStreamingStarted { + internal_id: request.internal_id, + request_url: request.url.clone(), + request_method: "GET".to_owned(), + request_headers: Vec::new(), + request_body: None, + body_source_id, + head, + network_request_headers: None, + }, + ) + .unwrap(); + for (chunk, text) in chunks.into_iter().zip(partial) { + vm.append_streaming_async_subresource_fetch_chunk(body_source_id, chunk); + assert_eq!(vm.eval("__decodingXhr.responseText").unwrap(), text); + } + vm.finish_streaming_async_subresource_fetch( + request.internal_id, + body_source_id, + Ok(()), + ) + .unwrap(); + } else { + let context_ptr: *const v8::Global = &vm.page_default_context; + vm.renderer_document_isolate + .with_entered_renderer_document_isolate(move |isolate| { + let scope = std::pin::pin!(v8::HandleScope::new(isolate)); + let scope = &mut scope.init(); + let context = unsafe { v8::Local::new(scope, &*context_ptr) }; + let scope = &mut v8::ContextScope::new(scope, context); + let xhr = context + .global(scope) + .get(scope, v8str(scope, "__decodingXhr").into()) + .and_then(|value| v8::Local::::try_from(value).ok()) + .unwrap(); + crate::network_host::apply_xhr_response_body_source( + scope, + xhr, + head, + moli_fetch::ResponseBody::materialized_bytes(bytes), + ); + Ok(()) + }) + .unwrap(); + } + assert_eq!(vm.eval("__decodingHeadersText").unwrap(), ""); + assert_eq!(vm.eval("__decodingXhr.responseText").unwrap(), expected); + assert_eq!(vm.eval("__decodingXhr.response === __decodingXhr.responseText && __decodingXhr.readyState === 4").unwrap(), "true"); + } + } +} + +#[test] +fn xhr_queued_response_retains_legacy_bytes_and_decodes_json_as_utf8() { + let mut vm = new_storage_test_vm("https://xhr-queued-decoding.test/"); + vm.eval(r#" + globalThis.__queuedDecoding = []; + for (const [url, type, override] of [ + ['data:text/plain;charset=windows-1252,%FF', '', null], + ['data:text/plain;charset=utf-8,%83%65%83%58%83%67', '', 'text/plain;charset=Shift_JIS'], + ['data:application/json;charset=windows-1252,%EF%BB%BF%7B%22x%22%3A1%7D', 'json', null], + ['data:text/html,%3Cmeta%20charset=windows-1252%3E%3Cx%3E%FF%3C%2Fx%3E', 'document', null], + ]) { + const xhr = new XMLHttpRequest(); + xhr.open('GET', url); + xhr.responseType = type; + xhr.onreadystatechange = () => { + if (xhr.readyState === 2 && override) xhr.overrideMimeType(override); + }; + xhr.onload = () => __queuedDecoding.push(type === 'json' ? xhr.response.x : + type === 'document' ? [xhr.response.querySelector('x').textContent, xhr.response.characterSet] : xhr.responseText); + xhr.send(); + } + "#).unwrap(); + assert_eq!( + vm.eval("JSON.stringify(__queuedDecoding)").unwrap(), + r#"["ÿ","テスト",1,["ÿ","windows-1252"]]"# + ); +} + #[test] fn xml_http_request_queued_document_response_uses_parsed_overrides_and_xml_errors() { let mut vm = new_storage_test_vm("https://xhr-queued-document.test/"); diff --git a/moli-renderer-v8/src/types.rs b/moli-renderer-v8/src/types.rs index 3c35401feb..7bb3d15e4a 100644 --- a/moli-renderer-v8/src/types.rs +++ b/moli-renderer-v8/src/types.rs @@ -1029,7 +1029,7 @@ pub(super) struct EventSourceStreamingChunkDelivery<'s> { } pub(super) struct XhrStreamingResponseState { - pending_utf8_bytes: Vec, + decoder: Option, loaded: usize, total: Option, } @@ -1046,44 +1046,20 @@ impl XhrStreamingResponseState { .ok() }); Self { - pending_utf8_bytes: Vec::new(), + decoder: None, loaded: 0, total, } } - pub(super) fn append(&mut self, bytes: &[u8]) -> (String, usize, Option) { + pub(super) fn append( + &mut self, + bytes: &[u8], + make_decoder: impl FnOnce() -> moli_encoding::XhrResponseDecoder, + ) -> (String, usize, Option) { self.loaded = self.loaded.saturating_add(bytes.len()); - self.pending_utf8_bytes.extend_from_slice(bytes); - - let mut decoded = String::new(); - let mut consumed = 0; - while consumed < self.pending_utf8_bytes.len() { - let remaining = &self.pending_utf8_bytes[consumed..]; - match std::str::from_utf8(remaining) { - Ok(text) => { - decoded.push_str(text); - consumed = self.pending_utf8_bytes.len(); - } - Err(error) => { - let valid_end = consumed + error.valid_up_to(); - decoded.push_str( - std::str::from_utf8(&self.pending_utf8_bytes[consumed..valid_end]) - .expect("Utf8Error::valid_up_to must identify a valid UTF-8 prefix"), - ); - consumed = valid_end; - let Some(invalid_len) = error.error_len() else { - break; - }; - decoded.push('\u{fffd}'); - consumed += invalid_len; - } - } - } - if consumed > 0 { - self.pending_utf8_bytes.drain(..consumed); - } - (decoded, self.loaded, self.total) + let decoder = self.decoder.get_or_insert_with(make_decoder); + (decoder.push(bytes), self.loaded, self.total) } }