diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs index c7ff1003a0..fd0e471aea 100644 --- a/moli-encoding/src/tests.rs +++ b/moli-encoding/src/tests.rs @@ -5,6 +5,138 @@ use encoding_rs::Encoding; use super::*; use moli_charset_parser::HTML_META_CHARSET_PRESCAN_LIMIT; +#[test] +fn xhr_response_decoder_preserves_encoding_across_every_chunk_boundary() { + use XhrResponseTextKind::{Html, Text, Xml}; + let xml = b"\xff"; + let xml_utf16 = "é"; + let cases = [ + ( + Text, + None, + b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(), + "機", + "UTF-8", + ), + ( + Text, + Some(encoding_rs::WINDOWS_1252), + b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(), + "機", + "UTF-8", + ), + ( + Text, + None, + b"\xef\xbb\xbf\xef\xbb\xbf".to_vec(), + "\u{feff}", + "UTF-8", + ), + (Text, None, b"\xff\xfeA\0\xe9\0".to_vec(), "Aé", "UTF-16LE"), + ( + Text, + Some(encoding_rs::UTF_8), + b"\xfe\xff\0A\0\xe9".to_vec(), + "Aé", + "UTF-16BE", + ), + ( + Text, + Some(encoding_rs::UTF_16LE), + b"\xe9\0".to_vec(), + "é", + "UTF-16LE", + ), + ( + Text, + Some(encoding_rs::SHIFT_JIS), + b"\x83e\x83X\x83g".to_vec(), + "テスト", + "Shift_JIS", + ), + (Text, None, b"a\xc2".to_vec(), "a\u{fffd}", "UTF-8"), + ( + Xml, + None, + xml.to_vec(), + "ÿ", + "windows-1252", + ), + ( + Xml, + Some(encoding_rs::UTF_8), + xml.to_vec(), + "\u{fffd}", + "UTF-8", + ), + ( + Text, + None, + xml.to_vec(), + "\u{fffd}", + "UTF-8", + ), + ( + Xml, + None, + xml_utf16 + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(), + xml_utf16, + "UTF-16LE", + ), + ( + Html, + None, + b"\xff".to_vec(), + "ÿ", + "windows-1252", + ), + ( + Html, + Some(encoding_rs::UTF_8), + b"\xff".to_vec(), + "\u{fffd}", + "UTF-8", + ), + ( + Html, + None, + b"\xe6\xa9\x9f".to_vec(), + "機", + "UTF-8", + ), + ]; + for (kind, transport, bytes, expected, encoding) in cases { + for boundary in 0..=bytes.len() { + let mut decoder = XhrResponseDecoder::new(kind, transport); + let mut text = decoder.push(&bytes[..boundary]); + text.push_str(&decoder.push(&bytes[boundary..])); + text.push_str(&decoder.finish()); + assert_eq!(text, expected, "boundary={boundary}, bytes={bytes:?}"); + assert_eq!(decoder.encoding_name(), encoding); + } + } +} + +#[test] +fn xhr_response_decoder_defers_partial_boms_and_flushes_incomplete_sequences() { + let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None); + assert_eq!(decoder.push(b"\xef"), ""); + assert_eq!(decoder.push(b"\xbb"), ""); + assert_eq!(decoder.push(b"\xbf"), ""); + assert_eq!(decoder.push(b"a\xc2"), "a"); + assert_eq!(decoder.finish(), "\u{fffd}"); + + let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None); + assert_eq!(decoder.push(b"\xff"), ""); + assert_eq!(decoder.push(b"\xfeA"), ""); + assert_eq!(decoder.push(b"\0\xe9"), "A"); + assert_eq!(decoder.push(b"\0"), "é"); + assert_eq!(decoder.finish(), ""); +} + fn gbk_bytes(input: &str) -> Vec { encoding_rs::GBK.encode(input).0.into_owned() } diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs index d99aeaec31..d4579d73ad 100644 --- a/moli-renderer-v8/src/dom_parser.rs +++ b/moli-renderer-v8/src/dom_parser.rs @@ -422,10 +422,26 @@ pub(crate) fn parse_detached_html_document_from_source<'s>( scope: &mut v8::PinScope<'s, '_>, document_url: Url, source: &str, +) -> Option> { + parse_detached_html_document_from_source_with_encoding(scope, document_url, source, None) +} + +pub(crate) fn parse_detached_html_document_from_source_with_encoding<'s>( + scope: &mut v8::PinScope<'s, '_>, + document_url: Url, + source: &str, + character_set: Option<&str>, ) -> Option> { let parsed = HtmlParser::with_scripting_enabled(false) .parse_without_declarative_shadow_roots(document_url, source.to_owned()); - build_detached_document(scope, parsed, DetachedDocumentKind::Html, false) + build_detached_document_from_dom_host_with_content_type( + scope, + DomHost::from_dom(parsed), + DetachedDocumentKind::Html, + false, + Some("text/html"), + character_set, + ) } pub(crate) fn parse_detached_html_document_from_source_with_declarative_shadow_roots<'s>( @@ -513,21 +529,6 @@ fn child_document_url_is_xml_like(url: &Url) -> bool { path.ends_with(".xml") || path.ends_with(".xhtml") || path.ends_with(".svg") } -fn build_detached_document<'s>( - scope: &mut v8::PinScope<'s, '_>, - parsed: NativeDom, - kind: DetachedDocumentKind, - expose_declarative_shadow_roots: bool, -) -> Option> { - build_detached_document_with_content_type( - scope, - parsed, - kind, - expose_declarative_shadow_roots, - None, - ) -} - fn build_detached_document_with_content_type<'s>( scope: &mut v8::PinScope<'s, '_>, parsed: NativeDom, diff --git a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs index e4eb1a2cc4..7d92c0859b 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs @@ -1884,7 +1884,11 @@ impl JsContextHost { let context = v8::Local::new(scope, state.pending.execution_context.context_global()?); let xhr = v8::Local::new(scope, xhr); state.body_writer.append(bytes); - let (decoded_text, loaded, total) = state.xhr_response.as_mut()?.append(bytes); + let headers = &state.head.headers; + let (decoded_text, loaded, total) = state.xhr_response.as_mut()?.append(bytes, || { + let scope = &mut v8::ContextScope::new(scope, context); + crate::network_host::xhr_response_text_decoder(scope, xhr, headers) + }); crate::types::XhrStreamingChunkDelivery { context, xhr, diff --git a/moli-renderer-v8/src/network_host.rs b/moli-renderer-v8/src/network_host.rs index a8b51bc281..9afb4002a2 100644 --- a/moli-renderer-v8/src/network_host.rs +++ b/moli-renderer-v8/src/network_host.rs @@ -200,8 +200,9 @@ pub(crate) use self::xhr::{ install_window_xml_http_request_template_bindings, install_xml_http_request_bindings, install_xml_http_request_event_target_bindings, progress_event_constructor_callback, set_xhr_state_number, throw_synchronous_xhr_failure, xhr_author_request_headers, - xhr_constructor_callback, xhr_ensure_send_allowed, xhr_state_bool_property, - xhr_state_number_property, xhr_state_string_property, + xhr_constructor_callback, xhr_dispatch_progress_event, xhr_ensure_send_allowed, + xhr_response_text_decoder, + xhr_state_bool_property, xhr_state_number_property, xhr_state_string_property, }; pub(in crate::network_host) const NETWORK_BODY_SLOT: &str = "__lmBody"; pub(in crate::network_host) const NETWORK_BODY_BYTES_SLOT: &str = "__lmBodyBytes"; diff --git a/moli-renderer-v8/src/network_host/xhr.rs b/moli-renderer-v8/src/network_host/xhr.rs index c4e47614f5..177ecedb24 100644 --- a/moli-renderer-v8/src/network_host/xhr.rs +++ b/moli-renderer-v8/src/network_host/xhr.rs @@ -1,5 +1,6 @@ mod bindings; mod delivery; +mod encoding; mod events; mod header_surface; mod instance_state; @@ -58,6 +59,8 @@ pub(crate) use self::delivery::{ apply_xhr_streaming_response_chunk, apply_xhr_streaming_response_head, apply_xhr_timeout, finish_xhr_abort, throw_synchronous_xhr_failure, }; +pub(crate) use self::encoding::xhr_response_text_decoder; +pub(crate) use self::events::xhr_dispatch_progress_event; pub(crate) use self::instance_state::xhr_has_upload_listeners; pub(crate) use self::instance_state::{ XHR_METHOD_SLOT, XHR_OPEN_GENERATION_SLOT, XHR_REQUEST_HEADERS_SLOT, XHR_RESPONSE_URL_SLOT, @@ -77,7 +80,7 @@ pub(crate) use self::request_state::begin_xhr_send; use self::request_state::{ XhrReadyState, clear_xhr_active_request, set_xhr_ready_state, xhr_ready_state, }; -use self::response_type::XmlHttpRequestResponseType; +use self::response_type::{XmlHttpRequestResponseType, xhr_response_type}; #[cfg(test)] pub(crate) use self::send::prepare_xhr_send_body; pub(crate) use self::send::{ diff --git a/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs b/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs index c40e03f2e9..1a680da356 100644 --- a/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs +++ b/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs @@ -11,8 +11,6 @@ const RESPONSE_STATUS_TEXT: &str = "statusText"; const RESPONSE_URL: &str = "url"; const RESPONSE_HEADERS: &str = "headers"; const RESPONSE_BODY: &str = "body"; -const RESPONSE_BODY_TEXT: &str = "bodyText"; -const RESPONSE_BODY_LENGTH: &str = "bodyLength"; /// Each queued job owns its complete outcome. A later open()/send() cannot /// overwrite a shared pending payload or let an old job consume the new one. @@ -45,10 +43,6 @@ struct XhrPendingResponse<'s> { headers: String, #[webapi(slot = RESPONSE_BODY)] body: Option>, - #[webapi(slot = RESPONSE_BODY_TEXT)] - body_text: String, - #[webapi(slot = RESPONSE_BODY_LENGTH)] - body_length: usize, } #[derive(WebApiObject)] @@ -70,38 +64,16 @@ pub(in crate::network_host::xhr) fn queue_xhr_response_delivery<'s>( response: Response, ) { let (head, body) = response.into_body(); - let response_type = xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT) - .as_deref() - .and_then(XmlHttpRequestResponseType::parse) - .unwrap_or(XmlHttpRequestResponseType::Default); - let (body, body_text, body_length) = match response_type { - XmlHttpRequestResponseType::ArrayBuffer | XmlHttpRequestResponseType::Blob => { - let bytes = body - .try_into_materialized_bytes() - .expect("XHR response body should remain materialized at the delivery boundary"); - let length = bytes.len(); - let body = - crate::network_host::network_body_source_object_from_bytes(scope, None, bytes); - (Some(body.into()), String::new(), length) - } - XmlHttpRequestResponseType::Json - | XmlHttpRequestResponseType::Document - | XmlHttpRequestResponseType::Default - | XmlHttpRequestResponseType::Text => { - let (text, bytes) = body - .try_into_lossy_materialized_text() - .expect("XHR response body should remain materialized at the delivery boundary"); - (None, text, bytes.len()) - } - }; + let bytes = body + .try_into_materialized_bytes() + .expect("XHR response body should remain materialized at the delivery boundary"); + let body = crate::network_host::network_body_source_object_from_bytes(scope, None, bytes); let response = XhrPendingResponse::new( head.status, head.status_text().to_owned(), head.final_url.to_string(), serde_json::to_string(&head.headers).unwrap_or_else(|_| "[]".to_owned()), - body, - body_text, - body_length, + Some(body.into()), ); queue_xhr_delivery(scope, xhr, XhrPendingDelivery::Response(response)); } @@ -171,10 +143,6 @@ fn xhr_complete_callback<'s>( .unwrap_or_default(); let body = get_private_value(scope, response, RESPONSE_BODY) .filter(|value| !value.is_null_or_undefined()); - let body_text = response_string(scope, response, RESPONSE_BODY_TEXT); - let body_length = get_private_value(scope, response, RESPONSE_BODY_LENGTH) - .and_then(|value| value.number_value(scope)) - .expect("queued XHR body length") as usize; apply_xhr_response_pending_body( scope, xhr, @@ -192,8 +160,6 @@ fn xhr_complete_callback<'s>( negotiated_http_version: None, }, body, - body_text, - Some(body_length), ); } diff --git a/moli-renderer-v8/src/network_host/xhr/delivery/response.rs b/moli-renderer-v8/src/network_host/xhr/delivery/response.rs index 3e470ecd85..06c111bca4 100644 --- a/moli-renderer-v8/src/network_host/xhr/delivery/response.rs +++ b/moli-renderer-v8/src/network_host/xhr/delivery/response.rs @@ -70,35 +70,12 @@ fn apply_xhr_response_body_source_with_mode( status_text: Option<&str>, mode: XhrResponseDeliveryMode, ) { - // XHR's Web-visible surface is still fully materialized. Branch by the - // requested responseType so byte consumers do not pay for an inaccessible - // lossy text view. - match xhr_response_type(scope, xhr) { - XmlHttpRequestResponseType::Json - | XmlHttpRequestResponseType::ArrayBuffer - | XmlHttpRequestResponseType::Blob => { - let body_bytes = body - .try_into_materialized_bytes() - .expect("XHR response body should remain materialized at the V8 boundary"); - apply_xhr_response_bytes(scope, xhr, head, body_bytes, status_text, mode); - } - XmlHttpRequestResponseType::Document - | XmlHttpRequestResponseType::Default - | XmlHttpRequestResponseType::Text => { - let (body_text, body_bytes) = body - .try_into_lossy_materialized_text() - .expect("XHR response body should remain materialized at the V8 boundary"); - apply_xhr_response_body( - scope, - xhr, - head, - Some(body_text), - body_bytes, - status_text, - mode, - ); - } - } + // Choose the response decoder only after HEADERS_RECEIVED handlers have + // had a chance to override the MIME type or change responseType. + let bytes = body + .try_into_materialized_bytes() + .expect("XHR response body should remain materialized at the V8 boundary"); + apply_xhr_response_body(scope, xhr, head, bytes, status_text, mode); } #[derive(Clone, Copy)] @@ -119,50 +96,8 @@ pub(in crate::network_host::xhr) fn apply_xhr_response_pending_body<'s>( xhr: v8::Local<'s, v8::Object>, head: moli_fetch::ResponseHead, body_value: Option>, - fallback_body_text: String, - fallback_body_len: Option, ) { - let response_type = xhr_response_type(scope, xhr); - if response_type == XmlHttpRequestResponseType::ArrayBuffer - && let Some(body_value) = body_value - && let Some(loaded) = buffer_value_byte_length(body_value) - { - apply_xhr_response_prebuilt_value( - scope, - xhr, - head, - response_type, - body_value, - loaded, - None, - XhrResponseDeliveryMode::Buffered, - ); - return; - } - - if body_value.is_none() - && matches!( - response_type, - XmlHttpRequestResponseType::Json - | XmlHttpRequestResponseType::Document - | XmlHttpRequestResponseType::Default - | XmlHttpRequestResponseType::Text - ) - { - let loaded = fallback_body_len.unwrap_or(fallback_body_text.len()); - apply_xhr_response_text( - scope, - xhr, - head, - fallback_body_text, - loaded, - None, - XhrResponseDeliveryMode::Buffered, - ); - return; - } - - let body_bytes = body_value + let bytes = body_value .and_then(|value| { v8::Local::::try_from(value) .ok() @@ -171,33 +106,21 @@ pub(in crate::network_host::xhr) fn apply_xhr_response_pending_body<'s>( }) .or_else(|| blob::buffer_source_bytes_from_value(scope, value)) }) - .unwrap_or_else(|| fallback_body_text.into_bytes()); - apply_xhr_response_bytes( + .unwrap_or_default(); + apply_xhr_response_body( scope, xhr, head, - body_bytes, + bytes, None, XhrResponseDeliveryMode::Buffered, ); } -fn apply_xhr_response_bytes( - scope: &mut v8::PinScope<'_, '_>, - xhr: v8::Local<'_, v8::Object>, - head: moli_fetch::ResponseHead, - body_bytes: Vec, - status_text: Option<&str>, - mode: XhrResponseDeliveryMode, -) { - apply_xhr_response_body(scope, xhr, head, None, body_bytes, status_text, mode); -} - fn apply_xhr_response_body( scope: &mut v8::PinScope<'_, '_>, xhr: v8::Local<'_, v8::Object>, head: moli_fetch::ResponseHead, - body_text: Option, body_bytes: Vec, status_text: Option<&str>, mode: XhrResponseDeliveryMode, @@ -205,15 +128,18 @@ fn apply_xhr_response_body( let Some(response_type) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else { return; }; - let body_text = match response_type { + let (body_text, character_set) = match response_type { XmlHttpRequestResponseType::Document | XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Text => { - Some(body_text.unwrap_or_else(|| String::from_utf8_lossy(&body_bytes).into_owned())) + let mut decoder = xhr_response_text_decoder(scope, xhr, &head.headers); + let mut text = decoder.push(&body_bytes); + text.push_str(&decoder.finish()); + (Some(text), decoder.encoding_name()) } XmlHttpRequestResponseType::Json | XmlHttpRequestResponseType::ArrayBuffer - | XmlHttpRequestResponseType::Blob => None, + | XmlHttpRequestResponseType::Blob => (None, "UTF-8"), }; let loaded = body_bytes.len() as f64; let progress = xhr_response_progress(&head, loaded); @@ -226,6 +152,7 @@ fn apply_xhr_response_body( &head, body_text.as_deref().unwrap_or(""), response_type, + character_set, ); set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document); document @@ -247,6 +174,7 @@ fn apply_xhr_response_body( &head, body_text.as_deref().unwrap_or(""), response_type, + character_set, ); set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document); } @@ -266,52 +194,6 @@ fn apply_xhr_response_body( ); } -fn apply_xhr_response_text( - scope: &mut v8::PinScope<'_, '_>, - xhr: v8::Local<'_, v8::Object>, - head: moli_fetch::ResponseHead, - body_text: String, - loaded: usize, - status_text: Option<&str>, - mode: XhrResponseDeliveryMode, -) { - let Some(response_type) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else { - return; - }; - let progress = xhr_response_progress(&head, loaded as f64); - let response_val: v8::Local<'_, v8::Value> = match response_type { - XmlHttpRequestResponseType::Json => parse_xhr_json_response(scope, body_text.as_bytes()), - XmlHttpRequestResponseType::Document => { - let document = - parse_xhr_response_document(scope, xhr, &head, &body_text, response_type); - set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document); - document - } - XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Text => { - if response_type == XmlHttpRequestResponseType::Default { - let document = - parse_xhr_response_document(scope, xhr, &head, &body_text, response_type); - set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document); - } - v8_string(scope, &body_text) - .map(|s| s.into()) - .unwrap_or_else(|| v8::undefined(scope).into()) - } - XmlHttpRequestResponseType::ArrayBuffer | XmlHttpRequestResponseType::Blob => { - v8::undefined(scope).into() - } - }; - finish_xhr_response( - scope, - xhr, - response_type, - response_val, - &body_text, - progress, - mode, - ); -} - fn parse_xhr_json_response<'s>( scope: &mut v8::PinScope<'s, '_>, bytes: &[u8], @@ -325,23 +207,6 @@ fn parse_xhr_json_response<'s>( .unwrap_or_else(|| v8::null(scope).into()) } -fn apply_xhr_response_prebuilt_value( - scope: &mut v8::PinScope<'_, '_>, - xhr: v8::Local<'_, v8::Object>, - head: moli_fetch::ResponseHead, - response_type: XmlHttpRequestResponseType, - response_val: v8::Local<'_, v8::Value>, - loaded: usize, - status_text: Option<&str>, - mode: XhrResponseDeliveryMode, -) { - let Some(_) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else { - return; - }; - let progress = xhr_response_progress(&head, loaded as f64); - finish_xhr_response(scope, xhr, response_type, response_val, "", progress, mode); -} - pub(crate) fn apply_xhr_streaming_response_head( scope: &mut v8::PinScope<'_, '_>, xhr: v8::Local<'_, v8::Object>, @@ -411,6 +276,16 @@ fn prepare_xhr_response( } if matches!(mode, XhrResponseDeliveryMode::Buffered) { set_xhr_response_head(scope, xhr, head, status_text); + if xhr_is_async(scope, xhr) { + let generation = xhr_state_number_property(scope, xhr, XHR_OPEN_GENERATION_SLOT); + super::super::events::xhr_fire_readystatechange(scope, xhr, XhrReadyState::HeadersReceived); + if scope.is_execution_terminating() + || xhr_is_aborted(scope, xhr) + || xhr_state_number_property(scope, xhr, XHR_OPEN_GENERATION_SLOT) != generation + { + return None; + } + } } Some(xhr_response_type(scope, xhr)) } @@ -457,15 +332,6 @@ fn finish_xhr_response( let dispatch_intermediate_events = matches!(mode, XhrResponseDeliveryMode::Buffered) && xhr_is_async(scope, xhr); - if dispatch_intermediate_events { - super::super::events::xhr_fire_readystatechange(scope, xhr, XhrReadyState::HeadersReceived); - if scope.is_execution_terminating() { - return; - } - if xhr_is_aborted(scope, xhr) { - return; - } - } set_xhr_state_string(scope, xhr, XHR_RESPONSE_TEXT_SLOT, response_text); set_xhr_state_value(scope, xhr, XHR_RESPONSE_SLOT, response_val); if dispatch_intermediate_events { @@ -561,26 +427,6 @@ fn identity_encoded_content_length(headers: &[(String, Vec)]) -> Option .then_some(first) } -fn xhr_response_type( - scope: &mut v8::PinScope<'_, '_>, - xhr: v8::Local<'_, v8::Object>, -) -> XmlHttpRequestResponseType { - xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT) - .as_deref() - .and_then(XmlHttpRequestResponseType::parse) - .unwrap_or(XmlHttpRequestResponseType::Default) -} - -fn buffer_value_byte_length(value: v8::Local<'_, v8::Value>) -> Option { - if let Ok(buffer) = v8::Local::::try_from(value) { - return Some(buffer.byte_length()); - } - if let Ok(view) = v8::Local::::try_from(value) { - return Some(view.byte_length()); - } - None -} - fn xhr_response_mime_essence( scope: &mut v8::PinScope<'_, '_>, xhr: v8::Local<'_, v8::Object>, @@ -603,6 +449,7 @@ fn parse_xhr_response_document<'s>( head: &moli_fetch::ResponseHead, body_text: &str, response_type: XmlHttpRequestResponseType, + character_set: &str, ) -> v8::Local<'s, v8::Value> { // XHR defaults a missing or invalid response MIME type to text/xml. // Its XML MIME types include any +xml subtype, unlike DOMParser's enum. @@ -613,7 +460,7 @@ fn parse_xhr_response_document<'s>( if (!is_xml && !is_html) || (is_html && response_type != XmlHttpRequestResponseType::Document) { return v8::null(scope).into(); } - let Some(document) = build_xhr_response_document(scope, xhr, xhr_response_url(head).clone(), body_text, &mime) + let Some(document) = build_xhr_response_document(scope, xhr, xhr_response_url(head).clone(), body_text, &mime, character_set) else { return v8::null(scope).into(); }; @@ -639,6 +486,7 @@ fn build_xhr_response_document<'s>( response_url: url::Url, body_text: &str, mime: &str, + character_set: &str, ) -> Option> { let xhr = v8::Global::new(scope, xhr); let xhr = v8::Local::new(scope, xhr); @@ -662,7 +510,12 @@ fn build_xhr_response_document<'s>( origin_document_handle, )?; let document = if is_html_document_mime(mime) { - dom_parser::parse_detached_html_document_from_source(scope, response_url, body_text) + dom_parser::parse_detached_html_document_from_source_with_encoding( + scope, + response_url, + body_text, + Some(character_set), + ) } else { dom_parser::parse_detached_xml_document_from_source( scope, diff --git a/moli-renderer-v8/src/network_host/xhr/encoding.rs b/moli-renderer-v8/src/network_host/xhr/encoding.rs new file mode 100644 index 0000000000..ab48879c34 --- /dev/null +++ b/moli-renderer-v8/src/network_host/xhr/encoding.rs @@ -0,0 +1,37 @@ +use moli_encoding::{XhrResponseDecoder, XhrResponseTextKind, encoding_for_label}; +use moli_web_mime::{effective_response_mime_essence, mime_charset, response_content_type}; + +use super::*; + +pub(crate) fn xhr_response_text_decoder( + scope: &mut v8::PinScope<'_, '_>, + xhr: v8::Local<'_, v8::Object>, + headers: &[(String, String)], +) -> XhrResponseDecoder { + let override_mime = xhr_state_string_property(scope, xhr, XHR_OVERRIDE_MIME_TYPE_SLOT); + // A MIME-only override must retain the response charset. Blink also + // ignores an empty override charset, but an unknown nonempty label + // replaces the transport label and lets content/default detection run. + let charset = override_mime + .as_deref() + .and_then(mime_charset) + .filter(|label| !label.is_empty()) + .or_else(|| { + response_content_type(headers) + .as_deref() + .and_then(mime_charset) + }); + let mime = effective_response_mime_essence(headers, override_mime.as_deref()) + .unwrap_or_else(|| "text/xml".to_owned()); + let kind = match xhr_response_type(scope, xhr) { + XmlHttpRequestResponseType::Document if mime == "text/html" => XhrResponseTextKind::Html, + XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Document + if matches!(mime.as_str(), "text/xml" | "application/xml") + || mime.ends_with("+xml") => + { + XhrResponseTextKind::Xml + } + _ => XhrResponseTextKind::Text, + }; + XhrResponseDecoder::new(kind, charset.as_deref().and_then(encoding_for_label)) +} diff --git a/moli-renderer-v8/src/network_host/xhr/response_type.rs b/moli-renderer-v8/src/network_host/xhr/response_type.rs index e9de63563c..2b321f123d 100644 --- a/moli-renderer-v8/src/network_host/xhr/response_type.rs +++ b/moli-renderer-v8/src/network_host/xhr/response_type.rs @@ -1,3 +1,4 @@ +use super::{XHR_RESPONSE_TYPE_SLOT, xhr_state_string_property}; use std::str::FromStr; #[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::IntoStaticStr)] @@ -22,6 +23,16 @@ impl XmlHttpRequestResponseType { } } +pub(super) fn xhr_response_type( + scope: &mut v8::PinScope<'_, '_>, + xhr: v8::Local<'_, v8::Object>, +) -> XmlHttpRequestResponseType { + xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT) + .as_deref() + .and_then(XmlHttpRequestResponseType::parse) + .unwrap_or(XmlHttpRequestResponseType::Default) +} + #[cfg(test)] mod tests { use super::XmlHttpRequestResponseType; diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs index e95e485ea5..5b8c01a5c4 100644 --- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs +++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs @@ -1454,6 +1454,146 @@ fn xml_http_request_document_response_requires_an_eligible_mime_and_well_formed_ } } +#[test] +fn xhr_response_decoding_uses_headers_received_overrides_for_buffered_and_streamed_bytes() { + for streaming in [false, true] { + for (chunks, partial, expected) in [ + ( + vec![vec![0x83], vec![0x65, 0x83], vec![0x58, 0x83, 0x67]], + vec!["", "テ", "テスト"], + "テスト", + ), + ( + vec![vec![0xff], vec![0xfe, b'A'], vec![0, 0xe9], vec![0]], + vec!["", "", "A", "Aé"], + "Aé", + ), + ] { + let mut vm = new_storage_test_vm("https://xhr-response-decoding.test/"); + vm.set_fetch_subresource_interception( + true, + Some(crate::types::SubresourceResourceType::Xhr), + ); + vm.eval( + r#" + globalThis.__decodingXhr = new XMLHttpRequest(); + globalThis.__decodingHeadersText = null; + __decodingXhr.open('GET', '/response'); + __decodingXhr.onreadystatechange = () => { + if (__decodingXhr.readyState === 2) { + __decodingHeadersText = __decodingXhr.responseText; + __decodingXhr.overrideMimeType('text/plain;charset=Shift_JIS'); + } + }; + __decodingXhr.send(); + "#, + ) + .unwrap(); + let requests = vm.take_pending_subresource_fetch_infos(); + assert_eq!(requests.len(), 1); + let request = &requests[0]; + let bytes = chunks.concat(); + let head = moli_fetch::ResponseHead { + final_url: request.url.clone(), + status: 200, + status_text: None, + headers: vec![ + ( + "Content-Type".to_owned(), + "text/plain;charset=UTF-8".to_owned(), + ), + ("Content-Length".to_owned(), bytes.len().to_string()), + ], + request_cookie_report: None, + cookie_set_reports: Vec::new(), + redirected: false, + redirect_chain: Vec::new(), + from_cache: false, + negotiated_http_version: None, + }; + if streaming { + let body_source_id = crate::network_host::new_network_body_source_id(); + vm.start_streaming_async_subresource_fetch( + crate::types::AsyncSubresourceStreamingStarted { + internal_id: request.internal_id, + request_url: request.url.clone(), + request_method: "GET".to_owned(), + request_headers: Vec::new(), + request_body: None, + body_source_id, + head, + network_request_headers: None, + }, + ) + .unwrap(); + for (chunk, text) in chunks.into_iter().zip(partial) { + vm.append_streaming_async_subresource_fetch_chunk(body_source_id, chunk); + assert_eq!(vm.eval("__decodingXhr.responseText").unwrap(), text); + } + vm.finish_streaming_async_subresource_fetch( + request.internal_id, + body_source_id, + Ok(()), + ) + .unwrap(); + } else { + let context_ptr: *const v8::Global = &vm.page_default_context; + vm.renderer_document_isolate + .with_entered_renderer_document_isolate(move |isolate| { + let scope = std::pin::pin!(v8::HandleScope::new(isolate)); + let scope = &mut scope.init(); + let context = unsafe { v8::Local::new(scope, &*context_ptr) }; + let scope = &mut v8::ContextScope::new(scope, context); + let xhr = context + .global(scope) + .get(scope, v8str(scope, "__decodingXhr").into()) + .and_then(|value| v8::Local::::try_from(value).ok()) + .unwrap(); + crate::network_host::apply_xhr_response_body_source( + scope, + xhr, + head, + moli_fetch::ResponseBody::materialized_bytes(bytes), + ); + Ok(()) + }) + .unwrap(); + } + assert_eq!(vm.eval("__decodingHeadersText").unwrap(), ""); + assert_eq!(vm.eval("__decodingXhr.responseText").unwrap(), expected); + assert_eq!(vm.eval("__decodingXhr.response === __decodingXhr.responseText && __decodingXhr.readyState === 4").unwrap(), "true"); + } + } +} + +#[test] +fn xhr_queued_response_retains_legacy_bytes_and_decodes_json_as_utf8() { + let mut vm = new_storage_test_vm("https://xhr-queued-decoding.test/"); + vm.eval(r#" + globalThis.__queuedDecoding = []; + for (const [url, type, override] of [ + ['data:text/plain;charset=windows-1252,%FF', '', null], + ['data:text/plain;charset=utf-8,%83%65%83%58%83%67', '', 'text/plain;charset=Shift_JIS'], + ['data:application/json;charset=windows-1252,%EF%BB%BF%7B%22x%22%3A1%7D', 'json', null], + ['data:text/html,%3Cmeta%20charset=windows-1252%3E%3Cx%3E%FF%3C%2Fx%3E', 'document', null], + ]) { + const xhr = new XMLHttpRequest(); + xhr.open('GET', url); + xhr.responseType = type; + xhr.onreadystatechange = () => { + if (xhr.readyState === 2 && override) xhr.overrideMimeType(override); + }; + xhr.onload = () => __queuedDecoding.push(type === 'json' ? xhr.response.x : + type === 'document' ? [xhr.response.querySelector('x').textContent, xhr.response.characterSet] : xhr.responseText); + xhr.send(); + } + "#).unwrap(); + assert_eq!( + vm.eval("JSON.stringify(__queuedDecoding)").unwrap(), + r#"["ÿ","テスト",1,["ÿ","windows-1252"]]"# + ); +} + #[test] fn xml_http_request_queued_document_response_uses_parsed_overrides_and_xml_errors() { let mut vm = new_storage_test_vm("https://xhr-queued-document.test/"); diff --git a/moli-renderer-v8/src/types.rs b/moli-renderer-v8/src/types.rs index 3c35401feb..7bb3d15e4a 100644 --- a/moli-renderer-v8/src/types.rs +++ b/moli-renderer-v8/src/types.rs @@ -1029,7 +1029,7 @@ pub(super) struct EventSourceStreamingChunkDelivery<'s> { } pub(super) struct XhrStreamingResponseState { - pending_utf8_bytes: Vec, + decoder: Option, loaded: usize, total: Option, } @@ -1046,44 +1046,20 @@ impl XhrStreamingResponseState { .ok() }); Self { - pending_utf8_bytes: Vec::new(), + decoder: None, loaded: 0, total, } } - pub(super) fn append(&mut self, bytes: &[u8]) -> (String, usize, Option) { + pub(super) fn append( + &mut self, + bytes: &[u8], + make_decoder: impl FnOnce() -> moli_encoding::XhrResponseDecoder, + ) -> (String, usize, Option) { self.loaded = self.loaded.saturating_add(bytes.len()); - self.pending_utf8_bytes.extend_from_slice(bytes); - - let mut decoded = String::new(); - let mut consumed = 0; - while consumed < self.pending_utf8_bytes.len() { - let remaining = &self.pending_utf8_bytes[consumed..]; - match std::str::from_utf8(remaining) { - Ok(text) => { - decoded.push_str(text); - consumed = self.pending_utf8_bytes.len(); - } - Err(error) => { - let valid_end = consumed + error.valid_up_to(); - decoded.push_str( - std::str::from_utf8(&self.pending_utf8_bytes[consumed..valid_end]) - .expect("Utf8Error::valid_up_to must identify a valid UTF-8 prefix"), - ); - consumed = valid_end; - let Some(invalid_len) = error.error_len() else { - break; - }; - decoded.push('\u{fffd}'); - consumed += invalid_len; - } - } - } - if consumed > 0 { - self.pending_utf8_bytes.drain(..consumed); - } - (decoded, self.loaded, self.total) + let decoder = self.decoder.get_or_insert_with(make_decoder); + (decoder.push(bytes), self.loaded, self.total) } }