diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index c7ff1003a0..fd0e471aea 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -5,6 +5,138 @@ use encoding_rs::Encoding;
use super::*;
use moli_charset_parser::HTML_META_CHARSET_PRESCAN_LIMIT;
+#[test]
+fn xhr_response_decoder_preserves_encoding_across_every_chunk_boundary() {
+ use XhrResponseTextKind::{Html, Text, Xml};
+ let xml = b"\xff";
+ let xml_utf16 = "é";
+ let cases = [
+ (
+ Text,
+ None,
+ b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),
+ "機",
+ "UTF-8",
+ ),
+ (
+ Text,
+ Some(encoding_rs::WINDOWS_1252),
+ b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),
+ "機",
+ "UTF-8",
+ ),
+ (
+ Text,
+ None,
+ b"\xef\xbb\xbf\xef\xbb\xbf".to_vec(),
+ "\u{feff}",
+ "UTF-8",
+ ),
+ (Text, None, b"\xff\xfeA\0\xe9\0".to_vec(), "Aé", "UTF-16LE"),
+ (
+ Text,
+ Some(encoding_rs::UTF_8),
+ b"\xfe\xff\0A\0\xe9".to_vec(),
+ "Aé",
+ "UTF-16BE",
+ ),
+ (
+ Text,
+ Some(encoding_rs::UTF_16LE),
+ b"\xe9\0".to_vec(),
+ "é",
+ "UTF-16LE",
+ ),
+ (
+ Text,
+ Some(encoding_rs::SHIFT_JIS),
+ b"\x83e\x83X\x83g".to_vec(),
+ "テスト",
+ "Shift_JIS",
+ ),
+ (Text, None, b"a\xc2".to_vec(), "a\u{fffd}", "UTF-8"),
+ (
+ Xml,
+ None,
+ xml.to_vec(),
+ "ÿ",
+ "windows-1252",
+ ),
+ (
+ Xml,
+ Some(encoding_rs::UTF_8),
+ xml.to_vec(),
+ "\u{fffd}",
+ "UTF-8",
+ ),
+ (
+ Text,
+ None,
+ xml.to_vec(),
+ "\u{fffd}",
+ "UTF-8",
+ ),
+ (
+ Xml,
+ None,
+ xml_utf16
+ .encode_utf16()
+ .flat_map(u16::to_le_bytes)
+ .collect(),
+ xml_utf16,
+ "UTF-16LE",
+ ),
+ (
+ Html,
+ None,
+ b"\xff".to_vec(),
+ "ÿ",
+ "windows-1252",
+ ),
+ (
+ Html,
+ Some(encoding_rs::UTF_8),
+ b"\xff".to_vec(),
+ "\u{fffd}",
+ "UTF-8",
+ ),
+ (
+ Html,
+ None,
+ b"\xe6\xa9\x9f".to_vec(),
+ "機",
+ "UTF-8",
+ ),
+ ];
+ for (kind, transport, bytes, expected, encoding) in cases {
+ for boundary in 0..=bytes.len() {
+ let mut decoder = XhrResponseDecoder::new(kind, transport);
+ let mut text = decoder.push(&bytes[..boundary]);
+ text.push_str(&decoder.push(&bytes[boundary..]));
+ text.push_str(&decoder.finish());
+ assert_eq!(text, expected, "boundary={boundary}, bytes={bytes:?}");
+ assert_eq!(decoder.encoding_name(), encoding);
+ }
+ }
+}
+
+#[test]
+fn xhr_response_decoder_defers_partial_boms_and_flushes_incomplete_sequences() {
+ let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);
+ assert_eq!(decoder.push(b"\xef"), "");
+ assert_eq!(decoder.push(b"\xbb"), "");
+ assert_eq!(decoder.push(b"\xbf"), "");
+ assert_eq!(decoder.push(b"a\xc2"), "a");
+ assert_eq!(decoder.finish(), "\u{fffd}");
+
+ let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);
+ assert_eq!(decoder.push(b"\xff"), "");
+ assert_eq!(decoder.push(b"\xfeA"), "");
+ assert_eq!(decoder.push(b"\0\xe9"), "A");
+ assert_eq!(decoder.push(b"\0"), "é");
+ assert_eq!(decoder.finish(), "");
+}
+
fn gbk_bytes(input: &str) -> Vec {
encoding_rs::GBK.encode(input).0.into_owned()
}
diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs
index d99aeaec31..d4579d73ad 100644
--- a/moli-renderer-v8/src/dom_parser.rs
+++ b/moli-renderer-v8/src/dom_parser.rs
@@ -422,10 +422,26 @@ pub(crate) fn parse_detached_html_document_from_source<'s>(
scope: &mut v8::PinScope<'s, '_>,
document_url: Url,
source: &str,
+) -> Option> {
+ parse_detached_html_document_from_source_with_encoding(scope, document_url, source, None)
+}
+
+pub(crate) fn parse_detached_html_document_from_source_with_encoding<'s>(
+ scope: &mut v8::PinScope<'s, '_>,
+ document_url: Url,
+ source: &str,
+ character_set: Option<&str>,
) -> Option> {
let parsed = HtmlParser::with_scripting_enabled(false)
.parse_without_declarative_shadow_roots(document_url, source.to_owned());
- build_detached_document(scope, parsed, DetachedDocumentKind::Html, false)
+ build_detached_document_from_dom_host_with_content_type(
+ scope,
+ DomHost::from_dom(parsed),
+ DetachedDocumentKind::Html,
+ false,
+ Some("text/html"),
+ character_set,
+ )
}
pub(crate) fn parse_detached_html_document_from_source_with_declarative_shadow_roots<'s>(
@@ -513,21 +529,6 @@ fn child_document_url_is_xml_like(url: &Url) -> bool {
path.ends_with(".xml") || path.ends_with(".xhtml") || path.ends_with(".svg")
}
-fn build_detached_document<'s>(
- scope: &mut v8::PinScope<'s, '_>,
- parsed: NativeDom,
- kind: DetachedDocumentKind,
- expose_declarative_shadow_roots: bool,
-) -> Option> {
- build_detached_document_with_content_type(
- scope,
- parsed,
- kind,
- expose_declarative_shadow_roots,
- None,
- )
-}
-
fn build_detached_document_with_content_type<'s>(
scope: &mut v8::PinScope<'s, '_>,
parsed: NativeDom,
diff --git a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs
index e4eb1a2cc4..7d92c0859b 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/resource_loading.rs
@@ -1884,7 +1884,11 @@ impl JsContextHost {
let context = v8::Local::new(scope, state.pending.execution_context.context_global()?);
let xhr = v8::Local::new(scope, xhr);
state.body_writer.append(bytes);
- let (decoded_text, loaded, total) = state.xhr_response.as_mut()?.append(bytes);
+ let headers = &state.head.headers;
+ let (decoded_text, loaded, total) = state.xhr_response.as_mut()?.append(bytes, || {
+ let scope = &mut v8::ContextScope::new(scope, context);
+ crate::network_host::xhr_response_text_decoder(scope, xhr, headers)
+ });
crate::types::XhrStreamingChunkDelivery {
context,
xhr,
diff --git a/moli-renderer-v8/src/network_host.rs b/moli-renderer-v8/src/network_host.rs
index a8b51bc281..9afb4002a2 100644
--- a/moli-renderer-v8/src/network_host.rs
+++ b/moli-renderer-v8/src/network_host.rs
@@ -200,8 +200,9 @@ pub(crate) use self::xhr::{
install_window_xml_http_request_template_bindings, install_xml_http_request_bindings,
install_xml_http_request_event_target_bindings, progress_event_constructor_callback,
set_xhr_state_number, throw_synchronous_xhr_failure, xhr_author_request_headers,
- xhr_constructor_callback, xhr_ensure_send_allowed, xhr_state_bool_property,
- xhr_state_number_property, xhr_state_string_property,
+ xhr_constructor_callback, xhr_dispatch_progress_event, xhr_ensure_send_allowed,
+ xhr_response_text_decoder,
+ xhr_state_bool_property, xhr_state_number_property, xhr_state_string_property,
};
pub(in crate::network_host) const NETWORK_BODY_SLOT: &str = "__lmBody";
pub(in crate::network_host) const NETWORK_BODY_BYTES_SLOT: &str = "__lmBodyBytes";
diff --git a/moli-renderer-v8/src/network_host/xhr.rs b/moli-renderer-v8/src/network_host/xhr.rs
index c4e47614f5..177ecedb24 100644
--- a/moli-renderer-v8/src/network_host/xhr.rs
+++ b/moli-renderer-v8/src/network_host/xhr.rs
@@ -1,5 +1,6 @@
mod bindings;
mod delivery;
+mod encoding;
mod events;
mod header_surface;
mod instance_state;
@@ -58,6 +59,8 @@ pub(crate) use self::delivery::{
apply_xhr_streaming_response_chunk, apply_xhr_streaming_response_head, apply_xhr_timeout,
finish_xhr_abort, throw_synchronous_xhr_failure,
};
+pub(crate) use self::encoding::xhr_response_text_decoder;
+pub(crate) use self::events::xhr_dispatch_progress_event;
pub(crate) use self::instance_state::xhr_has_upload_listeners;
pub(crate) use self::instance_state::{
XHR_METHOD_SLOT, XHR_OPEN_GENERATION_SLOT, XHR_REQUEST_HEADERS_SLOT, XHR_RESPONSE_URL_SLOT,
@@ -77,7 +80,7 @@ pub(crate) use self::request_state::begin_xhr_send;
use self::request_state::{
XhrReadyState, clear_xhr_active_request, set_xhr_ready_state, xhr_ready_state,
};
-use self::response_type::XmlHttpRequestResponseType;
+use self::response_type::{XmlHttpRequestResponseType, xhr_response_type};
#[cfg(test)]
pub(crate) use self::send::prepare_xhr_send_body;
pub(crate) use self::send::{
diff --git a/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs b/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs
index c40e03f2e9..1a680da356 100644
--- a/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs
+++ b/moli-renderer-v8/src/network_host/xhr/delivery/pending.rs
@@ -11,8 +11,6 @@ const RESPONSE_STATUS_TEXT: &str = "statusText";
const RESPONSE_URL: &str = "url";
const RESPONSE_HEADERS: &str = "headers";
const RESPONSE_BODY: &str = "body";
-const RESPONSE_BODY_TEXT: &str = "bodyText";
-const RESPONSE_BODY_LENGTH: &str = "bodyLength";
/// Each queued job owns its complete outcome. A later open()/send() cannot
/// overwrite a shared pending payload or let an old job consume the new one.
@@ -45,10 +43,6 @@ struct XhrPendingResponse<'s> {
headers: String,
#[webapi(slot = RESPONSE_BODY)]
body: Option>,
- #[webapi(slot = RESPONSE_BODY_TEXT)]
- body_text: String,
- #[webapi(slot = RESPONSE_BODY_LENGTH)]
- body_length: usize,
}
#[derive(WebApiObject)]
@@ -70,38 +64,16 @@ pub(in crate::network_host::xhr) fn queue_xhr_response_delivery<'s>(
response: Response,
) {
let (head, body) = response.into_body();
- let response_type = xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT)
- .as_deref()
- .and_then(XmlHttpRequestResponseType::parse)
- .unwrap_or(XmlHttpRequestResponseType::Default);
- let (body, body_text, body_length) = match response_type {
- XmlHttpRequestResponseType::ArrayBuffer | XmlHttpRequestResponseType::Blob => {
- let bytes = body
- .try_into_materialized_bytes()
- .expect("XHR response body should remain materialized at the delivery boundary");
- let length = bytes.len();
- let body =
- crate::network_host::network_body_source_object_from_bytes(scope, None, bytes);
- (Some(body.into()), String::new(), length)
- }
- XmlHttpRequestResponseType::Json
- | XmlHttpRequestResponseType::Document
- | XmlHttpRequestResponseType::Default
- | XmlHttpRequestResponseType::Text => {
- let (text, bytes) = body
- .try_into_lossy_materialized_text()
- .expect("XHR response body should remain materialized at the delivery boundary");
- (None, text, bytes.len())
- }
- };
+ let bytes = body
+ .try_into_materialized_bytes()
+ .expect("XHR response body should remain materialized at the delivery boundary");
+ let body = crate::network_host::network_body_source_object_from_bytes(scope, None, bytes);
let response = XhrPendingResponse::new(
head.status,
head.status_text().to_owned(),
head.final_url.to_string(),
serde_json::to_string(&head.headers).unwrap_or_else(|_| "[]".to_owned()),
- body,
- body_text,
- body_length,
+ Some(body.into()),
);
queue_xhr_delivery(scope, xhr, XhrPendingDelivery::Response(response));
}
@@ -171,10 +143,6 @@ fn xhr_complete_callback<'s>(
.unwrap_or_default();
let body = get_private_value(scope, response, RESPONSE_BODY)
.filter(|value| !value.is_null_or_undefined());
- let body_text = response_string(scope, response, RESPONSE_BODY_TEXT);
- let body_length = get_private_value(scope, response, RESPONSE_BODY_LENGTH)
- .and_then(|value| value.number_value(scope))
- .expect("queued XHR body length") as usize;
apply_xhr_response_pending_body(
scope,
xhr,
@@ -192,8 +160,6 @@ fn xhr_complete_callback<'s>(
negotiated_http_version: None,
},
body,
- body_text,
- Some(body_length),
);
}
diff --git a/moli-renderer-v8/src/network_host/xhr/delivery/response.rs b/moli-renderer-v8/src/network_host/xhr/delivery/response.rs
index 3e470ecd85..06c111bca4 100644
--- a/moli-renderer-v8/src/network_host/xhr/delivery/response.rs
+++ b/moli-renderer-v8/src/network_host/xhr/delivery/response.rs
@@ -70,35 +70,12 @@ fn apply_xhr_response_body_source_with_mode(
status_text: Option<&str>,
mode: XhrResponseDeliveryMode,
) {
- // XHR's Web-visible surface is still fully materialized. Branch by the
- // requested responseType so byte consumers do not pay for an inaccessible
- // lossy text view.
- match xhr_response_type(scope, xhr) {
- XmlHttpRequestResponseType::Json
- | XmlHttpRequestResponseType::ArrayBuffer
- | XmlHttpRequestResponseType::Blob => {
- let body_bytes = body
- .try_into_materialized_bytes()
- .expect("XHR response body should remain materialized at the V8 boundary");
- apply_xhr_response_bytes(scope, xhr, head, body_bytes, status_text, mode);
- }
- XmlHttpRequestResponseType::Document
- | XmlHttpRequestResponseType::Default
- | XmlHttpRequestResponseType::Text => {
- let (body_text, body_bytes) = body
- .try_into_lossy_materialized_text()
- .expect("XHR response body should remain materialized at the V8 boundary");
- apply_xhr_response_body(
- scope,
- xhr,
- head,
- Some(body_text),
- body_bytes,
- status_text,
- mode,
- );
- }
- }
+ // Choose the response decoder only after HEADERS_RECEIVED handlers have
+ // had a chance to override the MIME type or change responseType.
+ let bytes = body
+ .try_into_materialized_bytes()
+ .expect("XHR response body should remain materialized at the V8 boundary");
+ apply_xhr_response_body(scope, xhr, head, bytes, status_text, mode);
}
#[derive(Clone, Copy)]
@@ -119,50 +96,8 @@ pub(in crate::network_host::xhr) fn apply_xhr_response_pending_body<'s>(
xhr: v8::Local<'s, v8::Object>,
head: moli_fetch::ResponseHead,
body_value: Option>,
- fallback_body_text: String,
- fallback_body_len: Option,
) {
- let response_type = xhr_response_type(scope, xhr);
- if response_type == XmlHttpRequestResponseType::ArrayBuffer
- && let Some(body_value) = body_value
- && let Some(loaded) = buffer_value_byte_length(body_value)
- {
- apply_xhr_response_prebuilt_value(
- scope,
- xhr,
- head,
- response_type,
- body_value,
- loaded,
- None,
- XhrResponseDeliveryMode::Buffered,
- );
- return;
- }
-
- if body_value.is_none()
- && matches!(
- response_type,
- XmlHttpRequestResponseType::Json
- | XmlHttpRequestResponseType::Document
- | XmlHttpRequestResponseType::Default
- | XmlHttpRequestResponseType::Text
- )
- {
- let loaded = fallback_body_len.unwrap_or(fallback_body_text.len());
- apply_xhr_response_text(
- scope,
- xhr,
- head,
- fallback_body_text,
- loaded,
- None,
- XhrResponseDeliveryMode::Buffered,
- );
- return;
- }
-
- let body_bytes = body_value
+ let bytes = body_value
.and_then(|value| {
v8::Local::::try_from(value)
.ok()
@@ -171,33 +106,21 @@ pub(in crate::network_host::xhr) fn apply_xhr_response_pending_body<'s>(
})
.or_else(|| blob::buffer_source_bytes_from_value(scope, value))
})
- .unwrap_or_else(|| fallback_body_text.into_bytes());
- apply_xhr_response_bytes(
+ .unwrap_or_default();
+ apply_xhr_response_body(
scope,
xhr,
head,
- body_bytes,
+ bytes,
None,
XhrResponseDeliveryMode::Buffered,
);
}
-fn apply_xhr_response_bytes(
- scope: &mut v8::PinScope<'_, '_>,
- xhr: v8::Local<'_, v8::Object>,
- head: moli_fetch::ResponseHead,
- body_bytes: Vec,
- status_text: Option<&str>,
- mode: XhrResponseDeliveryMode,
-) {
- apply_xhr_response_body(scope, xhr, head, None, body_bytes, status_text, mode);
-}
-
fn apply_xhr_response_body(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
head: moli_fetch::ResponseHead,
- body_text: Option,
body_bytes: Vec,
status_text: Option<&str>,
mode: XhrResponseDeliveryMode,
@@ -205,15 +128,18 @@ fn apply_xhr_response_body(
let Some(response_type) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else {
return;
};
- let body_text = match response_type {
+ let (body_text, character_set) = match response_type {
XmlHttpRequestResponseType::Document
| XmlHttpRequestResponseType::Default
| XmlHttpRequestResponseType::Text => {
- Some(body_text.unwrap_or_else(|| String::from_utf8_lossy(&body_bytes).into_owned()))
+ let mut decoder = xhr_response_text_decoder(scope, xhr, &head.headers);
+ let mut text = decoder.push(&body_bytes);
+ text.push_str(&decoder.finish());
+ (Some(text), decoder.encoding_name())
}
XmlHttpRequestResponseType::Json
| XmlHttpRequestResponseType::ArrayBuffer
- | XmlHttpRequestResponseType::Blob => None,
+ | XmlHttpRequestResponseType::Blob => (None, "UTF-8"),
};
let loaded = body_bytes.len() as f64;
let progress = xhr_response_progress(&head, loaded);
@@ -226,6 +152,7 @@ fn apply_xhr_response_body(
&head,
body_text.as_deref().unwrap_or(""),
response_type,
+ character_set,
);
set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document);
document
@@ -247,6 +174,7 @@ fn apply_xhr_response_body(
&head,
body_text.as_deref().unwrap_or(""),
response_type,
+ character_set,
);
set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document);
}
@@ -266,52 +194,6 @@ fn apply_xhr_response_body(
);
}
-fn apply_xhr_response_text(
- scope: &mut v8::PinScope<'_, '_>,
- xhr: v8::Local<'_, v8::Object>,
- head: moli_fetch::ResponseHead,
- body_text: String,
- loaded: usize,
- status_text: Option<&str>,
- mode: XhrResponseDeliveryMode,
-) {
- let Some(response_type) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else {
- return;
- };
- let progress = xhr_response_progress(&head, loaded as f64);
- let response_val: v8::Local<'_, v8::Value> = match response_type {
- XmlHttpRequestResponseType::Json => parse_xhr_json_response(scope, body_text.as_bytes()),
- XmlHttpRequestResponseType::Document => {
- let document =
- parse_xhr_response_document(scope, xhr, &head, &body_text, response_type);
- set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document);
- document
- }
- XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Text => {
- if response_type == XmlHttpRequestResponseType::Default {
- let document =
- parse_xhr_response_document(scope, xhr, &head, &body_text, response_type);
- set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document);
- }
- v8_string(scope, &body_text)
- .map(|s| s.into())
- .unwrap_or_else(|| v8::undefined(scope).into())
- }
- XmlHttpRequestResponseType::ArrayBuffer | XmlHttpRequestResponseType::Blob => {
- v8::undefined(scope).into()
- }
- };
- finish_xhr_response(
- scope,
- xhr,
- response_type,
- response_val,
- &body_text,
- progress,
- mode,
- );
-}
-
fn parse_xhr_json_response<'s>(
scope: &mut v8::PinScope<'s, '_>,
bytes: &[u8],
@@ -325,23 +207,6 @@ fn parse_xhr_json_response<'s>(
.unwrap_or_else(|| v8::null(scope).into())
}
-fn apply_xhr_response_prebuilt_value(
- scope: &mut v8::PinScope<'_, '_>,
- xhr: v8::Local<'_, v8::Object>,
- head: moli_fetch::ResponseHead,
- response_type: XmlHttpRequestResponseType,
- response_val: v8::Local<'_, v8::Value>,
- loaded: usize,
- status_text: Option<&str>,
- mode: XhrResponseDeliveryMode,
-) {
- let Some(_) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else {
- return;
- };
- let progress = xhr_response_progress(&head, loaded as f64);
- finish_xhr_response(scope, xhr, response_type, response_val, "", progress, mode);
-}
-
pub(crate) fn apply_xhr_streaming_response_head(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
@@ -411,6 +276,16 @@ fn prepare_xhr_response(
}
if matches!(mode, XhrResponseDeliveryMode::Buffered) {
set_xhr_response_head(scope, xhr, head, status_text);
+ if xhr_is_async(scope, xhr) {
+ let generation = xhr_state_number_property(scope, xhr, XHR_OPEN_GENERATION_SLOT);
+ super::super::events::xhr_fire_readystatechange(scope, xhr, XhrReadyState::HeadersReceived);
+ if scope.is_execution_terminating()
+ || xhr_is_aborted(scope, xhr)
+ || xhr_state_number_property(scope, xhr, XHR_OPEN_GENERATION_SLOT) != generation
+ {
+ return None;
+ }
+ }
}
Some(xhr_response_type(scope, xhr))
}
@@ -457,15 +332,6 @@ fn finish_xhr_response(
let dispatch_intermediate_events =
matches!(mode, XhrResponseDeliveryMode::Buffered) && xhr_is_async(scope, xhr);
- if dispatch_intermediate_events {
- super::super::events::xhr_fire_readystatechange(scope, xhr, XhrReadyState::HeadersReceived);
- if scope.is_execution_terminating() {
- return;
- }
- if xhr_is_aborted(scope, xhr) {
- return;
- }
- }
set_xhr_state_string(scope, xhr, XHR_RESPONSE_TEXT_SLOT, response_text);
set_xhr_state_value(scope, xhr, XHR_RESPONSE_SLOT, response_val);
if dispatch_intermediate_events {
@@ -561,26 +427,6 @@ fn identity_encoded_content_length(headers: &[(String, Vec)]) -> Option
.then_some(first)
}
-fn xhr_response_type(
- scope: &mut v8::PinScope<'_, '_>,
- xhr: v8::Local<'_, v8::Object>,
-) -> XmlHttpRequestResponseType {
- xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT)
- .as_deref()
- .and_then(XmlHttpRequestResponseType::parse)
- .unwrap_or(XmlHttpRequestResponseType::Default)
-}
-
-fn buffer_value_byte_length(value: v8::Local<'_, v8::Value>) -> Option {
- if let Ok(buffer) = v8::Local::::try_from(value) {
- return Some(buffer.byte_length());
- }
- if let Ok(view) = v8::Local::::try_from(value) {
- return Some(view.byte_length());
- }
- None
-}
-
fn xhr_response_mime_essence(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
@@ -603,6 +449,7 @@ fn parse_xhr_response_document<'s>(
head: &moli_fetch::ResponseHead,
body_text: &str,
response_type: XmlHttpRequestResponseType,
+ character_set: &str,
) -> v8::Local<'s, v8::Value> {
// XHR defaults a missing or invalid response MIME type to text/xml.
// Its XML MIME types include any +xml subtype, unlike DOMParser's enum.
@@ -613,7 +460,7 @@ fn parse_xhr_response_document<'s>(
if (!is_xml && !is_html) || (is_html && response_type != XmlHttpRequestResponseType::Document) {
return v8::null(scope).into();
}
- let Some(document) = build_xhr_response_document(scope, xhr, xhr_response_url(head).clone(), body_text, &mime)
+ let Some(document) = build_xhr_response_document(scope, xhr, xhr_response_url(head).clone(), body_text, &mime, character_set)
else {
return v8::null(scope).into();
};
@@ -639,6 +486,7 @@ fn build_xhr_response_document<'s>(
response_url: url::Url,
body_text: &str,
mime: &str,
+ character_set: &str,
) -> Option> {
let xhr = v8::Global::new(scope, xhr);
let xhr = v8::Local::new(scope, xhr);
@@ -662,7 +510,12 @@ fn build_xhr_response_document<'s>(
origin_document_handle,
)?;
let document = if is_html_document_mime(mime) {
- dom_parser::parse_detached_html_document_from_source(scope, response_url, body_text)
+ dom_parser::parse_detached_html_document_from_source_with_encoding(
+ scope,
+ response_url,
+ body_text,
+ Some(character_set),
+ )
} else {
dom_parser::parse_detached_xml_document_from_source(
scope,
diff --git a/moli-renderer-v8/src/network_host/xhr/encoding.rs b/moli-renderer-v8/src/network_host/xhr/encoding.rs
new file mode 100644
index 0000000000..ab48879c34
--- /dev/null
+++ b/moli-renderer-v8/src/network_host/xhr/encoding.rs
@@ -0,0 +1,37 @@
+use moli_encoding::{XhrResponseDecoder, XhrResponseTextKind, encoding_for_label};
+use moli_web_mime::{effective_response_mime_essence, mime_charset, response_content_type};
+
+use super::*;
+
+pub(crate) fn xhr_response_text_decoder(
+ scope: &mut v8::PinScope<'_, '_>,
+ xhr: v8::Local<'_, v8::Object>,
+ headers: &[(String, String)],
+) -> XhrResponseDecoder {
+ let override_mime = xhr_state_string_property(scope, xhr, XHR_OVERRIDE_MIME_TYPE_SLOT);
+ // A MIME-only override must retain the response charset. Blink also
+ // ignores an empty override charset, but an unknown nonempty label
+ // replaces the transport label and lets content/default detection run.
+ let charset = override_mime
+ .as_deref()
+ .and_then(mime_charset)
+ .filter(|label| !label.is_empty())
+ .or_else(|| {
+ response_content_type(headers)
+ .as_deref()
+ .and_then(mime_charset)
+ });
+ let mime = effective_response_mime_essence(headers, override_mime.as_deref())
+ .unwrap_or_else(|| "text/xml".to_owned());
+ let kind = match xhr_response_type(scope, xhr) {
+ XmlHttpRequestResponseType::Document if mime == "text/html" => XhrResponseTextKind::Html,
+ XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Document
+ if matches!(mime.as_str(), "text/xml" | "application/xml")
+ || mime.ends_with("+xml") =>
+ {
+ XhrResponseTextKind::Xml
+ }
+ _ => XhrResponseTextKind::Text,
+ };
+ XhrResponseDecoder::new(kind, charset.as_deref().and_then(encoding_for_label))
+}
diff --git a/moli-renderer-v8/src/network_host/xhr/response_type.rs b/moli-renderer-v8/src/network_host/xhr/response_type.rs
index e9de63563c..2b321f123d 100644
--- a/moli-renderer-v8/src/network_host/xhr/response_type.rs
+++ b/moli-renderer-v8/src/network_host/xhr/response_type.rs
@@ -1,3 +1,4 @@
+use super::{XHR_RESPONSE_TYPE_SLOT, xhr_state_string_property};
use std::str::FromStr;
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::IntoStaticStr)]
@@ -22,6 +23,16 @@ impl XmlHttpRequestResponseType {
}
}
+pub(super) fn xhr_response_type(
+ scope: &mut v8::PinScope<'_, '_>,
+ xhr: v8::Local<'_, v8::Object>,
+) -> XmlHttpRequestResponseType {
+ xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT)
+ .as_deref()
+ .and_then(XmlHttpRequestResponseType::parse)
+ .unwrap_or(XmlHttpRequestResponseType::Default)
+}
+
#[cfg(test)]
mod tests {
use super::XmlHttpRequestResponseType;
diff --git a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs
index e95e485ea5..5b8c01a5c4 100644
--- a/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs
+++ b/moli-renderer-v8/src/script_vm/tests/dom_xhr/xhr.rs
@@ -1454,6 +1454,146 @@ fn xml_http_request_document_response_requires_an_eligible_mime_and_well_formed_
}
}
+#[test]
+fn xhr_response_decoding_uses_headers_received_overrides_for_buffered_and_streamed_bytes() {
+ for streaming in [false, true] {
+ for (chunks, partial, expected) in [
+ (
+ vec![vec![0x83], vec![0x65, 0x83], vec![0x58, 0x83, 0x67]],
+ vec!["", "テ", "テスト"],
+ "テスト",
+ ),
+ (
+ vec![vec![0xff], vec![0xfe, b'A'], vec![0, 0xe9], vec![0]],
+ vec!["", "", "A", "Aé"],
+ "Aé",
+ ),
+ ] {
+ let mut vm = new_storage_test_vm("https://xhr-response-decoding.test/");
+ vm.set_fetch_subresource_interception(
+ true,
+ Some(crate::types::SubresourceResourceType::Xhr),
+ );
+ vm.eval(
+ r#"
+ globalThis.__decodingXhr = new XMLHttpRequest();
+ globalThis.__decodingHeadersText = null;
+ __decodingXhr.open('GET', '/response');
+ __decodingXhr.onreadystatechange = () => {
+ if (__decodingXhr.readyState === 2) {
+ __decodingHeadersText = __decodingXhr.responseText;
+ __decodingXhr.overrideMimeType('text/plain;charset=Shift_JIS');
+ }
+ };
+ __decodingXhr.send();
+ "#,
+ )
+ .unwrap();
+ let requests = vm.take_pending_subresource_fetch_infos();
+ assert_eq!(requests.len(), 1);
+ let request = &requests[0];
+ let bytes = chunks.concat();
+ let head = moli_fetch::ResponseHead {
+ final_url: request.url.clone(),
+ status: 200,
+ status_text: None,
+ headers: vec![
+ (
+ "Content-Type".to_owned(),
+ "text/plain;charset=UTF-8".to_owned(),
+ ),
+ ("Content-Length".to_owned(), bytes.len().to_string()),
+ ],
+ request_cookie_report: None,
+ cookie_set_reports: Vec::new(),
+ redirected: false,
+ redirect_chain: Vec::new(),
+ from_cache: false,
+ negotiated_http_version: None,
+ };
+ if streaming {
+ let body_source_id = crate::network_host::new_network_body_source_id();
+ vm.start_streaming_async_subresource_fetch(
+ crate::types::AsyncSubresourceStreamingStarted {
+ internal_id: request.internal_id,
+ request_url: request.url.clone(),
+ request_method: "GET".to_owned(),
+ request_headers: Vec::new(),
+ request_body: None,
+ body_source_id,
+ head,
+ network_request_headers: None,
+ },
+ )
+ .unwrap();
+ for (chunk, text) in chunks.into_iter().zip(partial) {
+ vm.append_streaming_async_subresource_fetch_chunk(body_source_id, chunk);
+ assert_eq!(vm.eval("__decodingXhr.responseText").unwrap(), text);
+ }
+ vm.finish_streaming_async_subresource_fetch(
+ request.internal_id,
+ body_source_id,
+ Ok(()),
+ )
+ .unwrap();
+ } else {
+ let context_ptr: *const v8::Global = &vm.page_default_context;
+ vm.renderer_document_isolate
+ .with_entered_renderer_document_isolate(move |isolate| {
+ let scope = std::pin::pin!(v8::HandleScope::new(isolate));
+ let scope = &mut scope.init();
+ let context = unsafe { v8::Local::new(scope, &*context_ptr) };
+ let scope = &mut v8::ContextScope::new(scope, context);
+ let xhr = context
+ .global(scope)
+ .get(scope, v8str(scope, "__decodingXhr").into())
+ .and_then(|value| v8::Local::::try_from(value).ok())
+ .unwrap();
+ crate::network_host::apply_xhr_response_body_source(
+ scope,
+ xhr,
+ head,
+ moli_fetch::ResponseBody::materialized_bytes(bytes),
+ );
+ Ok(())
+ })
+ .unwrap();
+ }
+ assert_eq!(vm.eval("__decodingHeadersText").unwrap(), "");
+ assert_eq!(vm.eval("__decodingXhr.responseText").unwrap(), expected);
+ assert_eq!(vm.eval("__decodingXhr.response === __decodingXhr.responseText && __decodingXhr.readyState === 4").unwrap(), "true");
+ }
+ }
+}
+
+#[test]
+fn xhr_queued_response_retains_legacy_bytes_and_decodes_json_as_utf8() {
+ let mut vm = new_storage_test_vm("https://xhr-queued-decoding.test/");
+ vm.eval(r#"
+ globalThis.__queuedDecoding = [];
+ for (const [url, type, override] of [
+ ['data:text/plain;charset=windows-1252,%FF', '', null],
+ ['data:text/plain;charset=utf-8,%83%65%83%58%83%67', '', 'text/plain;charset=Shift_JIS'],
+ ['data:application/json;charset=windows-1252,%EF%BB%BF%7B%22x%22%3A1%7D', 'json', null],
+ ['data:text/html,%3Cmeta%20charset=windows-1252%3E%3Cx%3E%FF%3C%2Fx%3E', 'document', null],
+ ]) {
+ const xhr = new XMLHttpRequest();
+ xhr.open('GET', url);
+ xhr.responseType = type;
+ xhr.onreadystatechange = () => {
+ if (xhr.readyState === 2 && override) xhr.overrideMimeType(override);
+ };
+ xhr.onload = () => __queuedDecoding.push(type === 'json' ? xhr.response.x :
+ type === 'document' ? [xhr.response.querySelector('x').textContent, xhr.response.characterSet] : xhr.responseText);
+ xhr.send();
+ }
+ "#).unwrap();
+ assert_eq!(
+ vm.eval("JSON.stringify(__queuedDecoding)").unwrap(),
+ r#"["ÿ","テスト",1,["ÿ","windows-1252"]]"#
+ );
+}
+
#[test]
fn xml_http_request_queued_document_response_uses_parsed_overrides_and_xml_errors() {
let mut vm = new_storage_test_vm("https://xhr-queued-document.test/");
diff --git a/moli-renderer-v8/src/types.rs b/moli-renderer-v8/src/types.rs
index 3c35401feb..7bb3d15e4a 100644
--- a/moli-renderer-v8/src/types.rs
+++ b/moli-renderer-v8/src/types.rs
@@ -1029,7 +1029,7 @@ pub(super) struct EventSourceStreamingChunkDelivery<'s> {
}
pub(super) struct XhrStreamingResponseState {
- pending_utf8_bytes: Vec,
+ decoder: Option,
loaded: usize,
total: Option,
}
@@ -1046,44 +1046,20 @@ impl XhrStreamingResponseState {
.ok()
});
Self {
- pending_utf8_bytes: Vec::new(),
+ decoder: None,
loaded: 0,
total,
}
}
- pub(super) fn append(&mut self, bytes: &[u8]) -> (String, usize, Option) {
+ pub(super) fn append(
+ &mut self,
+ bytes: &[u8],
+ make_decoder: impl FnOnce() -> moli_encoding::XhrResponseDecoder,
+ ) -> (String, usize, Option) {
self.loaded = self.loaded.saturating_add(bytes.len());
- self.pending_utf8_bytes.extend_from_slice(bytes);
-
- let mut decoded = String::new();
- let mut consumed = 0;
- while consumed < self.pending_utf8_bytes.len() {
- let remaining = &self.pending_utf8_bytes[consumed..];
- match std::str::from_utf8(remaining) {
- Ok(text) => {
- decoded.push_str(text);
- consumed = self.pending_utf8_bytes.len();
- }
- Err(error) => {
- let valid_end = consumed + error.valid_up_to();
- decoded.push_str(
- std::str::from_utf8(&self.pending_utf8_bytes[consumed..valid_end])
- .expect("Utf8Error::valid_up_to must identify a valid UTF-8 prefix"),
- );
- consumed = valid_end;
- let Some(invalid_len) = error.error_len() else {
- break;
- };
- decoded.push('\u{fffd}');
- consumed += invalid_len;
- }
- }
- }
- if consumed > 0 {
- self.pending_utf8_bytes.drain(..consumed);
- }
- (decoded, self.loaded, self.total)
+ let decoder = self.decoder.get_or_insert_with(make_decoder);
+ (decoder.push(bytes), self.loaded, self.total)
}
}