fix(xhr): decode response bytes using the response encoding

Honor response and override charsets, BOMs, and XML or HTML encoding
declarations across buffered, queued, and streaming delivery. Reuse the
HTML document decoder with UTF-8 fallback and expose its selected charset.
Keep JSON decoding fixed to UTF-8 and binary responses byte-exact.

Select the decoder after HEADERS_RECEIVED handlers can change the MIME
override. Keep queued response bytes intact and handle split BOMs and
multibyte characters in streaming text.
This commit is contained in:
ldm0
2026-11-02 06:02:17 +08:00
parent b83835e507
commit f9231bead7
11 changed files with 400 additions and 276 deletions
+132
View File
@@ -5,6 +5,138 @@ use encoding_rs::Encoding;
use super::*;
use moli_charset_parser::HTML_META_CHARSET_PRESCAN_LIMIT;
#[test]
fn xhr_response_decoder_preserves_encoding_across_every_chunk_boundary() {
use XhrResponseTextKind::{Html, Text, Xml};
let xml = b"<?xml version='1.0' encoding='windows-1252'?><x>\xff</x>";
let xml_utf16 = "<?xml version='1.0' encoding='UTF-16'?><x>é</x>";
let cases = [
(
Text,
None,
b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),
"機",
"UTF-8",
),
(
Text,
Some(encoding_rs::WINDOWS_1252),
b"\xef\xbb\xbf\xe6\xa9\x9f".to_vec(),
"機",
"UTF-8",
),
(
Text,
None,
b"\xef\xbb\xbf\xef\xbb\xbf".to_vec(),
"\u{feff}",
"UTF-8",
),
(Text, None, b"\xff\xfeA\0\xe9\0".to_vec(), "Aé", "UTF-16LE"),
(
Text,
Some(encoding_rs::UTF_8),
b"\xfe\xff\0A\0\xe9".to_vec(),
"Aé",
"UTF-16BE",
),
(
Text,
Some(encoding_rs::UTF_16LE),
b"\xe9\0".to_vec(),
"é",
"UTF-16LE",
),
(
Text,
Some(encoding_rs::SHIFT_JIS),
b"\x83e\x83X\x83g".to_vec(),
"テスト",
"Shift_JIS",
),
(Text, None, b"a\xc2".to_vec(), "a\u{fffd}", "UTF-8"),
(
Xml,
None,
xml.to_vec(),
"<?xml version='1.0' encoding='windows-1252'?><x>ÿ</x>",
"windows-1252",
),
(
Xml,
Some(encoding_rs::UTF_8),
xml.to_vec(),
"<?xml version='1.0' encoding='windows-1252'?><x>\u{fffd}</x>",
"UTF-8",
),
(
Text,
None,
xml.to_vec(),
"<?xml version='1.0' encoding='windows-1252'?><x>\u{fffd}</x>",
"UTF-8",
),
(
Xml,
None,
xml_utf16
.encode_utf16()
.flat_map(u16::to_le_bytes)
.collect(),
xml_utf16,
"UTF-16LE",
),
(
Html,
None,
b"<meta charset=windows-1252><x>\xff</x>".to_vec(),
"<meta charset=windows-1252><x>ÿ</x>",
"windows-1252",
),
(
Html,
Some(encoding_rs::UTF_8),
b"<meta charset=windows-1252><x>\xff</x>".to_vec(),
"<meta charset=windows-1252><x>\u{fffd}</x>",
"UTF-8",
),
(
Html,
None,
b"<x>\xe6\xa9\x9f</x>".to_vec(),
"<x>機</x>",
"UTF-8",
),
];
for (kind, transport, bytes, expected, encoding) in cases {
for boundary in 0..=bytes.len() {
let mut decoder = XhrResponseDecoder::new(kind, transport);
let mut text = decoder.push(&bytes[..boundary]);
text.push_str(&decoder.push(&bytes[boundary..]));
text.push_str(&decoder.finish());
assert_eq!(text, expected, "boundary={boundary}, bytes={bytes:?}");
assert_eq!(decoder.encoding_name(), encoding);
}
}
}
#[test]
fn xhr_response_decoder_defers_partial_boms_and_flushes_incomplete_sequences() {
let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);
assert_eq!(decoder.push(b"\xef"), "");
assert_eq!(decoder.push(b"\xbb"), "");
assert_eq!(decoder.push(b"\xbf"), "");
assert_eq!(decoder.push(b"a\xc2"), "a");
assert_eq!(decoder.finish(), "\u{fffd}");
let mut decoder = XhrResponseDecoder::new(XhrResponseTextKind::Text, None);
assert_eq!(decoder.push(b"\xff"), "");
assert_eq!(decoder.push(b"\xfeA"), "");
assert_eq!(decoder.push(b"\0\xe9"), "A");
assert_eq!(decoder.push(b"\0"), "é");
assert_eq!(decoder.finish(), "");
}
fn gbk_bytes(input: &str) -> Vec<u8> {
encoding_rs::GBK.encode(input).0.into_owned()
}
+17 -16
View File
@@ -422,10 +422,26 @@ pub(crate) fn parse_detached_html_document_from_source<'s>(
scope: &mut v8::PinScope<'s, '_>,
document_url: Url,
source: &str,
) -> Option<v8::Local<'s, v8::Object>> {
parse_detached_html_document_from_source_with_encoding(scope, document_url, source, None)
}
pub(crate) fn parse_detached_html_document_from_source_with_encoding<'s>(
scope: &mut v8::PinScope<'s, '_>,
document_url: Url,
source: &str,
character_set: Option<&str>,
) -> Option<v8::Local<'s, v8::Object>> {
let parsed = HtmlParser::with_scripting_enabled(false)
.parse_without_declarative_shadow_roots(document_url, source.to_owned());
build_detached_document(scope, parsed, DetachedDocumentKind::Html, false)
build_detached_document_from_dom_host_with_content_type(
scope,
DomHost::from_dom(parsed),
DetachedDocumentKind::Html,
false,
Some("text/html"),
character_set,
)
}
pub(crate) fn parse_detached_html_document_from_source_with_declarative_shadow_roots<'s>(
@@ -513,21 +529,6 @@ fn child_document_url_is_xml_like(url: &Url) -> bool {
path.ends_with(".xml") || path.ends_with(".xhtml") || path.ends_with(".svg")
}
fn build_detached_document<'s>(
scope: &mut v8::PinScope<'s, '_>,
parsed: NativeDom,
kind: DetachedDocumentKind,
expose_declarative_shadow_roots: bool,
) -> Option<v8::Local<'s, v8::Object>> {
build_detached_document_with_content_type(
scope,
parsed,
kind,
expose_declarative_shadow_roots,
None,
)
}
fn build_detached_document_with_content_type<'s>(
scope: &mut v8::PinScope<'s, '_>,
parsed: NativeDom,
@@ -1884,7 +1884,11 @@ impl JsContextHost {
let context = v8::Local::new(scope, state.pending.execution_context.context_global()?);
let xhr = v8::Local::new(scope, xhr);
state.body_writer.append(bytes);
let (decoded_text, loaded, total) = state.xhr_response.as_mut()?.append(bytes);
let headers = &state.head.headers;
let (decoded_text, loaded, total) = state.xhr_response.as_mut()?.append(bytes, || {
let scope = &mut v8::ContextScope::new(scope, context);
crate::network_host::xhr_response_text_decoder(scope, xhr, headers)
});
crate::types::XhrStreamingChunkDelivery {
context,
xhr,
+3 -2
View File
@@ -200,8 +200,9 @@ pub(crate) use self::xhr::{
install_window_xml_http_request_template_bindings, install_xml_http_request_bindings,
install_xml_http_request_event_target_bindings, progress_event_constructor_callback,
set_xhr_state_number, throw_synchronous_xhr_failure, xhr_author_request_headers,
xhr_constructor_callback, xhr_ensure_send_allowed, xhr_state_bool_property,
xhr_state_number_property, xhr_state_string_property,
xhr_constructor_callback, xhr_dispatch_progress_event, xhr_ensure_send_allowed,
xhr_response_text_decoder,
xhr_state_bool_property, xhr_state_number_property, xhr_state_string_property,
};
pub(in crate::network_host) const NETWORK_BODY_SLOT: &str = "__lmBody";
pub(in crate::network_host) const NETWORK_BODY_BYTES_SLOT: &str = "__lmBodyBytes";
+4 -1
View File
@@ -1,5 +1,6 @@
mod bindings;
mod delivery;
mod encoding;
mod events;
mod header_surface;
mod instance_state;
@@ -58,6 +59,8 @@ pub(crate) use self::delivery::{
apply_xhr_streaming_response_chunk, apply_xhr_streaming_response_head, apply_xhr_timeout,
finish_xhr_abort, throw_synchronous_xhr_failure,
};
pub(crate) use self::encoding::xhr_response_text_decoder;
pub(crate) use self::events::xhr_dispatch_progress_event;
pub(crate) use self::instance_state::xhr_has_upload_listeners;
pub(crate) use self::instance_state::{
XHR_METHOD_SLOT, XHR_OPEN_GENERATION_SLOT, XHR_REQUEST_HEADERS_SLOT, XHR_RESPONSE_URL_SLOT,
@@ -77,7 +80,7 @@ pub(crate) use self::request_state::begin_xhr_send;
use self::request_state::{
XhrReadyState, clear_xhr_active_request, set_xhr_ready_state, xhr_ready_state,
};
use self::response_type::XmlHttpRequestResponseType;
use self::response_type::{XmlHttpRequestResponseType, xhr_response_type};
#[cfg(test)]
pub(crate) use self::send::prepare_xhr_send_body;
pub(crate) use self::send::{
@@ -11,8 +11,6 @@ const RESPONSE_STATUS_TEXT: &str = "statusText";
const RESPONSE_URL: &str = "url";
const RESPONSE_HEADERS: &str = "headers";
const RESPONSE_BODY: &str = "body";
const RESPONSE_BODY_TEXT: &str = "bodyText";
const RESPONSE_BODY_LENGTH: &str = "bodyLength";
/// Each queued job owns its complete outcome. A later open()/send() cannot
/// overwrite a shared pending payload or let an old job consume the new one.
@@ -45,10 +43,6 @@ struct XhrPendingResponse<'s> {
headers: String,
#[webapi(slot = RESPONSE_BODY)]
body: Option<v8::Local<'s, v8::Value>>,
#[webapi(slot = RESPONSE_BODY_TEXT)]
body_text: String,
#[webapi(slot = RESPONSE_BODY_LENGTH)]
body_length: usize,
}
#[derive(WebApiObject)]
@@ -70,38 +64,16 @@ pub(in crate::network_host::xhr) fn queue_xhr_response_delivery<'s>(
response: Response,
) {
let (head, body) = response.into_body();
let response_type = xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT)
.as_deref()
.and_then(XmlHttpRequestResponseType::parse)
.unwrap_or(XmlHttpRequestResponseType::Default);
let (body, body_text, body_length) = match response_type {
XmlHttpRequestResponseType::ArrayBuffer | XmlHttpRequestResponseType::Blob => {
let bytes = body
.try_into_materialized_bytes()
.expect("XHR response body should remain materialized at the delivery boundary");
let length = bytes.len();
let body =
crate::network_host::network_body_source_object_from_bytes(scope, None, bytes);
(Some(body.into()), String::new(), length)
}
XmlHttpRequestResponseType::Json
| XmlHttpRequestResponseType::Document
| XmlHttpRequestResponseType::Default
| XmlHttpRequestResponseType::Text => {
let (text, bytes) = body
.try_into_lossy_materialized_text()
.expect("XHR response body should remain materialized at the delivery boundary");
(None, text, bytes.len())
}
};
let bytes = body
.try_into_materialized_bytes()
.expect("XHR response body should remain materialized at the delivery boundary");
let body = crate::network_host::network_body_source_object_from_bytes(scope, None, bytes);
let response = XhrPendingResponse::new(
head.status,
head.status_text().to_owned(),
head.final_url.to_string(),
serde_json::to_string(&head.headers).unwrap_or_else(|_| "[]".to_owned()),
body,
body_text,
body_length,
Some(body.into()),
);
queue_xhr_delivery(scope, xhr, XhrPendingDelivery::Response(response));
}
@@ -171,10 +143,6 @@ fn xhr_complete_callback<'s>(
.unwrap_or_default();
let body = get_private_value(scope, response, RESPONSE_BODY)
.filter(|value| !value.is_null_or_undefined());
let body_text = response_string(scope, response, RESPONSE_BODY_TEXT);
let body_length = get_private_value(scope, response, RESPONSE_BODY_LENGTH)
.and_then(|value| value.number_value(scope))
.expect("queued XHR body length") as usize;
apply_xhr_response_pending_body(
scope,
xhr,
@@ -192,8 +160,6 @@ fn xhr_complete_callback<'s>(
negotiated_http_version: None,
},
body,
body_text,
Some(body_length),
);
}
@@ -70,35 +70,12 @@ fn apply_xhr_response_body_source_with_mode(
status_text: Option<&str>,
mode: XhrResponseDeliveryMode,
) {
// XHR's Web-visible surface is still fully materialized. Branch by the
// requested responseType so byte consumers do not pay for an inaccessible
// lossy text view.
match xhr_response_type(scope, xhr) {
XmlHttpRequestResponseType::Json
| XmlHttpRequestResponseType::ArrayBuffer
| XmlHttpRequestResponseType::Blob => {
let body_bytes = body
.try_into_materialized_bytes()
.expect("XHR response body should remain materialized at the V8 boundary");
apply_xhr_response_bytes(scope, xhr, head, body_bytes, status_text, mode);
}
XmlHttpRequestResponseType::Document
| XmlHttpRequestResponseType::Default
| XmlHttpRequestResponseType::Text => {
let (body_text, body_bytes) = body
.try_into_lossy_materialized_text()
.expect("XHR response body should remain materialized at the V8 boundary");
apply_xhr_response_body(
scope,
xhr,
head,
Some(body_text),
body_bytes,
status_text,
mode,
);
}
}
// Choose the response decoder only after HEADERS_RECEIVED handlers have
// had a chance to override the MIME type or change responseType.
let bytes = body
.try_into_materialized_bytes()
.expect("XHR response body should remain materialized at the V8 boundary");
apply_xhr_response_body(scope, xhr, head, bytes, status_text, mode);
}
#[derive(Clone, Copy)]
@@ -119,50 +96,8 @@ pub(in crate::network_host::xhr) fn apply_xhr_response_pending_body<'s>(
xhr: v8::Local<'s, v8::Object>,
head: moli_fetch::ResponseHead,
body_value: Option<v8::Local<'s, v8::Value>>,
fallback_body_text: String,
fallback_body_len: Option<usize>,
) {
let response_type = xhr_response_type(scope, xhr);
if response_type == XmlHttpRequestResponseType::ArrayBuffer
&& let Some(body_value) = body_value
&& let Some(loaded) = buffer_value_byte_length(body_value)
{
apply_xhr_response_prebuilt_value(
scope,
xhr,
head,
response_type,
body_value,
loaded,
None,
XhrResponseDeliveryMode::Buffered,
);
return;
}
if body_value.is_none()
&& matches!(
response_type,
XmlHttpRequestResponseType::Json
| XmlHttpRequestResponseType::Document
| XmlHttpRequestResponseType::Default
| XmlHttpRequestResponseType::Text
)
{
let loaded = fallback_body_len.unwrap_or(fallback_body_text.len());
apply_xhr_response_text(
scope,
xhr,
head,
fallback_body_text,
loaded,
None,
XhrResponseDeliveryMode::Buffered,
);
return;
}
let body_bytes = body_value
let bytes = body_value
.and_then(|value| {
v8::Local::<v8::Object>::try_from(value)
.ok()
@@ -171,33 +106,21 @@ pub(in crate::network_host::xhr) fn apply_xhr_response_pending_body<'s>(
})
.or_else(|| blob::buffer_source_bytes_from_value(scope, value))
})
.unwrap_or_else(|| fallback_body_text.into_bytes());
apply_xhr_response_bytes(
.unwrap_or_default();
apply_xhr_response_body(
scope,
xhr,
head,
body_bytes,
bytes,
None,
XhrResponseDeliveryMode::Buffered,
);
}
fn apply_xhr_response_bytes(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
head: moli_fetch::ResponseHead,
body_bytes: Vec<u8>,
status_text: Option<&str>,
mode: XhrResponseDeliveryMode,
) {
apply_xhr_response_body(scope, xhr, head, None, body_bytes, status_text, mode);
}
fn apply_xhr_response_body(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
head: moli_fetch::ResponseHead,
body_text: Option<String>,
body_bytes: Vec<u8>,
status_text: Option<&str>,
mode: XhrResponseDeliveryMode,
@@ -205,15 +128,18 @@ fn apply_xhr_response_body(
let Some(response_type) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else {
return;
};
let body_text = match response_type {
let (body_text, character_set) = match response_type {
XmlHttpRequestResponseType::Document
| XmlHttpRequestResponseType::Default
| XmlHttpRequestResponseType::Text => {
Some(body_text.unwrap_or_else(|| String::from_utf8_lossy(&body_bytes).into_owned()))
let mut decoder = xhr_response_text_decoder(scope, xhr, &head.headers);
let mut text = decoder.push(&body_bytes);
text.push_str(&decoder.finish());
(Some(text), decoder.encoding_name())
}
XmlHttpRequestResponseType::Json
| XmlHttpRequestResponseType::ArrayBuffer
| XmlHttpRequestResponseType::Blob => None,
| XmlHttpRequestResponseType::Blob => (None, "UTF-8"),
};
let loaded = body_bytes.len() as f64;
let progress = xhr_response_progress(&head, loaded);
@@ -226,6 +152,7 @@ fn apply_xhr_response_body(
&head,
body_text.as_deref().unwrap_or(""),
response_type,
character_set,
);
set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document);
document
@@ -247,6 +174,7 @@ fn apply_xhr_response_body(
&head,
body_text.as_deref().unwrap_or(""),
response_type,
character_set,
);
set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document);
}
@@ -266,52 +194,6 @@ fn apply_xhr_response_body(
);
}
fn apply_xhr_response_text(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
head: moli_fetch::ResponseHead,
body_text: String,
loaded: usize,
status_text: Option<&str>,
mode: XhrResponseDeliveryMode,
) {
let Some(response_type) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else {
return;
};
let progress = xhr_response_progress(&head, loaded as f64);
let response_val: v8::Local<'_, v8::Value> = match response_type {
XmlHttpRequestResponseType::Json => parse_xhr_json_response(scope, body_text.as_bytes()),
XmlHttpRequestResponseType::Document => {
let document =
parse_xhr_response_document(scope, xhr, &head, &body_text, response_type);
set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document);
document
}
XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Text => {
if response_type == XmlHttpRequestResponseType::Default {
let document =
parse_xhr_response_document(scope, xhr, &head, &body_text, response_type);
set_xhr_state_value(scope, xhr, XHR_RESPONSE_XML_SLOT, document);
}
v8_string(scope, &body_text)
.map(|s| s.into())
.unwrap_or_else(|| v8::undefined(scope).into())
}
XmlHttpRequestResponseType::ArrayBuffer | XmlHttpRequestResponseType::Blob => {
v8::undefined(scope).into()
}
};
finish_xhr_response(
scope,
xhr,
response_type,
response_val,
&body_text,
progress,
mode,
);
}
fn parse_xhr_json_response<'s>(
scope: &mut v8::PinScope<'s, '_>,
bytes: &[u8],
@@ -325,23 +207,6 @@ fn parse_xhr_json_response<'s>(
.unwrap_or_else(|| v8::null(scope).into())
}
fn apply_xhr_response_prebuilt_value(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
head: moli_fetch::ResponseHead,
response_type: XmlHttpRequestResponseType,
response_val: v8::Local<'_, v8::Value>,
loaded: usize,
status_text: Option<&str>,
mode: XhrResponseDeliveryMode,
) {
let Some(_) = prepare_xhr_response(scope, xhr, &head, status_text, mode) else {
return;
};
let progress = xhr_response_progress(&head, loaded as f64);
finish_xhr_response(scope, xhr, response_type, response_val, "", progress, mode);
}
pub(crate) fn apply_xhr_streaming_response_head(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
@@ -411,6 +276,16 @@ fn prepare_xhr_response(
}
if matches!(mode, XhrResponseDeliveryMode::Buffered) {
set_xhr_response_head(scope, xhr, head, status_text);
if xhr_is_async(scope, xhr) {
let generation = xhr_state_number_property(scope, xhr, XHR_OPEN_GENERATION_SLOT);
super::super::events::xhr_fire_readystatechange(scope, xhr, XhrReadyState::HeadersReceived);
if scope.is_execution_terminating()
|| xhr_is_aborted(scope, xhr)
|| xhr_state_number_property(scope, xhr, XHR_OPEN_GENERATION_SLOT) != generation
{
return None;
}
}
}
Some(xhr_response_type(scope, xhr))
}
@@ -457,15 +332,6 @@ fn finish_xhr_response(
let dispatch_intermediate_events =
matches!(mode, XhrResponseDeliveryMode::Buffered) && xhr_is_async(scope, xhr);
if dispatch_intermediate_events {
super::super::events::xhr_fire_readystatechange(scope, xhr, XhrReadyState::HeadersReceived);
if scope.is_execution_terminating() {
return;
}
if xhr_is_aborted(scope, xhr) {
return;
}
}
set_xhr_state_string(scope, xhr, XHR_RESPONSE_TEXT_SLOT, response_text);
set_xhr_state_value(scope, xhr, XHR_RESPONSE_SLOT, response_val);
if dispatch_intermediate_events {
@@ -561,26 +427,6 @@ fn identity_encoded_content_length(headers: &[(String, Vec<u8>)]) -> Option<u64>
.then_some(first)
}
fn xhr_response_type(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
) -> XmlHttpRequestResponseType {
xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT)
.as_deref()
.and_then(XmlHttpRequestResponseType::parse)
.unwrap_or(XmlHttpRequestResponseType::Default)
}
fn buffer_value_byte_length(value: v8::Local<'_, v8::Value>) -> Option<usize> {
if let Ok(buffer) = v8::Local::<v8::ArrayBuffer>::try_from(value) {
return Some(buffer.byte_length());
}
if let Ok(view) = v8::Local::<v8::ArrayBufferView>::try_from(value) {
return Some(view.byte_length());
}
None
}
fn xhr_response_mime_essence(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
@@ -603,6 +449,7 @@ fn parse_xhr_response_document<'s>(
head: &moli_fetch::ResponseHead,
body_text: &str,
response_type: XmlHttpRequestResponseType,
character_set: &str,
) -> v8::Local<'s, v8::Value> {
// XHR defaults a missing or invalid response MIME type to text/xml.
// Its XML MIME types include any +xml subtype, unlike DOMParser's enum.
@@ -613,7 +460,7 @@ fn parse_xhr_response_document<'s>(
if (!is_xml && !is_html) || (is_html && response_type != XmlHttpRequestResponseType::Document) {
return v8::null(scope).into();
}
let Some(document) = build_xhr_response_document(scope, xhr, xhr_response_url(head).clone(), body_text, &mime)
let Some(document) = build_xhr_response_document(scope, xhr, xhr_response_url(head).clone(), body_text, &mime, character_set)
else {
return v8::null(scope).into();
};
@@ -639,6 +486,7 @@ fn build_xhr_response_document<'s>(
response_url: url::Url,
body_text: &str,
mime: &str,
character_set: &str,
) -> Option<v8::Local<'s, v8::Object>> {
let xhr = v8::Global::new(scope, xhr);
let xhr = v8::Local::new(scope, xhr);
@@ -662,7 +510,12 @@ fn build_xhr_response_document<'s>(
origin_document_handle,
)?;
let document = if is_html_document_mime(mime) {
dom_parser::parse_detached_html_document_from_source(scope, response_url, body_text)
dom_parser::parse_detached_html_document_from_source_with_encoding(
scope,
response_url,
body_text,
Some(character_set),
)
} else {
dom_parser::parse_detached_xml_document_from_source(
scope,
@@ -0,0 +1,37 @@
use moli_encoding::{XhrResponseDecoder, XhrResponseTextKind, encoding_for_label};
use moli_web_mime::{effective_response_mime_essence, mime_charset, response_content_type};
use super::*;
pub(crate) fn xhr_response_text_decoder(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
headers: &[(String, String)],
) -> XhrResponseDecoder {
let override_mime = xhr_state_string_property(scope, xhr, XHR_OVERRIDE_MIME_TYPE_SLOT);
// A MIME-only override must retain the response charset. Blink also
// ignores an empty override charset, but an unknown nonempty label
// replaces the transport label and lets content/default detection run.
let charset = override_mime
.as_deref()
.and_then(mime_charset)
.filter(|label| !label.is_empty())
.or_else(|| {
response_content_type(headers)
.as_deref()
.and_then(mime_charset)
});
let mime = effective_response_mime_essence(headers, override_mime.as_deref())
.unwrap_or_else(|| "text/xml".to_owned());
let kind = match xhr_response_type(scope, xhr) {
XmlHttpRequestResponseType::Document if mime == "text/html" => XhrResponseTextKind::Html,
XmlHttpRequestResponseType::Default | XmlHttpRequestResponseType::Document
if matches!(mime.as_str(), "text/xml" | "application/xml")
|| mime.ends_with("+xml") =>
{
XhrResponseTextKind::Xml
}
_ => XhrResponseTextKind::Text,
};
XhrResponseDecoder::new(kind, charset.as_deref().and_then(encoding_for_label))
}
@@ -1,3 +1,4 @@
use super::{XHR_RESPONSE_TYPE_SLOT, xhr_state_string_property};
use std::str::FromStr;
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::EnumString, strum::IntoStaticStr)]
@@ -22,6 +23,16 @@ impl XmlHttpRequestResponseType {
}
}
pub(super) fn xhr_response_type(
scope: &mut v8::PinScope<'_, '_>,
xhr: v8::Local<'_, v8::Object>,
) -> XmlHttpRequestResponseType {
xhr_state_string_property(scope, xhr, XHR_RESPONSE_TYPE_SLOT)
.as_deref()
.and_then(XmlHttpRequestResponseType::parse)
.unwrap_or(XmlHttpRequestResponseType::Default)
}
#[cfg(test)]
mod tests {
use super::XmlHttpRequestResponseType;
@@ -1454,6 +1454,146 @@ fn xml_http_request_document_response_requires_an_eligible_mime_and_well_formed_
}
}
#[test]
fn xhr_response_decoding_uses_headers_received_overrides_for_buffered_and_streamed_bytes() {
for streaming in [false, true] {
for (chunks, partial, expected) in [
(
vec![vec![0x83], vec![0x65, 0x83], vec![0x58, 0x83, 0x67]],
vec!["", "テ", "テスト"],
"テスト",
),
(
vec![vec![0xff], vec![0xfe, b'A'], vec![0, 0xe9], vec![0]],
vec!["", "", "A", "Aé"],
"Aé",
),
] {
let mut vm = new_storage_test_vm("https://xhr-response-decoding.test/");
vm.set_fetch_subresource_interception(
true,
Some(crate::types::SubresourceResourceType::Xhr),
);
vm.eval(
r#"
globalThis.__decodingXhr = new XMLHttpRequest();
globalThis.__decodingHeadersText = null;
__decodingXhr.open('GET', '/response');
__decodingXhr.onreadystatechange = () => {
if (__decodingXhr.readyState === 2) {
__decodingHeadersText = __decodingXhr.responseText;
__decodingXhr.overrideMimeType('text/plain;charset=Shift_JIS');
}
};
__decodingXhr.send();
"#,
)
.unwrap();
let requests = vm.take_pending_subresource_fetch_infos();
assert_eq!(requests.len(), 1);
let request = &requests[0];
let bytes = chunks.concat();
let head = moli_fetch::ResponseHead {
final_url: request.url.clone(),
status: 200,
status_text: None,
headers: vec![
(
"Content-Type".to_owned(),
"text/plain;charset=UTF-8".to_owned(),
),
("Content-Length".to_owned(), bytes.len().to_string()),
],
request_cookie_report: None,
cookie_set_reports: Vec::new(),
redirected: false,
redirect_chain: Vec::new(),
from_cache: false,
negotiated_http_version: None,
};
if streaming {
let body_source_id = crate::network_host::new_network_body_source_id();
vm.start_streaming_async_subresource_fetch(
crate::types::AsyncSubresourceStreamingStarted {
internal_id: request.internal_id,
request_url: request.url.clone(),
request_method: "GET".to_owned(),
request_headers: Vec::new(),
request_body: None,
body_source_id,
head,
network_request_headers: None,
},
)
.unwrap();
for (chunk, text) in chunks.into_iter().zip(partial) {
vm.append_streaming_async_subresource_fetch_chunk(body_source_id, chunk);
assert_eq!(vm.eval("__decodingXhr.responseText").unwrap(), text);
}
vm.finish_streaming_async_subresource_fetch(
request.internal_id,
body_source_id,
Ok(()),
)
.unwrap();
} else {
let context_ptr: *const v8::Global<v8::Context> = &vm.page_default_context;
vm.renderer_document_isolate
.with_entered_renderer_document_isolate(move |isolate| {
let scope = std::pin::pin!(v8::HandleScope::new(isolate));
let scope = &mut scope.init();
let context = unsafe { v8::Local::new(scope, &*context_ptr) };
let scope = &mut v8::ContextScope::new(scope, context);
let xhr = context
.global(scope)
.get(scope, v8str(scope, "__decodingXhr").into())
.and_then(|value| v8::Local::<v8::Object>::try_from(value).ok())
.unwrap();
crate::network_host::apply_xhr_response_body_source(
scope,
xhr,
head,
moli_fetch::ResponseBody::materialized_bytes(bytes),
);
Ok(())
})
.unwrap();
}
assert_eq!(vm.eval("__decodingHeadersText").unwrap(), "");
assert_eq!(vm.eval("__decodingXhr.responseText").unwrap(), expected);
assert_eq!(vm.eval("__decodingXhr.response === __decodingXhr.responseText && __decodingXhr.readyState === 4").unwrap(), "true");
}
}
}
#[test]
fn xhr_queued_response_retains_legacy_bytes_and_decodes_json_as_utf8() {
let mut vm = new_storage_test_vm("https://xhr-queued-decoding.test/");
vm.eval(r#"
globalThis.__queuedDecoding = [];
for (const [url, type, override] of [
['data:text/plain;charset=windows-1252,%FF', '', null],
['data:text/plain;charset=utf-8,%83%65%83%58%83%67', '', 'text/plain;charset=Shift_JIS'],
['data:application/json;charset=windows-1252,%EF%BB%BF%7B%22x%22%3A1%7D', 'json', null],
['data:text/html,%3Cmeta%20charset=windows-1252%3E%3Cx%3E%FF%3C%2Fx%3E', 'document', null],
]) {
const xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.responseType = type;
xhr.onreadystatechange = () => {
if (xhr.readyState === 2 && override) xhr.overrideMimeType(override);
};
xhr.onload = () => __queuedDecoding.push(type === 'json' ? xhr.response.x :
type === 'document' ? [xhr.response.querySelector('x').textContent, xhr.response.characterSet] : xhr.responseText);
xhr.send();
}
"#).unwrap();
assert_eq!(
vm.eval("JSON.stringify(__queuedDecoding)").unwrap(),
r#"["ÿ","テスト",1,["ÿ","windows-1252"]]"#
);
}
#[test]
fn xml_http_request_queued_document_response_uses_parsed_overrides_and_xml_errors() {
let mut vm = new_storage_test_vm("https://xhr-queued-document.test/");
+9 -33
View File
@@ -1029,7 +1029,7 @@ pub(super) struct EventSourceStreamingChunkDelivery<'s> {
}
pub(super) struct XhrStreamingResponseState {
pending_utf8_bytes: Vec<u8>,
decoder: Option<moli_encoding::XhrResponseDecoder>,
loaded: usize,
total: Option<usize>,
}
@@ -1046,44 +1046,20 @@ impl XhrStreamingResponseState {
.ok()
});
Self {
pending_utf8_bytes: Vec::new(),
decoder: None,
loaded: 0,
total,
}
}
pub(super) fn append(&mut self, bytes: &[u8]) -> (String, usize, Option<usize>) {
pub(super) fn append(
&mut self,
bytes: &[u8],
make_decoder: impl FnOnce() -> moli_encoding::XhrResponseDecoder,
) -> (String, usize, Option<usize>) {
self.loaded = self.loaded.saturating_add(bytes.len());
self.pending_utf8_bytes.extend_from_slice(bytes);
let mut decoded = String::new();
let mut consumed = 0;
while consumed < self.pending_utf8_bytes.len() {
let remaining = &self.pending_utf8_bytes[consumed..];
match std::str::from_utf8(remaining) {
Ok(text) => {
decoded.push_str(text);
consumed = self.pending_utf8_bytes.len();
}
Err(error) => {
let valid_end = consumed + error.valid_up_to();
decoded.push_str(
std::str::from_utf8(&self.pending_utf8_bytes[consumed..valid_end])
.expect("Utf8Error::valid_up_to must identify a valid UTF-8 prefix"),
);
consumed = valid_end;
let Some(invalid_len) = error.error_len() else {
break;
};
decoded.push('\u{fffd}');
consumed += invalid_len;
}
}
}
if consumed > 0 {
self.pending_utf8_bytes.drain(..consumed);
}
(decoded, self.loaded, self.total)
let decoder = self.decoder.get_or_insert_with(make_decoder);
(decoder.push(bytes), self.loaded, self.total)
}
}