From 3c3bd0806489da612cddfe975d28071b55569672 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Thu, 10 Sep 2026 17:46:05 +0800 Subject: [PATCH] fix(fetch): remove an initial UTF-8 BOM from Body text and JSON Decode buffered and fetched text/json bodies with UTF-8 BOM removal before creating strings or invoking the native JSON parser. Remove exactly one initial BOM, retain replacement behavior for malformed UTF-8, and keep binary/form consumers unchanged. Borrow valid UTF-8 while materializing. Cover constructed Request/Response bodies, delayed ReadableStream input, data URLs, and chunked HTTP in window, child frame, and worker realms. Include split/repeated BOMs, UTF-16 markers and MIME charset, SyntaxError, clone ordering, raw bytes, and URL-encoded/multipart form values. Validation: cargo fmt --all; cargo clippy --workspace --all-targets --all-features -- -D warnings; cargo nextest run --no-fail-fast (17,946 passed, 13 skipped); 9/9 CLI probes. A 40-case WPT comparison adds four passing cases and ten passing subtests without regressions. --- .../wpt-cross-current/passed-cases.txt | 4 + moli-core/tests/fixtures/runtime/body_utf8.js | 106 ++++++++++++++++++ moli-core/tests/web_apis/body_state.rs | 95 ++++++++++++++++ .../src/network_host/body_source.rs | 13 ++- 4 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 moli-core/tests/fixtures/runtime/body_utf8.js diff --git a/moli-benchmark/wpt-cross-current/passed-cases.txt b/moli-benchmark/wpt-cross-current/passed-cases.txt index ef07e2c492..77a9485c0c 100644 --- a/moli-benchmark/wpt-cross-current/passed-cases.txt +++ b/moli-benchmark/wpt-cross-current/passed-cases.txt @@ -5189,6 +5189,8 @@ event-timing/supported-types-consistent-with-self.html fetch/api/abort/destroyed-context.html fetch/api/basic/response-null-body.any.js?moli-wpt-any=dedicatedworker fetch/api/basic/response-null-body.any.js?moli-wpt-any=window +fetch/api/basic/text-utf8.any.js?moli-wpt-any=dedicatedworker +fetch/api/basic/text-utf8.any.js?moli-wpt-any=window fetch/api/cors/data-url-iframe.html fetch/api/cors/data-url-shared-worker.html fetch/api/cors/data-url-worker.html @@ -5206,6 +5208,8 @@ fetch/api/request/request-error.any.js?moli-wpt-any=window fetch/api/request/request-init-stream.any.js?moli-wpt-any=dedicatedworker fetch/api/request/request-init-stream.any.js?moli-wpt-any=window fetch/api/request/url-encoding.html +fetch/api/response/json.any.js?moli-wpt-any=dedicatedworker +fetch/api/response/json.any.js?moli-wpt-any=window fetch/api/response/multi-globals/url-parsing.html fetch/api/response/response-body-read-task-handling.html fetch/api/response/response-error-from-stream.any.js?moli-wpt-any=dedicatedworker diff --git a/moli-core/tests/fixtures/runtime/body_utf8.js b/moli-core/tests/fixtures/runtime/body_utf8.js new file mode 100644 index 0000000000..be0c6f676d --- /dev/null +++ b/moli-core/tests/fixtures/runtime/body_utf8.js @@ -0,0 +1,106 @@ +async function runBodyUtf8Probe(scenario, url) { + const errors = []; + const check = (value, label) => { if (!value) errors.push(label); }; + const encode = value => new TextEncoder().encode(value); + const sameBytes = (actual, expected) => actual.length === expected.length && + actual.every((value, index) => value === expected[index]); + const sources = ['request', 'response', 'request-stream', 'response-stream', 'data', 'network']; + const make = async (source, bytes, type = 'text/plain;charset=UTF-16') => { + let input = bytes; + if (source.endsWith('-stream')) { + let offset = 0; + input = new ReadableStream({async pull(controller) { + await Promise.resolve(); + if (offset === bytes.length) controller.close(); + else { controller.enqueue(bytes.slice(offset, offset + 1)); ++offset; } + }}); + } + if (source.startsWith('request')) { + return new Request(url, {method: 'POST', body: input, duplex: 'half', headers: {'Content-Type': type}}); + } + if (source.startsWith('response')) return new Response(input, {headers: {'Content-Type': type}}); + const hex = Array.from(bytes, value => value.toString(16).padStart(2, '0')); + if (source === 'data') return fetch('data:' + type + ',' + hex.map(value => '%' + value).join('')); + const target = new URL(url); + target.searchParams.set('hex', hex.join('')); + target.searchParams.set('type', type); + return fetch(target); + }; + const consumeTextual = async (source, bytes, method, expected, label, ready) => { + const body = await make(source, bytes); + const clone = body.clone(); + let raw; + if (ready) raw = await clone.bytes(); + try { + const value = await body[method](); + check(expected !== undefined && JSON.stringify(value) === JSON.stringify(expected), label + ': decoded value'); + } catch (error) { + check(expected === undefined && error instanceof SyntaxError, label + ': rejection ' + error); + } + check(body.bodyUsed, label + ': consumed body'); + if (!ready) raw = await clone.bytes(); + check(sameBytes(raw, bytes), label + ': clone preserves exact bytes'); + }; + + for (const source of sources) { + if (scenario === 'text') { + const cases = [ + [encode('\uFEFFA中'), 'A中'], + [encode('\uFEFF\uFEFFx'), '\uFEFFx'], + [encode('x\uFEFF'), 'x\uFEFF'], + [encode('\uFEFF'), ''], + [new Uint8Array([0xEF]), '\uFFFD'], + [new Uint8Array([0xEF, 0xBB]), '\uFFFD'], + [new Uint8Array([0xEF, 0xBB, 0x41]), '\uFFFDA'], + [new Uint8Array([0xFF, 0xFE, 0x41, 0]), '\uFFFD\uFFFDA\0'], + [new Uint8Array([0xEF, 0xBB, 0xBF, 0xF0, 0x9F, 0x41]), '\uFFFDA'], + [encode('A中'), 'A中'], + [new Uint8Array(), ''], + ]; + for (const [index, [bytes, expected]] of cases.entries()) { + await consumeTextual(source, bytes, 'text', expected, source + '/text/' + index, index % 2 === 0); + } + } else if (scenario === 'json') { + const cases = [ + [encode('\uFEFF{"b":1,"a":2,"b":3}'), {b: 3, a: 2}], + [encode('\uFEFF"\uFEFFvalue"'), '\uFEFFvalue'], + [encode('\uFEFF\uFEFF{}'), undefined], + [encode(' \uFEFF{}'), undefined], + [encode('\uFEFF'), undefined], + [new Uint8Array([0xFF, 0xFE, 0x7B, 0, 0x7D, 0]), undefined], + [new Uint8Array([0xEF, 0xBB, 0xBF, 0x22, 0xFF, 0x22]), '\uFFFD'], + [encode('0'), 0], + ]; + for (const [index, [bytes, expected]] of cases.entries()) { + await consumeTextual(source, bytes, 'json', expected, source + '/json/' + index, index % 2 === 0); + } + } else if (scenario === 'bytes') { + const bytes = encode('\uFEFF\uFEFFvalue\uFEFF'); + for (const method of ['bytes', 'arrayBuffer', 'blob', 'stream']) { + const body = await make(source, bytes); + let actual; + if (method === 'stream') { + const reader = body.body.getReader(); + const values = []; + for (;;) { + const {done, value} = await reader.read(); + if (done) break; + values.push(...value); + } + actual = new Uint8Array(values); + } else { + const value = await body[method](); + actual = method === 'bytes' ? value : new Uint8Array(method === 'blob' ? await value.arrayBuffer() : value); + } + check(sameBytes(actual, bytes), source + '/' + method + ': raw BOM bytes'); + } + const form = await (await make(source, encode('\uFEFFname=\uFEFFvalue'), + 'application/x-www-form-urlencoded')).formData(); + check(form.get('\uFEFFname') === '\uFEFFvalue', source + ': urlencoded BOMs'); + const multipart = '--probe\r\nContent-Disposition: form-data; name="name"\r\n\r\n\uFEFFvalue\uFEFF\r\n--probe--\r\n'; + const fields = await (await make(source, encode(multipart), 'multipart/form-data;boundary=probe')).formData(); + check(fields.get('name') === '\uFEFFvalue\uFEFF', source + ': multipart BOMs'); + } + } + return {errors}; +} diff --git a/moli-core/tests/web_apis/body_state.rs b/moli-core/tests/web_apis/body_state.rs index b97fba20fc..cc692c0f3c 100644 --- a/moli-core/tests/web_apis/body_state.rs +++ b/moli-core/tests/web_apis/body_state.rs @@ -1,5 +1,100 @@ use super::*; +async fn assert_body_utf8_decoding(scenario: &str) -> Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let response_url = format!("http://{}/", listener.local_addr()?); + let mut tasks = tokio::task::JoinSet::new(); + tasks.spawn(async move { + let mut connections = tokio::task::JoinSet::new(); + while let Ok((mut stream, _)) = listener.accept().await { + connections.spawn(async move { + let mut request = Vec::new(); + let mut buffer = [0u8; 1024]; + while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { + let len = stream.read(&mut buffer).await.unwrap(); + if len == 0 { + return; + } + request.extend_from_slice(&buffer[..len]); + assert!(request.len() < 8192, "unexpectedly large request headers"); + } + let request = String::from_utf8(request).unwrap(); + let path = request.split_whitespace().nth(1).unwrap(); + let url = url::Url::parse(&format!("http://fixture{path}")).unwrap(); + let hex = url.query_pairs().find(|(name, _)| name == "hex") + .map(|(_, value)| value.into_owned()).unwrap_or_default(); + let body: Vec = hex.as_bytes().chunks_exact(2).map(|pair| { + u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap() + }).collect(); + let mime = url.query_pairs().find(|(name, _)| name == "type") + .map(|(_, value)| value.into_owned()).unwrap(); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Type: {mime}\r\nAccess-Control-Allow-Origin: *\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + ); + if stream.write_all(head.as_bytes()).await.is_err() { + return; + } + // Split the BOM over transport chunks before delivering the + // remaining bytes. Consumers must decode after fully reading. + let split = body.len().min(3); + for chunk in body[..split].chunks(1).chain(std::iter::once(&body[split..])) { + if chunk.is_empty() { + continue; + } + let mut encoded = format!("{:X}\r\n", chunk.len()).into_bytes(); + encoded.extend_from_slice(chunk); + encoded.extend_from_slice(b"\r\n"); + if stream.write_all(&encoded).await.is_err() { + return; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + let _ = stream.write_all(b"0\r\n\r\n").await; + }); + } + }); + let server = FixtureServer::spawn().await?; + let browser = Browser::new(AppConfig::default())?; + let source = format!( + "{}\nrunBodyUtf8Probe({}, {}).then(finish, error => finish({{error: String(error)}}));", + include_str!("../fixtures/runtime/body_utf8.js"), + serde_json::to_string(scenario)?, + serde_json::to_string(&response_url)?, + ); + for target in ["window", "child", "worker"] { + let observed = tokio::time::timeout( + Duration::from_secs(20), + super::event_dispatch::run_probe(&browser, &server, target, &source), + ) + .await??; + assert_eq!( + observed, + serde_json::json!({"errors": []}), + "{scenario}/{target}" + ); + } + server.shutdown().await; + tasks.shutdown().await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetch_body_text_decodes_utf8_and_removes_only_one_initial_bom() -> Result<()> { + assert_body_utf8_decoding("text").await +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetch_body_json_decodes_utf8_before_parsing_and_preserves_syntax_errors() -> Result<()> { + assert_body_utf8_decoding("json").await +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetch_body_binary_and_form_consumers_preserve_bom_bytes_and_values() -> Result<()> { + assert_body_utf8_decoding("bytes").await +} + #[tokio::test(flavor = "multi_thread")] async fn fetched_null_bodies_stay_unused_through_consumption_cloning_and_abort() -> Result<()> { use tokio::io::{AsyncReadExt, AsyncWriteExt}; diff --git a/moli-renderer-v8/src/network_host/body_source.rs b/moli-renderer-v8/src/network_host/body_source.rs index e9621d62ed..f726227213 100644 --- a/moli-renderer-v8/src/network_host/body_source.rs +++ b/moli-renderer-v8/src/network_host/body_source.rs @@ -1848,11 +1848,16 @@ fn body_materialization_value<'s>( kind: PendingBodyMaterializationKind, ) -> Result, v8::Local<'s, v8::Value>> { match kind { - PendingBodyMaterializationKind::Text => v8_string(scope, &String::from_utf8_lossy(bytes)) - .map(Into::into) - .ok_or_else(|| v8::undefined(scope).into()), + PendingBodyMaterializationKind::Text => { + // Fetch's UTF-8 decode removes one initial UTF-8 BOM without + // selecting a different encoding from the bytes or MIME type. + let text = encoding_rs::UTF_8.decode_with_bom_removal(bytes).0; + v8_string(scope, &text) + .map(Into::into) + .ok_or_else(|| v8::undefined(scope).into()) + } PendingBodyMaterializationKind::Json => { - let text = String::from_utf8_lossy(bytes).into_owned(); + let text = encoding_rs::UTF_8.decode_with_bom_removal(bytes).0; let Some(text) = v8_string(scope, &text) else { return Err(v8::undefined(scope).into()); };