fix(fetch): remove an initial UTF-8 BOM from Body text and JSON

Decode buffered and fetched text/json bodies with UTF-8 BOM removal before
creating strings or invoking the native JSON parser. Remove exactly one
initial BOM, retain replacement behavior for malformed UTF-8, and keep
binary/form consumers unchanged. Borrow valid UTF-8 while materializing.

Cover constructed Request/Response bodies, delayed ReadableStream input,
data URLs, and chunked HTTP in window, child frame, and worker realms.
Include split/repeated BOMs, UTF-16 markers and MIME charset, SyntaxError,
clone ordering, raw bytes, and URL-encoded/multipart form values.

Validation: cargo fmt --all; cargo clippy --workspace --all-targets
--all-features -- -D warnings; cargo nextest run --no-fail-fast
(17,946 passed, 13 skipped); 9/9 CLI probes. A 40-case WPT comparison adds
four passing cases and ten passing subtests without regressions.

moli-benchmark/wpt-cross-current/passed-cases.txt

@@ -5189,6 +5189,8 @@ event-timing/supported-types-consistent-with-self.html
 fetch/api/abort/destroyed-context.html
 fetch/api/basic/response-null-body.any.js?moli-wpt-any=dedicatedworker
 fetch/api/basic/response-null-body.any.js?moli-wpt-any=window
+fetch/api/basic/text-utf8.any.js?moli-wpt-any=dedicatedworker
+fetch/api/basic/text-utf8.any.js?moli-wpt-any=window
 fetch/api/cors/data-url-iframe.html
 fetch/api/cors/data-url-shared-worker.html
 fetch/api/cors/data-url-worker.html
@@ -5206,6 +5208,8 @@ fetch/api/request/request-error.any.js?moli-wpt-any=window
 fetch/api/request/request-init-stream.any.js?moli-wpt-any=dedicatedworker
 fetch/api/request/request-init-stream.any.js?moli-wpt-any=window
 fetch/api/request/url-encoding.html
+fetch/api/response/json.any.js?moli-wpt-any=dedicatedworker
+fetch/api/response/json.any.js?moli-wpt-any=window
 fetch/api/response/multi-globals/url-parsing.html
 fetch/api/response/response-body-read-task-handling.html
 fetch/api/response/response-error-from-stream.any.js?moli-wpt-any=dedicatedworker

moli-core/tests/fixtures/runtime/body_utf8.js

@@ -0,0 +1,106 @@
+async function runBodyUtf8Probe(scenario, url) {
+  const errors = [];
+  const check = (value, label) => { if (!value) errors.push(label); };
+  const encode = value => new TextEncoder().encode(value);
+  const sameBytes = (actual, expected) => actual.length === expected.length &&
+    actual.every((value, index) => value === expected[index]);
+  const sources = ['request', 'response', 'request-stream', 'response-stream', 'data', 'network'];
+  const make = async (source, bytes, type = 'text/plain;charset=UTF-16') => {
+    let input = bytes;
+    if (source.endsWith('-stream')) {
+      let offset = 0;
+      input = new ReadableStream({async pull(controller) {
+        await Promise.resolve();
+        if (offset === bytes.length) controller.close();
+        else { controller.enqueue(bytes.slice(offset, offset + 1)); ++offset; }
+      }});
+    }
+    if (source.startsWith('request')) {
+      return new Request(url, {method: 'POST', body: input, duplex: 'half', headers: {'Content-Type': type}});
+    }
+    if (source.startsWith('response')) return new Response(input, {headers: {'Content-Type': type}});
+    const hex = Array.from(bytes, value => value.toString(16).padStart(2, '0'));
+    if (source === 'data') return fetch('data:' + type + ',' + hex.map(value => '%' + value).join(''));
+    const target = new URL(url);
+    target.searchParams.set('hex', hex.join(''));
+    target.searchParams.set('type', type);
+    return fetch(target);
+  };
+  const consumeTextual = async (source, bytes, method, expected, label, ready) => {
+    const body = await make(source, bytes);
+    const clone = body.clone();
+    let raw;
+    if (ready) raw = await clone.bytes();
+    try {
+      const value = await body[method]();
+      check(expected !== undefined && JSON.stringify(value) === JSON.stringify(expected), label + ': decoded value');
+    } catch (error) {
+      check(expected === undefined && error instanceof SyntaxError, label + ': rejection ' + error);
+    }
+    check(body.bodyUsed, label + ': consumed body');
+    if (!ready) raw = await clone.bytes();
+    check(sameBytes(raw, bytes), label + ': clone preserves exact bytes');
+  };
+
+  for (const source of sources) {
+    if (scenario === 'text') {
+      const cases = [
+        [encode('\uFEFFA中'), 'A中'],
+        [encode('\uFEFF\uFEFFx'), '\uFEFFx'],
+        [encode('x\uFEFF'), 'x\uFEFF'],
+        [encode('\uFEFF'), ''],
+        [new Uint8Array([0xEF]), '\uFFFD'],
+        [new Uint8Array([0xEF, 0xBB]), '\uFFFD'],
+        [new Uint8Array([0xEF, 0xBB, 0x41]), '\uFFFDA'],
+        [new Uint8Array([0xFF, 0xFE, 0x41, 0]), '\uFFFD\uFFFDA\0'],
+        [new Uint8Array([0xEF, 0xBB, 0xBF, 0xF0, 0x9F, 0x41]), '\uFFFDA'],
+        [encode('A中'), 'A中'],
+        [new Uint8Array(), ''],
+      ];
+      for (const [index, [bytes, expected]] of cases.entries()) {
+        await consumeTextual(source, bytes, 'text', expected, source + '/text/' + index, index % 2 === 0);
+      }
+    } else if (scenario === 'json') {
+      const cases = [
+        [encode('\uFEFF{"b":1,"a":2,"b":3}'), {b: 3, a: 2}],
+        [encode('\uFEFF"\uFEFFvalue"'), '\uFEFFvalue'],
+        [encode('\uFEFF\uFEFF{}'), undefined],
+        [encode(' \uFEFF{}'), undefined],
+        [encode('\uFEFF'), undefined],
+        [new Uint8Array([0xFF, 0xFE, 0x7B, 0, 0x7D, 0]), undefined],
+        [new Uint8Array([0xEF, 0xBB, 0xBF, 0x22, 0xFF, 0x22]), '\uFFFD'],
+        [encode('0'), 0],
+      ];
+      for (const [index, [bytes, expected]] of cases.entries()) {
+        await consumeTextual(source, bytes, 'json', expected, source + '/json/' + index, index % 2 === 0);
+      }
+    } else if (scenario === 'bytes') {
+      const bytes = encode('\uFEFF\uFEFFvalue\uFEFF');
+      for (const method of ['bytes', 'arrayBuffer', 'blob', 'stream']) {
+        const body = await make(source, bytes);
+        let actual;
+        if (method === 'stream') {
+          const reader = body.body.getReader();
+          const values = [];
+          for (;;) {
+            const {done, value} = await reader.read();
+            if (done) break;
+            values.push(...value);
+          }
+          actual = new Uint8Array(values);
+        } else {
+          const value = await body[method]();
+          actual = method === 'bytes' ? value : new Uint8Array(method === 'blob' ? await value.arrayBuffer() : value);
+        }
+        check(sameBytes(actual, bytes), source + '/' + method + ': raw BOM bytes');
+      }
+      const form = await (await make(source, encode('\uFEFFname=\uFEFFvalue'),
+        'application/x-www-form-urlencoded')).formData();
+      check(form.get('\uFEFFname') === '\uFEFFvalue', source + ': urlencoded BOMs');
+      const multipart = '--probe\r\nContent-Disposition: form-data; name="name"\r\n\r\n\uFEFFvalue\uFEFF\r\n--probe--\r\n';
+      const fields = await (await make(source, encode(multipart), 'multipart/form-data;boundary=probe')).formData();
+      check(fields.get('name') === '\uFEFFvalue\uFEFF', source + ': multipart BOMs');
+    }
+  }
+  return {errors};
+}

moli-core/tests/web_apis/body_state.rs

@@ -1,5 +1,100 @@
 use super::*;
 
+async fn assert_body_utf8_decoding(scenario: &str) -> Result<()> {
+    use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
+    let response_url = format!("http://{}/", listener.local_addr()?);
+    let mut tasks = tokio::task::JoinSet::new();
+    tasks.spawn(async move {
+        let mut connections = tokio::task::JoinSet::new();
+        while let Ok((mut stream, _)) = listener.accept().await {
+            connections.spawn(async move {
+                let mut request = Vec::new();
+                let mut buffer = [0u8; 1024];
+                while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {
+                    let len = stream.read(&mut buffer).await.unwrap();
+                    if len == 0 {
+                        return;
+                    }
+                    request.extend_from_slice(&buffer[..len]);
+                    assert!(request.len() < 8192, "unexpectedly large request headers");
+                }
+                let request = String::from_utf8(request).unwrap();
+                let path = request.split_whitespace().nth(1).unwrap();
+                let url = url::Url::parse(&format!("http://fixture{path}")).unwrap();
+                let hex = url.query_pairs().find(|(name, _)| name == "hex")
+                    .map(|(_, value)| value.into_owned()).unwrap_or_default();
+                let body: Vec<u8> = hex.as_bytes().chunks_exact(2).map(|pair| {
+                    u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()
+                }).collect();
+                let mime = url.query_pairs().find(|(name, _)| name == "type")
+                    .map(|(_, value)| value.into_owned()).unwrap();
+                let head = format!(
+                    "HTTP/1.1 200 OK\r\nContent-Type: {mime}\r\nAccess-Control-Allow-Origin: *\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n"
+                );
+                if stream.write_all(head.as_bytes()).await.is_err() {
+                    return;
+                }
+                // Split the BOM over transport chunks before delivering the
+                // remaining bytes. Consumers must decode after fully reading.
+                let split = body.len().min(3);
+                for chunk in body[..split].chunks(1).chain(std::iter::once(&body[split..])) {
+                    if chunk.is_empty() {
+                        continue;
+                    }
+                    let mut encoded = format!("{:X}\r\n", chunk.len()).into_bytes();
+                    encoded.extend_from_slice(chunk);
+                    encoded.extend_from_slice(b"\r\n");
+                    if stream.write_all(&encoded).await.is_err() {
+                        return;
+                    }
+                    tokio::time::sleep(Duration::from_millis(5)).await;
+                }
+                let _ = stream.write_all(b"0\r\n\r\n").await;
+            });
+        }
+    });
+    let server = FixtureServer::spawn().await?;
+    let browser = Browser::new(AppConfig::default())?;
+    let source = format!(
+        "{}\nrunBodyUtf8Probe({}, {}).then(finish, error => finish({{error: String(error)}}));",
+        include_str!("../fixtures/runtime/body_utf8.js"),
+        serde_json::to_string(scenario)?,
+        serde_json::to_string(&response_url)?,
+    );
+    for target in ["window", "child", "worker"] {
+        let observed = tokio::time::timeout(
+            Duration::from_secs(20),
+            super::event_dispatch::run_probe(&browser, &server, target, &source),
+        )
+        .await??;
+        assert_eq!(
+            observed,
+            serde_json::json!({"errors": []}),
+            "{scenario}/{target}"
+        );
+    }
+    server.shutdown().await;
+    tasks.shutdown().await;
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn fetch_body_text_decodes_utf8_and_removes_only_one_initial_bom() -> Result<()> {
+    assert_body_utf8_decoding("text").await
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn fetch_body_json_decodes_utf8_before_parsing_and_preserves_syntax_errors() -> Result<()> {
+    assert_body_utf8_decoding("json").await
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn fetch_body_binary_and_form_consumers_preserve_bom_bytes_and_values() -> Result<()> {
+    assert_body_utf8_decoding("bytes").await
+}
+
 #[tokio::test(flavor = "multi_thread")]
 async fn fetched_null_bodies_stay_unused_through_consumption_cloning_and_abort() -> Result<()> {
     use tokio::io::{AsyncReadExt, AsyncWriteExt};

moli-renderer-v8/src/network_host/body_source.rs

@@ -1848,11 +1848,16 @@ fn body_materialization_value<'s>(
     kind: PendingBodyMaterializationKind,
 ) -> Result<v8::Local<'s, v8::Value>, v8::Local<'s, v8::Value>> {
     match kind {
-        PendingBodyMaterializationKind::Text => v8_string(scope, &String::from_utf8_lossy(bytes))
-            .map(Into::into)
-            .ok_or_else(|| v8::undefined(scope).into()),
+        PendingBodyMaterializationKind::Text => {
+            // Fetch's UTF-8 decode removes one initial UTF-8 BOM without
+            // selecting a different encoding from the bytes or MIME type.
+            let text = encoding_rs::UTF_8.decode_with_bom_removal(bytes).0;
+            v8_string(scope, &text)
+                .map(Into::into)
+                .ok_or_else(|| v8::undefined(scope).into())
+        }
         PendingBodyMaterializationKind::Json => {
-            let text = String::from_utf8_lossy(bytes).into_owned();
+            let text = encoding_rs::UTF_8.decode_with_bom_removal(bytes).0;
             let Some(text) = v8_string(scope, &text) else {
                 return Err(v8::undefined(scope).into());
             };