diff --git a/Cargo.lock b/Cargo.lock index a00b2c30fab..8f1876a3b96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -783,6 +783,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "core_maths" version = "0.1.1" @@ -1345,11 +1351,16 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] @@ -4028,6 +4039,12 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "netscape-cookie-file-parser" version = "0.3.0" @@ -5511,6 +5528,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simplecss" version = "0.2.2" diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.toml index b06acead4f8..1b2af1cb264 100644 --- a/moli-renderer-v8/Cargo.toml +++ b/moli-renderer-v8/Cargo.toml @@ -30,7 +30,7 @@ crossbeam-channel = "0.5.15" cssparser = "0.38" data-url = "0.3.2" dom = { package = "stylo_dom", version = "0.22" } -encoding_rs = "0.8" +encoding_rs = "0.8.42" euclid = "0.22" kurbo = "0.13.1" futures-util = "0.3.31" diff --git a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs index f34ee99b247..ae39733d1c6 100644 --- a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs +++ b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs @@ -666,6 +666,29 @@ fn text_decoder_stream_uses_encoding_rs_labels_and_streaming_decode() { assert_eq!(settled, r#"["太平洋:false"]"#); } +#[test] +fn text_decoder_preserves_legacy_sequences_across_empty_chunks() { + let mut vm = new_storage_test_vm("https://legacy-decoder-empty-chunks.test/"); + vm.eval(include_str!( + "../../../../tests/fixtures/legacy-decoder-empty-chunks.js" + )) + .expect("legacy decoder contract should load"); + vm.eval("runLegacyDecoderEmptyChunkContract().then(result => globalThis.__legacyDecoderEmptyChunkResults = result)") + .expect("legacy decoder contract should execute"); + let facts: serde_json::Value = serde_json::from_str( + &vm.eval("JSON.stringify(__legacyDecoderEmptyChunkResults)") + .expect("legacy decoder results should serialize"), + ) + .expect("legacy decoder results should be valid JSON"); + let checks = facts["checks"].as_array().expect("legacy decoder checks"); + let failures = checks + .iter() + .filter(|check| check["passed"] != true) + .collect::>(); + assert!(failures.is_empty(), "{failures:#?}"); + assert_eq!(checks.len(), 54); +} + #[test] fn text_encoder_stream_preserves_utf16_chunk_boundaries() { let mut vm = new_storage_test_vm("https://text-encoder-boundaries.test/"); diff --git a/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs b/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs index 6e38652c59f..91f26d52341 100644 --- a/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs +++ b/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs @@ -1,5 +1,38 @@ use super::*; +#[tokio::test] +async fn worker_text_decoder_preserves_legacy_sequences_across_empty_chunks() { + ensure_v8(); + let source = [ + include_str!("../../../../../tests/fixtures/legacy-decoder-empty-chunks.js"), + r#" +runLegacyDecoderEmptyChunkContract().then( + result => { postMessage(result); close(); }, + error => { postMessage({ failure: String(error) }); close(); } +); +"#, + ] + .join("\n"); + let mut handle = spawn_worker( + source, + "https://legacy-decoder-empty-chunks.test/worker.js".into(), + ); + let message = timeout(TIMEOUT, handle.recv()) + .await + .expect("legacy decoder contract should complete") + .expect("worker result channel should stay open"); + handle.terminate_and_join(); + let facts: serde_json::Value = serde_json::from_str(&expect_post_json(message)) + .expect("legacy decoder results should be valid JSON"); + let checks = facts["checks"].as_array().expect("legacy decoder checks"); + let failures = checks + .iter() + .filter(|check| check["passed"] != true) + .collect::>(); + assert!(failures.is_empty(), "{failures:#?}"); + assert_eq!(checks.len(), 54); +} + #[tokio::test] async fn worker_text_encoder_stream_preserves_utf16_chunk_boundaries() { ensure_v8(); diff --git a/moli-renderer-v8/tests/fixtures/legacy-decoder-empty-chunks.js b/moli-renderer-v8/tests/fixtures/legacy-decoder-empty-chunks.js new file mode 100644 index 00000000000..8129fcb7dbc --- /dev/null +++ b/moli-renderer-v8/tests/fixtures/legacy-decoder-empty-chunks.js @@ -0,0 +1,72 @@ +globalThis.runLegacyDecoderEmptyChunkContract = async function () { + const checks = []; + async function check(name, operation) { + globalThis.__legacyDecoderEmptyChunkProgress = name; + try { checks.push({ name, passed: await operation() === true }); } + catch (error) { checks.push({ name, passed: false, error: String(error) }); } + } + const vectors = [ + ["big5", [0xfe, 0x40], "\u9442"], + ["shift_jis", [0x81, 0x87], "\u221e"], + ["euc-kr", [0x81, 0x41], "\uac02"], + ]; + const emptyChunks = count => Array.from({ length: count }, () => new Uint8Array()); + function chunks(bytes, emptyCount) { + return [new Uint8Array(), Uint8Array.of(bytes[0]), ...emptyChunks(emptyCount), + Uint8Array.of(bytes[1]), new Uint8Array()]; + } + async function streamResult(encoding, fatal, input) { + const source = new ReadableStream({ start(controller) { + for (const chunk of input) controller.enqueue(chunk); + controller.close(); + } }); + const reader = source.pipeThrough(new TextDecoderStream(encoding, { fatal })).getReader(); + const result = { chunks: [], error: undefined }; + try { + while (true) { + const item = await reader.read(); + if (item.done) break; + result.chunks.push(item.value); + } + } catch (error) { result.error = error; } + return result; + } + for (const [encoding, bytes, expected] of vectors) { + for (const fatal of [false, true]) { + await check(`${encoding}, fatal=${fatal}: whole input`, () => + new TextDecoder(encoding, { fatal }).decode(Uint8Array.from(bytes)) === expected); + for (const count of [0, 1, 4]) { + await check(`${encoding}, fatal=${fatal}: decode across ${count} empty chunks`, () => { + const decoder = new TextDecoder(encoding, { fatal }); + let output = ""; + for (const chunk of chunks(bytes, count)) output += decoder.decode(chunk, { stream: true }); + output += decoder.decode(); + return output === expected; + }); + await check(`${encoding}, fatal=${fatal}: stream across ${count} empty chunks`, async () => { + const result = await streamResult(encoding, fatal, chunks(bytes, count)); + return result.error === undefined && result.chunks.every(chunk => chunk.length > 0) && + result.chunks.join("") === expected; + }); + } + await check(`${encoding}, fatal=${fatal}: pending byte at decode EOF`, () => { + const decoder = new TextDecoder(encoding, { fatal }); + if (decoder.decode(Uint8Array.of(bytes[0]), { stream: true }) !== "") return false; + for (const chunk of emptyChunks(4)) { + if (decoder.decode(chunk, { stream: true }) !== "") return false; + } + let replaced = false, failed = false; + try { replaced = decoder.decode() === "\ufffd"; } + catch (error) { failed = error instanceof TypeError; } + return (fatal ? failed : replaced) && decoder.decode(Uint8Array.of(65)) === "A"; + }); + await check(`${encoding}, fatal=${fatal}: pending byte at stream EOF`, async () => { + const result = await streamResult(encoding, fatal, + [Uint8Array.of(bytes[0]), ...emptyChunks(4)]); + return fatal ? result.error instanceof TypeError && result.chunks.length === 0 : + result.error === undefined && JSON.stringify(result.chunks) === JSON.stringify(["\ufffd"]); + }); + } + } + return { checks }; +}; diff --git a/moli-webapi/Cargo.toml b/moli-webapi/Cargo.toml index 33bc9917272..3ff61a2898c 100644 --- a/moli-webapi/Cargo.toml +++ b/moli-webapi/Cargo.toml @@ -9,7 +9,7 @@ brotlic = "0.8.2" flate2 = { version = "1.1.9", default-features = false, features = ["zlib"] } moli-streams = { path = "../moli-streams" } moli-file-api = { path = "../moli-file-api" } -encoding_rs = "0.8" +encoding_rs = "0.8.42" parking_lot = "0.12" moli-websocket-types = { path = "../moli-types/moli-websocket-types" } moli-indexeddb-types = { path = "../moli-types/moli-indexeddb-types" }