fix(encoding): preserve legacy decoder state across empty chunks

This commit is contained in:
ldm0
2026-11-21 20:03:34 +08:00
parent 14eb6f8eed
commit e7a79b321f
6 changed files with 155 additions and 4 deletions
Generated
+25 -2
View File
@@ -783,6 +783,12 @@ dependencies = [
"version_check",
]
[[package]]
name = "core_detect"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
[[package]]
name = "core_maths"
version = "0.1.1"
@@ -1345,11 +1351,16 @@ dependencies = [
[[package]]
name = "encoding_rs"
version = "0.8.35"
version = "0.8.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
dependencies = [
"cfg-if",
"core_detect",
"multiversion_no_op",
"rustversion",
"scopeguard",
"simdutf8",
]
[[package]]
@@ -4028,6 +4039,12 @@ dependencies = [
"pxfm",
]
[[package]]
name = "multiversion_no_op"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
[[package]]
name = "netscape-cookie-file-parser"
version = "0.3.0"
@@ -5511,6 +5528,12 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
[[package]]
name = "simdutf8"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
[[package]]
name = "simplecss"
version = "0.2.2"
+1 -1
View File
@@ -30,7 +30,7 @@ crossbeam-channel = "0.5.15"
cssparser = "0.38"
data-url = "0.3.2"
dom = { package = "stylo_dom", version = "0.22" }
encoding_rs = "0.8"
encoding_rs = "0.8.42"
euclid = "0.22"
kurbo = "0.13.1"
futures-util = "0.3.31"
@@ -666,6 +666,29 @@ fn text_decoder_stream_uses_encoding_rs_labels_and_streaming_decode() {
assert_eq!(settled, r#"["太平洋:false"]"#);
}
#[test]
fn text_decoder_preserves_legacy_sequences_across_empty_chunks() {
let mut vm = new_storage_test_vm("https://legacy-decoder-empty-chunks.test/");
vm.eval(include_str!(
"../../../../tests/fixtures/legacy-decoder-empty-chunks.js"
))
.expect("legacy decoder contract should load");
vm.eval("runLegacyDecoderEmptyChunkContract().then(result => globalThis.__legacyDecoderEmptyChunkResults = result)")
.expect("legacy decoder contract should execute");
let facts: serde_json::Value = serde_json::from_str(
&vm.eval("JSON.stringify(__legacyDecoderEmptyChunkResults)")
.expect("legacy decoder results should serialize"),
)
.expect("legacy decoder results should be valid JSON");
let checks = facts["checks"].as_array().expect("legacy decoder checks");
let failures = checks
.iter()
.filter(|check| check["passed"] != true)
.collect::<Vec<_>>();
assert!(failures.is_empty(), "{failures:#?}");
assert_eq!(checks.len(), 54);
}
#[test]
fn text_encoder_stream_preserves_utf16_chunk_boundaries() {
let mut vm = new_storage_test_vm("https://text-encoder-boundaries.test/");
@@ -1,5 +1,38 @@
use super::*;
#[tokio::test]
async fn worker_text_decoder_preserves_legacy_sequences_across_empty_chunks() {
ensure_v8();
let source = [
include_str!("../../../../../tests/fixtures/legacy-decoder-empty-chunks.js"),
r#"
runLegacyDecoderEmptyChunkContract().then(
result => { postMessage(result); close(); },
error => { postMessage({ failure: String(error) }); close(); }
);
"#,
]
.join("\n");
let mut handle = spawn_worker(
source,
"https://legacy-decoder-empty-chunks.test/worker.js".into(),
);
let message = timeout(TIMEOUT, handle.recv())
.await
.expect("legacy decoder contract should complete")
.expect("worker result channel should stay open");
handle.terminate_and_join();
let facts: serde_json::Value = serde_json::from_str(&expect_post_json(message))
.expect("legacy decoder results should be valid JSON");
let checks = facts["checks"].as_array().expect("legacy decoder checks");
let failures = checks
.iter()
.filter(|check| check["passed"] != true)
.collect::<Vec<_>>();
assert!(failures.is_empty(), "{failures:#?}");
assert_eq!(checks.len(), 54);
}
#[tokio::test]
async fn worker_text_encoder_stream_preserves_utf16_chunk_boundaries() {
ensure_v8();
@@ -0,0 +1,72 @@
globalThis.runLegacyDecoderEmptyChunkContract = async function () {
const checks = [];
async function check(name, operation) {
globalThis.__legacyDecoderEmptyChunkProgress = name;
try { checks.push({ name, passed: await operation() === true }); }
catch (error) { checks.push({ name, passed: false, error: String(error) }); }
}
const vectors = [
["big5", [0xfe, 0x40], "\u9442"],
["shift_jis", [0x81, 0x87], "\u221e"],
["euc-kr", [0x81, 0x41], "\uac02"],
];
const emptyChunks = count => Array.from({ length: count }, () => new Uint8Array());
function chunks(bytes, emptyCount) {
return [new Uint8Array(), Uint8Array.of(bytes[0]), ...emptyChunks(emptyCount),
Uint8Array.of(bytes[1]), new Uint8Array()];
}
async function streamResult(encoding, fatal, input) {
const source = new ReadableStream({ start(controller) {
for (const chunk of input) controller.enqueue(chunk);
controller.close();
} });
const reader = source.pipeThrough(new TextDecoderStream(encoding, { fatal })).getReader();
const result = { chunks: [], error: undefined };
try {
while (true) {
const item = await reader.read();
if (item.done) break;
result.chunks.push(item.value);
}
} catch (error) { result.error = error; }
return result;
}
for (const [encoding, bytes, expected] of vectors) {
for (const fatal of [false, true]) {
await check(`${encoding}, fatal=${fatal}: whole input`, () =>
new TextDecoder(encoding, { fatal }).decode(Uint8Array.from(bytes)) === expected);
for (const count of [0, 1, 4]) {
await check(`${encoding}, fatal=${fatal}: decode across ${count} empty chunks`, () => {
const decoder = new TextDecoder(encoding, { fatal });
let output = "";
for (const chunk of chunks(bytes, count)) output += decoder.decode(chunk, { stream: true });
output += decoder.decode();
return output === expected;
});
await check(`${encoding}, fatal=${fatal}: stream across ${count} empty chunks`, async () => {
const result = await streamResult(encoding, fatal, chunks(bytes, count));
return result.error === undefined && result.chunks.every(chunk => chunk.length > 0) &&
result.chunks.join("") === expected;
});
}
await check(`${encoding}, fatal=${fatal}: pending byte at decode EOF`, () => {
const decoder = new TextDecoder(encoding, { fatal });
if (decoder.decode(Uint8Array.of(bytes[0]), { stream: true }) !== "") return false;
for (const chunk of emptyChunks(4)) {
if (decoder.decode(chunk, { stream: true }) !== "") return false;
}
let replaced = false, failed = false;
try { replaced = decoder.decode() === "\ufffd"; }
catch (error) { failed = error instanceof TypeError; }
return (fatal ? failed : replaced) && decoder.decode(Uint8Array.of(65)) === "A";
});
await check(`${encoding}, fatal=${fatal}: pending byte at stream EOF`, async () => {
const result = await streamResult(encoding, fatal,
[Uint8Array.of(bytes[0]), ...emptyChunks(4)]);
return fatal ? result.error instanceof TypeError && result.chunks.length === 0 :
result.error === undefined && JSON.stringify(result.chunks) === JSON.stringify(["\ufffd"]);
});
}
}
return { checks };
};
+1 -1
View File
@@ -9,7 +9,7 @@ brotlic = "0.8.2"
flate2 = { version = "1.1.9", default-features = false, features = ["zlib"] }
moli-streams = { path = "../moli-streams" }
moli-file-api = { path = "../moli-file-api" }
encoding_rs = "0.8"
encoding_rs = "0.8.42"
parking_lot = "0.12"
moli-websocket-types = { path = "../moli-types/moli-websocket-types" }
moli-indexeddb-types = { path = "../moli-types/moli-indexeddb-types" }