mirror of
https://github.com/lexmount/moli.git
synced 2026-11-21 16:03:10 +00:00
fix(encoding): preserve legacy decoder state across empty chunks
This commit is contained in:
Generated
+25
-2
@@ -783,6 +783,12 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core_detect"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
|
||||
|
||||
[[package]]
|
||||
name = "core_maths"
|
||||
version = "0.1.1"
|
||||
@@ -1345,11 +1351,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.35"
|
||||
version = "0.8.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"core_detect",
|
||||
"multiversion_no_op",
|
||||
"rustversion",
|
||||
"scopeguard",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4028,6 +4039,12 @@ dependencies = [
|
||||
"pxfm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multiversion_no_op"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
|
||||
|
||||
[[package]]
|
||||
name = "netscape-cookie-file-parser"
|
||||
version = "0.3.0"
|
||||
@@ -5511,6 +5528,12 @@ version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
|
||||
|
||||
[[package]]
|
||||
name = "simplecss"
|
||||
version = "0.2.2"
|
||||
|
||||
@@ -30,7 +30,7 @@ crossbeam-channel = "0.5.15"
|
||||
cssparser = "0.38"
|
||||
data-url = "0.3.2"
|
||||
dom = { package = "stylo_dom", version = "0.22" }
|
||||
encoding_rs = "0.8"
|
||||
encoding_rs = "0.8.42"
|
||||
euclid = "0.22"
|
||||
kurbo = "0.13.1"
|
||||
futures-util = "0.3.31"
|
||||
|
||||
@@ -666,6 +666,29 @@ fn text_decoder_stream_uses_encoding_rs_labels_and_streaming_decode() {
|
||||
assert_eq!(settled, r#"["太平洋:false"]"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_decoder_preserves_legacy_sequences_across_empty_chunks() {
|
||||
let mut vm = new_storage_test_vm("https://legacy-decoder-empty-chunks.test/");
|
||||
vm.eval(include_str!(
|
||||
"../../../../tests/fixtures/legacy-decoder-empty-chunks.js"
|
||||
))
|
||||
.expect("legacy decoder contract should load");
|
||||
vm.eval("runLegacyDecoderEmptyChunkContract().then(result => globalThis.__legacyDecoderEmptyChunkResults = result)")
|
||||
.expect("legacy decoder contract should execute");
|
||||
let facts: serde_json::Value = serde_json::from_str(
|
||||
&vm.eval("JSON.stringify(__legacyDecoderEmptyChunkResults)")
|
||||
.expect("legacy decoder results should serialize"),
|
||||
)
|
||||
.expect("legacy decoder results should be valid JSON");
|
||||
let checks = facts["checks"].as_array().expect("legacy decoder checks");
|
||||
let failures = checks
|
||||
.iter()
|
||||
.filter(|check| check["passed"] != true)
|
||||
.collect::<Vec<_>>();
|
||||
assert!(failures.is_empty(), "{failures:#?}");
|
||||
assert_eq!(checks.len(), 54);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_encoder_stream_preserves_utf16_chunk_boundaries() {
|
||||
let mut vm = new_storage_test_vm("https://text-encoder-boundaries.test/");
|
||||
|
||||
+33
@@ -1,5 +1,38 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_text_decoder_preserves_legacy_sequences_across_empty_chunks() {
|
||||
ensure_v8();
|
||||
let source = [
|
||||
include_str!("../../../../../tests/fixtures/legacy-decoder-empty-chunks.js"),
|
||||
r#"
|
||||
runLegacyDecoderEmptyChunkContract().then(
|
||||
result => { postMessage(result); close(); },
|
||||
error => { postMessage({ failure: String(error) }); close(); }
|
||||
);
|
||||
"#,
|
||||
]
|
||||
.join("\n");
|
||||
let mut handle = spawn_worker(
|
||||
source,
|
||||
"https://legacy-decoder-empty-chunks.test/worker.js".into(),
|
||||
);
|
||||
let message = timeout(TIMEOUT, handle.recv())
|
||||
.await
|
||||
.expect("legacy decoder contract should complete")
|
||||
.expect("worker result channel should stay open");
|
||||
handle.terminate_and_join();
|
||||
let facts: serde_json::Value = serde_json::from_str(&expect_post_json(message))
|
||||
.expect("legacy decoder results should be valid JSON");
|
||||
let checks = facts["checks"].as_array().expect("legacy decoder checks");
|
||||
let failures = checks
|
||||
.iter()
|
||||
.filter(|check| check["passed"] != true)
|
||||
.collect::<Vec<_>>();
|
||||
assert!(failures.is_empty(), "{failures:#?}");
|
||||
assert_eq!(checks.len(), 54);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_text_encoder_stream_preserves_utf16_chunk_boundaries() {
|
||||
ensure_v8();
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
globalThis.runLegacyDecoderEmptyChunkContract = async function () {
|
||||
const checks = [];
|
||||
async function check(name, operation) {
|
||||
globalThis.__legacyDecoderEmptyChunkProgress = name;
|
||||
try { checks.push({ name, passed: await operation() === true }); }
|
||||
catch (error) { checks.push({ name, passed: false, error: String(error) }); }
|
||||
}
|
||||
const vectors = [
|
||||
["big5", [0xfe, 0x40], "\u9442"],
|
||||
["shift_jis", [0x81, 0x87], "\u221e"],
|
||||
["euc-kr", [0x81, 0x41], "\uac02"],
|
||||
];
|
||||
const emptyChunks = count => Array.from({ length: count }, () => new Uint8Array());
|
||||
function chunks(bytes, emptyCount) {
|
||||
return [new Uint8Array(), Uint8Array.of(bytes[0]), ...emptyChunks(emptyCount),
|
||||
Uint8Array.of(bytes[1]), new Uint8Array()];
|
||||
}
|
||||
async function streamResult(encoding, fatal, input) {
|
||||
const source = new ReadableStream({ start(controller) {
|
||||
for (const chunk of input) controller.enqueue(chunk);
|
||||
controller.close();
|
||||
} });
|
||||
const reader = source.pipeThrough(new TextDecoderStream(encoding, { fatal })).getReader();
|
||||
const result = { chunks: [], error: undefined };
|
||||
try {
|
||||
while (true) {
|
||||
const item = await reader.read();
|
||||
if (item.done) break;
|
||||
result.chunks.push(item.value);
|
||||
}
|
||||
} catch (error) { result.error = error; }
|
||||
return result;
|
||||
}
|
||||
for (const [encoding, bytes, expected] of vectors) {
|
||||
for (const fatal of [false, true]) {
|
||||
await check(`${encoding}, fatal=${fatal}: whole input`, () =>
|
||||
new TextDecoder(encoding, { fatal }).decode(Uint8Array.from(bytes)) === expected);
|
||||
for (const count of [0, 1, 4]) {
|
||||
await check(`${encoding}, fatal=${fatal}: decode across ${count} empty chunks`, () => {
|
||||
const decoder = new TextDecoder(encoding, { fatal });
|
||||
let output = "";
|
||||
for (const chunk of chunks(bytes, count)) output += decoder.decode(chunk, { stream: true });
|
||||
output += decoder.decode();
|
||||
return output === expected;
|
||||
});
|
||||
await check(`${encoding}, fatal=${fatal}: stream across ${count} empty chunks`, async () => {
|
||||
const result = await streamResult(encoding, fatal, chunks(bytes, count));
|
||||
return result.error === undefined && result.chunks.every(chunk => chunk.length > 0) &&
|
||||
result.chunks.join("") === expected;
|
||||
});
|
||||
}
|
||||
await check(`${encoding}, fatal=${fatal}: pending byte at decode EOF`, () => {
|
||||
const decoder = new TextDecoder(encoding, { fatal });
|
||||
if (decoder.decode(Uint8Array.of(bytes[0]), { stream: true }) !== "") return false;
|
||||
for (const chunk of emptyChunks(4)) {
|
||||
if (decoder.decode(chunk, { stream: true }) !== "") return false;
|
||||
}
|
||||
let replaced = false, failed = false;
|
||||
try { replaced = decoder.decode() === "\ufffd"; }
|
||||
catch (error) { failed = error instanceof TypeError; }
|
||||
return (fatal ? failed : replaced) && decoder.decode(Uint8Array.of(65)) === "A";
|
||||
});
|
||||
await check(`${encoding}, fatal=${fatal}: pending byte at stream EOF`, async () => {
|
||||
const result = await streamResult(encoding, fatal,
|
||||
[Uint8Array.of(bytes[0]), ...emptyChunks(4)]);
|
||||
return fatal ? result.error instanceof TypeError && result.chunks.length === 0 :
|
||||
result.error === undefined && JSON.stringify(result.chunks) === JSON.stringify(["\ufffd"]);
|
||||
});
|
||||
}
|
||||
}
|
||||
return { checks };
|
||||
};
|
||||
@@ -9,7 +9,7 @@ brotlic = "0.8.2"
|
||||
flate2 = { version = "1.1.9", default-features = false, features = ["zlib"] }
|
||||
moli-streams = { path = "../moli-streams" }
|
||||
moli-file-api = { path = "../moli-file-api" }
|
||||
encoding_rs = "0.8"
|
||||
encoding_rs = "0.8.42"
|
||||
parking_lot = "0.12"
|
||||
moli-websocket-types = { path = "../moli-types/moli-websocket-types" }
|
||||
moli-indexeddb-types = { path = "../moli-types/moli-indexeddb-types" }
|
||||
|
||||
Reference in New Issue
Block a user