fix(encoding): preserve decoder and BOM state after stream errors

Preserve codec state across streaming errors, serialize BOM state only for successful output, and reset both at non-stream boundaries. Cover ISO-2022-JP recovery and Unicode BOM handling in native, Window, and Worker tests.
This commit is contained in:
ldm0
2026-11-20 22:47:53 +08:00
parent f3963c7048
commit ed64edc8f4
4 changed files with 332 additions and 38 deletions
@@ -666,6 +666,31 @@ fn text_decoder_stream_uses_encoding_rs_labels_and_streaming_decode() {
assert_eq!(settled, r#"["太平洋:false"]"#);
}
#[test]
fn text_decoder_preserves_fatal_stream_state_and_serialized_bom_state() {
let mut vm = new_storage_test_vm("https://text-decoder-fatal-recovery.test/");
vm.eval(include_str!(
"../../../../tests/fixtures/text-decoder-fatal-recovery.js"
))
.expect("fatal decoder recovery contract should load");
vm.eval("runFatalDecoderRecoveryContract().then(result => globalThis.__fatalDecoderRecoveryResults = result)")
.expect("fatal decoder recovery contract should execute");
let facts: serde_json::Value = serde_json::from_str(
&vm.eval("JSON.stringify(__fatalDecoderRecoveryResults)")
.expect("fatal decoder recovery results should serialize"),
)
.expect("fatal decoder recovery results should be valid JSON");
let checks = facts["checks"]
.as_array()
.expect("fatal decoder recovery checks");
let failures = checks
.iter()
.filter(|check| check["passed"] != true)
.collect::<Vec<_>>();
assert!(failures.is_empty(), "{failures:#?}");
assert_eq!(checks.len(), 30);
}
#[test]
fn text_decoder_preserves_legacy_sequences_across_empty_chunks() {
let mut vm = new_storage_test_vm("https://legacy-decoder-empty-chunks.test/");
@@ -1,5 +1,41 @@
use super::*;
#[tokio::test]
async fn worker_text_decoder_preserves_fatal_stream_state_and_serialized_bom_state() {
ensure_v8();
let source = [
include_str!("../../../../../tests/fixtures/text-decoder-fatal-recovery.js"),
r#"
runFatalDecoderRecoveryContract().then(
result => { postMessage(result); close(); },
error => { postMessage({ failure: String(error) }); close(); }
);
"#,
]
.join("\n");
let mut handle = spawn_worker(
source,
"https://text-decoder-fatal-recovery.test/worker.js".into(),
);
let message = timeout(TIMEOUT, handle.recv())
.await
.expect("fatal decoder recovery contract should complete");
handle.terminate_and_join();
let facts: serde_json::Value = serde_json::from_str(&expect_post_json(
message.expect("worker result channel should stay open"),
))
.expect("fatal decoder recovery results should be valid JSON");
let checks = facts["checks"]
.as_array()
.expect("fatal decoder recovery checks");
let failures = checks
.iter()
.filter(|check| check["passed"] != true)
.collect::<Vec<_>>();
assert!(failures.is_empty(), "{failures:#?}");
assert_eq!(checks.len(), 30);
}
#[tokio::test]
async fn worker_text_decoder_preserves_legacy_sequences_across_empty_chunks() {
ensure_v8();
@@ -0,0 +1,61 @@
globalThis.runFatalDecoderRecoveryContract = async function () {
const checks = [];
function check(name, operation) {
try { checks.push({ name, passed: operation() === true }); }
catch (error) { checks.push({ name, passed: false, error: String(error) }); }
}
function fails(decoder, bytes, stream) {
try { decoder.decode(Uint8Array.from(bytes), { stream }); return false; }
catch (error) { return error instanceof TypeError; }
}
for (const [encoding, bom, invalid, unit] of [
["utf-8", [0xef, 0xbb, 0xbf], [0xff], value => [value]],
["utf-16le", [0xff, 0xfe], [0, 0xdc], value => [value, 0]],
["utf-16be", [0xfe, 0xff], [0xdc, 0], value => [0, value]],
]) {
for (const ignoreBOM of [false, true]) {
check(`${encoding} first fatal call, ignoreBOM=${ignoreBOM}`, () => {
const decoder = new TextDecoder(encoding, { fatal: true, ignoreBOM });
if (!fails(decoder, [...bom, ...invalid], true)) return false;
const next = decoder.decode(Uint8Array.from([...bom, ...unit(65)]));
return next === (ignoreBOM ? "\ufeffA" : "A");
});
for (const prefix of ["empty", "ASCII", "BOM"]) {
check(`${encoding} successful ${prefix} before error, ignoreBOM=${ignoreBOM}`, () => {
const decoder = new TextDecoder(encoding, { fatal: true, ignoreBOM });
const bytes = prefix === "BOM" ? bom : prefix === "ASCII" ? unit(65) : [];
const expected = prefix === "ASCII" ? "A" : prefix === "BOM" && ignoreBOM ? "\ufeff" : "";
if (decoder.decode(Uint8Array.from(bytes), { stream: true }) !== expected) return false;
if (!fails(decoder, invalid, true)) return false;
const next = decoder.decode(Uint8Array.from([...bom, ...unit(66)]));
const expectedNext = ignoreBOM || prefix !== "empty" ? "\ufeffB" : "B";
const fresh = decoder.decode(Uint8Array.from([...bom, ...unit(67)]));
return next === expectedNext && fresh === (ignoreBOM ? "\ufeffC" : "C");
});
}
}
}
for (const [name, escape, byte, expected] of [
["Roman", 0x4a, 0x7e, "\u203e"],
["Katakana", 0x49, 0x42, "\uff82"],
]) {
const switchBytes = [0x1b, 0x28, escape];
check(`${name} recovery after streaming error`, () => {
const decoder = new TextDecoder("iso-2022-jp", { fatal: true });
return decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte) &&
fails(decoder, [...switchBytes, 0xff], true) &&
decoder.decode(Uint8Array.of(byte)) === expected;
});
check(`${name} reset after successful non-stream decode`, () => {
const decoder = new TextDecoder("iso-2022-jp", { fatal: true });
return decoder.decode(Uint8Array.from([...switchBytes, byte])) === expected &&
decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte);
});
check(`${name} reset after non-stream error`, () => {
const decoder = new TextDecoder("iso-2022-jp", { fatal: true });
return fails(decoder, [...switchBytes, 0xff], false) &&
decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte);
});
}
return { checks };
};
+210 -38
View File
@@ -73,12 +73,12 @@ struct TextCodecStoreState {
decoders: HashMap<u32, TextDecoderState>,
}
#[derive(Default)]
pub struct TextDecoderState {
encoding: Option<&'static Encoding>,
decoder: Option<Decoder>,
encoding: &'static Encoding,
decoder: Decoder,
fatal: bool,
ignore_bom: bool,
bom_seen: bool,
}
#[derive(Debug)]
@@ -203,12 +203,12 @@ fn remove_decoder_state(state: &Weak<Mutex<Option<TextCodecStoreState>>>, decode
impl TextDecoderState {
fn new(encoding: &'static Encoding, fatal: bool, ignore_bom: bool) -> Self {
let decoder = Some(new_encoding_decoder(encoding, ignore_bom));
Self {
encoding: Some(encoding),
decoder,
encoding,
decoder: encoding.new_decoder_without_bom_handling(),
fatal,
ignore_bom,
bom_seen: false,
}
}
@@ -216,35 +216,49 @@ impl TextDecoderState {
let last = !stream;
let mut output = String::new();
if self.fatal {
if self.decode_without_replacement(bytes, &mut output, last)? && last {
self.reset_decoder();
}
} else if self.decode_with_replacement(bytes, &mut output, last) && last {
let result = if self.fatal {
self.decode_without_replacement(bytes, &mut output, last)
} else {
self.decode_with_replacement(bytes, &mut output, last);
Ok(())
};
// A fatal error discards the output queue without serializing it.
if result.is_ok() {
self.serialize_output(&mut output);
}
if last {
self.reset_decoder();
}
result?;
Ok(output)
}
fn decode_with_replacement(&mut self, bytes: &[u8], output: &mut String, last: bool) -> bool {
fn serialize_output(&mut self, output: &mut String) {
let unicode = self.encoding == encoding_rs::UTF_8
|| self.encoding == encoding_rs::UTF_16LE
|| self.encoding == encoding_rs::UTF_16BE;
if unicode && !self.ignore_bom && !self.bom_seen && !output.is_empty() {
self.bom_seen = true;
if output.starts_with('\u{feff}') {
output.remove(0);
}
}
}
fn decode_with_replacement(&mut self, bytes: &[u8], output: &mut String, last: bool) {
let mut total_read = 0usize;
loop {
let input = &bytes[total_read..];
let reserve = self
.decoder
.as_ref()
.and_then(|decoder| decoder.max_utf8_buffer_length(input.len()))
.max_utf8_buffer_length(input.len())
.unwrap_or_else(|| input.len().saturating_mul(3).saturating_add(16));
output.reserve(reserve);
let (result, read, _) = self
.decoder
.as_mut()
.expect("TextDecoderState should always own a decoder")
.decode_to_string(input, output, last);
let (result, read, _) = self.decoder.decode_to_string(input, output, last);
total_read += read;
match result {
CoderResult::InputEmpty => return true,
CoderResult::InputEmpty => return,
CoderResult::OutputFull => continue,
}
}
@@ -255,27 +269,23 @@ impl TextDecoderState {
bytes: &[u8],
output: &mut String,
last: bool,
) -> Result<bool, TextDecodeError> {
) -> Result<(), TextDecodeError> {
let mut total_read = 0usize;
loop {
let input = &bytes[total_read..];
let reserve = self
.decoder
.as_ref()
.and_then(|decoder| decoder.max_utf8_buffer_length_without_replacement(input.len()))
.max_utf8_buffer_length_without_replacement(input.len())
.unwrap_or_else(|| input.len().saturating_mul(3).saturating_add(16));
output.reserve(reserve);
let (result, read) = self
.decoder
.as_mut()
.expect("TextDecoderState should always own a decoder")
.decode_to_string_without_replacement(input, output, last);
total_read += read;
match result {
DecoderResult::InputEmpty => return Ok(true),
DecoderResult::InputEmpty => return Ok(()),
DecoderResult::OutputFull => continue,
DecoderResult::Malformed(_, _) => {
self.reset_decoder();
return Err(TextDecodeError::MalformedInput);
}
}
@@ -283,17 +293,8 @@ impl TextDecoderState {
}
fn reset_decoder(&mut self) {
if let Some(encoding) = self.encoding {
self.decoder = Some(new_encoding_decoder(encoding, self.ignore_bom));
}
}
}
fn new_encoding_decoder(encoding: &'static Encoding, ignore_bom: bool) -> Decoder {
if ignore_bom {
encoding.new_decoder_without_bom_handling()
} else {
encoding.new_decoder_with_bom_removal()
self.decoder = self.encoding.new_decoder_without_bom_handling();
self.bom_seen = false;
}
}
@@ -412,4 +413,175 @@ mod tests {
}
}
}
#[test]
fn text_decoder_state_preserves_multibyte_sequences_across_empty_chunks() {
let cases: &[(&'static Encoding, &[u8], &str)] = &[
(encoding_rs::BIG5, &[0xfe, 0x40], "\u{9442}"),
(encoding_rs::SHIFT_JIS, &[0x81, 0x87], "\u{221e}"),
(encoding_rs::EUC_KR, &[0x81, 0x41], "\u{ac02}"),
];
for &(encoding, input, expected) in cases {
for fatal in [false, true] {
for split in 0..=input.len() {
let mut state = TextDecoderState::new(encoding, fatal, false);
let mut output = state.decode(&input[..split], true).unwrap();
for _ in 0..4 {
assert_eq!(state.decode(&[], true).unwrap(), "");
}
output.push_str(&state.decode(&input[split..], true).unwrap());
for _ in 0..4 {
assert_eq!(state.decode(&[], true).unwrap(), "");
}
output.push_str(&state.decode(&[], false).unwrap());
assert_eq!(
output,
expected,
"{} fatal={fatal} split={split}",
encoding.name()
);
}
}
}
}
#[test]
fn legacy_decoder_preserves_pending_bytes_when_output_is_full() {
for (encoding, input, expected) in [
(encoding_rs::BIG5, [0xfe, 0x40], "\u{9442}"),
(encoding_rs::SHIFT_JIS, [0x81, 0x87], "\u{221e}"),
(encoding_rs::EUC_KR, [0x81, 0x41], "\u{ac02}"),
] {
let mut decoder = encoding.new_decoder_without_bom_handling();
let mut output = [0; 8];
assert_eq!(
decoder.decode_to_utf8_without_replacement(&input[..1], &mut output, false),
(DecoderResult::InputEmpty, 1, 0)
);
assert_eq!(
decoder.decode_to_utf8_without_replacement(&input[1..], &mut [], false),
(DecoderResult::OutputFull, 0, 0)
);
let (result, read, written) =
decoder.decode_to_utf8_without_replacement(&input[1..], &mut output, true);
assert_eq!((result, read), (DecoderResult::InputEmpty, 1));
assert_eq!(
&output[..written],
expected.as_bytes(),
"{}",
encoding.name()
);
}
}
#[test]
fn text_decoder_state_preserves_iso2022jp_after_fatal_stream_errors() {
for (escape, byte, expected) in [(0x4a, 0x7e, "\u{203e}"), (0x49, 0x42, "\u{ff82}")] {
let mut state = TextDecoderState::new(encoding_rs::ISO_2022_JP, true, false);
assert_eq!(
state.decode(&[byte], false).unwrap(),
char::from(byte).to_string()
);
assert!(matches!(
state.decode(&[0x1b, 0x28, escape, 0xff], true),
Err(TextDecodeError::MalformedInput)
));
assert_eq!(state.decode(&[], true).unwrap(), "");
assert_eq!(state.decode(&[byte], false).unwrap(), expected);
assert_eq!(
state.decode(&[byte], false).unwrap(),
char::from(byte).to_string()
);
assert!(matches!(
state.decode(&[0x1b, 0x28, escape, 0xff], false),
Err(TextDecodeError::MalformedInput)
));
assert_eq!(
state.decode(&[byte], false).unwrap(),
char::from(byte).to_string()
);
}
}
#[test]
fn text_decoder_state_serializes_bom_only_after_successful_decode() {
let cases: &[(&'static Encoding, &[u8], &[u8], &[u8])] = &[
(encoding_rs::UTF_8, &[0xef, 0xbb, 0xbf], &[65], &[0xff]),
(encoding_rs::UTF_16LE, &[0xff, 0xfe], &[65, 0], &[0, 0xdc]),
(encoding_rs::UTF_16BE, &[0xfe, 0xff], &[0, 65], &[0xdc, 0]),
];
for &(encoding, bom, ascii, invalid) in cases {
for ignore_bom in [false, true] {
let input = [bom, ascii].concat();
for prefix in [&[][..], ascii, bom] {
let mut state = TextDecoderState::new(encoding, true, ignore_bom);
let expected = if prefix == ascii {
"A"
} else if prefix == bom && ignore_bom {
"\u{feff}"
} else {
""
};
assert_eq!(state.decode(prefix, true).unwrap(), expected);
assert!(matches!(
state.decode(invalid, true),
Err(TextDecodeError::MalformedInput)
));
let expected = if ignore_bom || !prefix.is_empty() {
"\u{feff}A"
} else {
"A"
};
assert_eq!(state.decode(&input, false).unwrap(), expected);
let fresh = if ignore_bom { "\u{feff}A" } else { "A" };
assert_eq!(state.decode(&input, false).unwrap(), fresh);
}
let mut state = TextDecoderState::new(encoding, true, ignore_bom);
assert!(matches!(
state.decode(&[bom, invalid].concat(), true),
Err(TextDecodeError::MalformedInput)
));
let expected = if ignore_bom { "\u{feff}A" } else { "A" };
assert_eq!(state.decode(&input, false).unwrap(), expected);
}
}
}
#[test]
fn text_decoder_state_preserves_bom_serialization_across_all_chunk_partitions() {
let cases: &[(&'static Encoding, &[u8], &[u8])] = &[
(encoding_rs::UTF_8, &[0xef, 0xbb, 0xbf], &[65]),
(encoding_rs::UTF_16LE, &[0xff, 0xfe], &[65, 0]),
(encoding_rs::UTF_16BE, &[0xfe, 0xff], &[0, 65]),
];
for &(encoding, bom, ascii) in cases {
let input = [bom, ascii, bom, ascii].concat();
for fatal in [false, true] {
for ignore_bom in [false, true] {
let expected = if ignore_bom {
"\u{feff}A\u{feff}A"
} else {
"A\u{feff}A"
};
for first in 0..=input.len() {
for second in first..=input.len() {
let mut state = TextDecoderState::new(encoding, fatal, ignore_bom);
let mut actual = String::new();
for chunk in [&input[..first], &input[first..second], &input[second..]]
{
actual.push_str(&state.decode(chunk, true).unwrap());
assert_eq!(state.decode(&[], true).unwrap(), "");
}
actual.push_str(&state.decode(&[], false).unwrap());
assert_eq!(
actual,
expected,
"{} cuts={first}/{second}",
encoding.name()
);
}
}
}
}
}
}
}