From ed64edc8f490653b1f086da5bc7b05edcf02e5cb Mon Sep 17 00:00:00 2001 From: ldm0 Date: Sat, 17 Oct 2026 20:47:23 +0800 Subject: [PATCH] fix(encoding): preserve decoder and BOM state after stream errors Preserve codec state across streaming errors, serialize BOM state only for successful output, and reset both at non-stream boundaries. Cover ISO-2022-JP recovery and Unicode BOM handling in native, Window, and Worker tests. --- .../webidl_and_transform_streams.rs | 25 ++ .../background_apis_and_worker_events.rs | 36 +++ .../fixtures/text-decoder-fatal-recovery.js | 61 +++++ moli-webapi/src/text_codec.rs | 248 +++++++++++++++--- 4 files changed, 332 insertions(+), 38 deletions(-) create mode 100644 moli-renderer-v8/tests/fixtures/text-decoder-fatal-recovery.js diff --git a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs index ae39733d1c6..11ea3079598 100644 --- a/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs +++ b/moli-renderer-v8/src/script_vm/tests/webidl_fetch/webidl_and_transform_streams.rs @@ -666,6 +666,31 @@ fn text_decoder_stream_uses_encoding_rs_labels_and_streaming_decode() { assert_eq!(settled, r#"["太平洋:false"]"#); } +#[test] +fn text_decoder_preserves_fatal_stream_state_and_serialized_bom_state() { + let mut vm = new_storage_test_vm("https://text-decoder-fatal-recovery.test/"); + vm.eval(include_str!( + "../../../../tests/fixtures/text-decoder-fatal-recovery.js" + )) + .expect("fatal decoder recovery contract should load"); + vm.eval("runFatalDecoderRecoveryContract().then(result => globalThis.__fatalDecoderRecoveryResults = result)") + .expect("fatal decoder recovery contract should execute"); + let facts: serde_json::Value = serde_json::from_str( + &vm.eval("JSON.stringify(__fatalDecoderRecoveryResults)") + .expect("fatal decoder recovery results should serialize"), + ) + .expect("fatal decoder recovery results should be valid JSON"); + let checks = facts["checks"] + .as_array() + .expect("fatal decoder recovery checks"); + let failures = checks + .iter() + .filter(|check| check["passed"] != true) + .collect::>(); + assert!(failures.is_empty(), "{failures:#?}"); + assert_eq!(checks.len(), 30); +} + #[test] fn text_decoder_preserves_legacy_sequences_across_empty_chunks() { let mut vm = new_storage_test_vm("https://legacy-decoder-empty-chunks.test/"); diff --git a/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs b/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs index 91f26d52341..ffaa5dfd751 100644 --- a/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs +++ b/moli-renderer-v8/src/worker/thread/tests/lifecycle/background_apis_and_worker_events.rs @@ -1,5 +1,41 @@ use super::*; +#[tokio::test] +async fn worker_text_decoder_preserves_fatal_stream_state_and_serialized_bom_state() { + ensure_v8(); + let source = [ + include_str!("../../../../../tests/fixtures/text-decoder-fatal-recovery.js"), + r#" +runFatalDecoderRecoveryContract().then( + result => { postMessage(result); close(); }, + error => { postMessage({ failure: String(error) }); close(); } +); +"#, + ] + .join("\n"); + let mut handle = spawn_worker( + source, + "https://text-decoder-fatal-recovery.test/worker.js".into(), + ); + let message = timeout(TIMEOUT, handle.recv()) + .await + .expect("fatal decoder recovery contract should complete"); + handle.terminate_and_join(); + let facts: serde_json::Value = serde_json::from_str(&expect_post_json( + message.expect("worker result channel should stay open"), + )) + .expect("fatal decoder recovery results should be valid JSON"); + let checks = facts["checks"] + .as_array() + .expect("fatal decoder recovery checks"); + let failures = checks + .iter() + .filter(|check| check["passed"] != true) + .collect::>(); + assert!(failures.is_empty(), "{failures:#?}"); + assert_eq!(checks.len(), 30); +} + #[tokio::test] async fn worker_text_decoder_preserves_legacy_sequences_across_empty_chunks() { ensure_v8(); diff --git a/moli-renderer-v8/tests/fixtures/text-decoder-fatal-recovery.js b/moli-renderer-v8/tests/fixtures/text-decoder-fatal-recovery.js new file mode 100644 index 00000000000..bc8512faf40 --- /dev/null +++ b/moli-renderer-v8/tests/fixtures/text-decoder-fatal-recovery.js @@ -0,0 +1,61 @@ +globalThis.runFatalDecoderRecoveryContract = async function () { + const checks = []; + function check(name, operation) { + try { checks.push({ name, passed: operation() === true }); } + catch (error) { checks.push({ name, passed: false, error: String(error) }); } + } + function fails(decoder, bytes, stream) { + try { decoder.decode(Uint8Array.from(bytes), { stream }); return false; } + catch (error) { return error instanceof TypeError; } + } + for (const [encoding, bom, invalid, unit] of [ + ["utf-8", [0xef, 0xbb, 0xbf], [0xff], value => [value]], + ["utf-16le", [0xff, 0xfe], [0, 0xdc], value => [value, 0]], + ["utf-16be", [0xfe, 0xff], [0xdc, 0], value => [0, value]], + ]) { + for (const ignoreBOM of [false, true]) { + check(`${encoding} first fatal call, ignoreBOM=${ignoreBOM}`, () => { + const decoder = new TextDecoder(encoding, { fatal: true, ignoreBOM }); + if (!fails(decoder, [...bom, ...invalid], true)) return false; + const next = decoder.decode(Uint8Array.from([...bom, ...unit(65)])); + return next === (ignoreBOM ? "\ufeffA" : "A"); + }); + for (const prefix of ["empty", "ASCII", "BOM"]) { + check(`${encoding} successful ${prefix} before error, ignoreBOM=${ignoreBOM}`, () => { + const decoder = new TextDecoder(encoding, { fatal: true, ignoreBOM }); + const bytes = prefix === "BOM" ? bom : prefix === "ASCII" ? unit(65) : []; + const expected = prefix === "ASCII" ? "A" : prefix === "BOM" && ignoreBOM ? "\ufeff" : ""; + if (decoder.decode(Uint8Array.from(bytes), { stream: true }) !== expected) return false; + if (!fails(decoder, invalid, true)) return false; + const next = decoder.decode(Uint8Array.from([...bom, ...unit(66)])); + const expectedNext = ignoreBOM || prefix !== "empty" ? "\ufeffB" : "B"; + const fresh = decoder.decode(Uint8Array.from([...bom, ...unit(67)])); + return next === expectedNext && fresh === (ignoreBOM ? "\ufeffC" : "C"); + }); + } + } + } + for (const [name, escape, byte, expected] of [ + ["Roman", 0x4a, 0x7e, "\u203e"], + ["Katakana", 0x49, 0x42, "\uff82"], + ]) { + const switchBytes = [0x1b, 0x28, escape]; + check(`${name} recovery after streaming error`, () => { + const decoder = new TextDecoder("iso-2022-jp", { fatal: true }); + return decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte) && + fails(decoder, [...switchBytes, 0xff], true) && + decoder.decode(Uint8Array.of(byte)) === expected; + }); + check(`${name} reset after successful non-stream decode`, () => { + const decoder = new TextDecoder("iso-2022-jp", { fatal: true }); + return decoder.decode(Uint8Array.from([...switchBytes, byte])) === expected && + decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte); + }); + check(`${name} reset after non-stream error`, () => { + const decoder = new TextDecoder("iso-2022-jp", { fatal: true }); + return fails(decoder, [...switchBytes, 0xff], false) && + decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte); + }); + } + return { checks }; +}; diff --git a/moli-webapi/src/text_codec.rs b/moli-webapi/src/text_codec.rs index cd0d18f7fc9..56e76c6435b 100644 --- a/moli-webapi/src/text_codec.rs +++ b/moli-webapi/src/text_codec.rs @@ -73,12 +73,12 @@ struct TextCodecStoreState { decoders: HashMap, } -#[derive(Default)] pub struct TextDecoderState { - encoding: Option<&'static Encoding>, - decoder: Option, + encoding: &'static Encoding, + decoder: Decoder, fatal: bool, ignore_bom: bool, + bom_seen: bool, } #[derive(Debug)] @@ -203,12 +203,12 @@ fn remove_decoder_state(state: &Weak>>, decode impl TextDecoderState { fn new(encoding: &'static Encoding, fatal: bool, ignore_bom: bool) -> Self { - let decoder = Some(new_encoding_decoder(encoding, ignore_bom)); Self { - encoding: Some(encoding), - decoder, + encoding, + decoder: encoding.new_decoder_without_bom_handling(), fatal, ignore_bom, + bom_seen: false, } } @@ -216,35 +216,49 @@ impl TextDecoderState { let last = !stream; let mut output = String::new(); - if self.fatal { - if self.decode_without_replacement(bytes, &mut output, last)? && last { - self.reset_decoder(); - } - } else if self.decode_with_replacement(bytes, &mut output, last) && last { + let result = if self.fatal { + self.decode_without_replacement(bytes, &mut output, last) + } else { + self.decode_with_replacement(bytes, &mut output, last); + Ok(()) + }; + // A fatal error discards the output queue without serializing it. + if result.is_ok() { + self.serialize_output(&mut output); + } + if last { self.reset_decoder(); } + result?; Ok(output) } - fn decode_with_replacement(&mut self, bytes: &[u8], output: &mut String, last: bool) -> bool { + fn serialize_output(&mut self, output: &mut String) { + let unicode = self.encoding == encoding_rs::UTF_8 + || self.encoding == encoding_rs::UTF_16LE + || self.encoding == encoding_rs::UTF_16BE; + if unicode && !self.ignore_bom && !self.bom_seen && !output.is_empty() { + self.bom_seen = true; + if output.starts_with('\u{feff}') { + output.remove(0); + } + } + } + + fn decode_with_replacement(&mut self, bytes: &[u8], output: &mut String, last: bool) { let mut total_read = 0usize; loop { let input = &bytes[total_read..]; let reserve = self .decoder - .as_ref() - .and_then(|decoder| decoder.max_utf8_buffer_length(input.len())) + .max_utf8_buffer_length(input.len()) .unwrap_or_else(|| input.len().saturating_mul(3).saturating_add(16)); output.reserve(reserve); - let (result, read, _) = self - .decoder - .as_mut() - .expect("TextDecoderState should always own a decoder") - .decode_to_string(input, output, last); + let (result, read, _) = self.decoder.decode_to_string(input, output, last); total_read += read; match result { - CoderResult::InputEmpty => return true, + CoderResult::InputEmpty => return, CoderResult::OutputFull => continue, } } @@ -255,27 +269,23 @@ impl TextDecoderState { bytes: &[u8], output: &mut String, last: bool, - ) -> Result { + ) -> Result<(), TextDecodeError> { let mut total_read = 0usize; loop { let input = &bytes[total_read..]; let reserve = self .decoder - .as_ref() - .and_then(|decoder| decoder.max_utf8_buffer_length_without_replacement(input.len())) + .max_utf8_buffer_length_without_replacement(input.len()) .unwrap_or_else(|| input.len().saturating_mul(3).saturating_add(16)); output.reserve(reserve); let (result, read) = self .decoder - .as_mut() - .expect("TextDecoderState should always own a decoder") .decode_to_string_without_replacement(input, output, last); total_read += read; match result { - DecoderResult::InputEmpty => return Ok(true), + DecoderResult::InputEmpty => return Ok(()), DecoderResult::OutputFull => continue, DecoderResult::Malformed(_, _) => { - self.reset_decoder(); return Err(TextDecodeError::MalformedInput); } } @@ -283,17 +293,8 @@ impl TextDecoderState { } fn reset_decoder(&mut self) { - if let Some(encoding) = self.encoding { - self.decoder = Some(new_encoding_decoder(encoding, self.ignore_bom)); - } - } -} - -fn new_encoding_decoder(encoding: &'static Encoding, ignore_bom: bool) -> Decoder { - if ignore_bom { - encoding.new_decoder_without_bom_handling() - } else { - encoding.new_decoder_with_bom_removal() + self.decoder = self.encoding.new_decoder_without_bom_handling(); + self.bom_seen = false; } } @@ -412,4 +413,175 @@ mod tests { } } } + #[test] + fn text_decoder_state_preserves_multibyte_sequences_across_empty_chunks() { + let cases: &[(&'static Encoding, &[u8], &str)] = &[ + (encoding_rs::BIG5, &[0xfe, 0x40], "\u{9442}"), + (encoding_rs::SHIFT_JIS, &[0x81, 0x87], "\u{221e}"), + (encoding_rs::EUC_KR, &[0x81, 0x41], "\u{ac02}"), + ]; + for &(encoding, input, expected) in cases { + for fatal in [false, true] { + for split in 0..=input.len() { + let mut state = TextDecoderState::new(encoding, fatal, false); + let mut output = state.decode(&input[..split], true).unwrap(); + for _ in 0..4 { + assert_eq!(state.decode(&[], true).unwrap(), ""); + } + output.push_str(&state.decode(&input[split..], true).unwrap()); + for _ in 0..4 { + assert_eq!(state.decode(&[], true).unwrap(), ""); + } + output.push_str(&state.decode(&[], false).unwrap()); + assert_eq!( + output, + expected, + "{} fatal={fatal} split={split}", + encoding.name() + ); + } + } + } + } + + #[test] + fn legacy_decoder_preserves_pending_bytes_when_output_is_full() { + for (encoding, input, expected) in [ + (encoding_rs::BIG5, [0xfe, 0x40], "\u{9442}"), + (encoding_rs::SHIFT_JIS, [0x81, 0x87], "\u{221e}"), + (encoding_rs::EUC_KR, [0x81, 0x41], "\u{ac02}"), + ] { + let mut decoder = encoding.new_decoder_without_bom_handling(); + let mut output = [0; 8]; + assert_eq!( + decoder.decode_to_utf8_without_replacement(&input[..1], &mut output, false), + (DecoderResult::InputEmpty, 1, 0) + ); + assert_eq!( + decoder.decode_to_utf8_without_replacement(&input[1..], &mut [], false), + (DecoderResult::OutputFull, 0, 0) + ); + let (result, read, written) = + decoder.decode_to_utf8_without_replacement(&input[1..], &mut output, true); + assert_eq!((result, read), (DecoderResult::InputEmpty, 1)); + assert_eq!( + &output[..written], + expected.as_bytes(), + "{}", + encoding.name() + ); + } + } + + #[test] + fn text_decoder_state_preserves_iso2022jp_after_fatal_stream_errors() { + for (escape, byte, expected) in [(0x4a, 0x7e, "\u{203e}"), (0x49, 0x42, "\u{ff82}")] { + let mut state = TextDecoderState::new(encoding_rs::ISO_2022_JP, true, false); + assert_eq!( + state.decode(&[byte], false).unwrap(), + char::from(byte).to_string() + ); + assert!(matches!( + state.decode(&[0x1b, 0x28, escape, 0xff], true), + Err(TextDecodeError::MalformedInput) + )); + assert_eq!(state.decode(&[], true).unwrap(), ""); + assert_eq!(state.decode(&[byte], false).unwrap(), expected); + assert_eq!( + state.decode(&[byte], false).unwrap(), + char::from(byte).to_string() + ); + assert!(matches!( + state.decode(&[0x1b, 0x28, escape, 0xff], false), + Err(TextDecodeError::MalformedInput) + )); + assert_eq!( + state.decode(&[byte], false).unwrap(), + char::from(byte).to_string() + ); + } + } + + #[test] + fn text_decoder_state_serializes_bom_only_after_successful_decode() { + let cases: &[(&'static Encoding, &[u8], &[u8], &[u8])] = &[ + (encoding_rs::UTF_8, &[0xef, 0xbb, 0xbf], &[65], &[0xff]), + (encoding_rs::UTF_16LE, &[0xff, 0xfe], &[65, 0], &[0, 0xdc]), + (encoding_rs::UTF_16BE, &[0xfe, 0xff], &[0, 65], &[0xdc, 0]), + ]; + for &(encoding, bom, ascii, invalid) in cases { + for ignore_bom in [false, true] { + let input = [bom, ascii].concat(); + for prefix in [&[][..], ascii, bom] { + let mut state = TextDecoderState::new(encoding, true, ignore_bom); + let expected = if prefix == ascii { + "A" + } else if prefix == bom && ignore_bom { + "\u{feff}" + } else { + "" + }; + assert_eq!(state.decode(prefix, true).unwrap(), expected); + assert!(matches!( + state.decode(invalid, true), + Err(TextDecodeError::MalformedInput) + )); + let expected = if ignore_bom || !prefix.is_empty() { + "\u{feff}A" + } else { + "A" + }; + assert_eq!(state.decode(&input, false).unwrap(), expected); + let fresh = if ignore_bom { "\u{feff}A" } else { "A" }; + assert_eq!(state.decode(&input, false).unwrap(), fresh); + } + let mut state = TextDecoderState::new(encoding, true, ignore_bom); + assert!(matches!( + state.decode(&[bom, invalid].concat(), true), + Err(TextDecodeError::MalformedInput) + )); + let expected = if ignore_bom { "\u{feff}A" } else { "A" }; + assert_eq!(state.decode(&input, false).unwrap(), expected); + } + } + } + + #[test] + fn text_decoder_state_preserves_bom_serialization_across_all_chunk_partitions() { + let cases: &[(&'static Encoding, &[u8], &[u8])] = &[ + (encoding_rs::UTF_8, &[0xef, 0xbb, 0xbf], &[65]), + (encoding_rs::UTF_16LE, &[0xff, 0xfe], &[65, 0]), + (encoding_rs::UTF_16BE, &[0xfe, 0xff], &[0, 65]), + ]; + for &(encoding, bom, ascii) in cases { + let input = [bom, ascii, bom, ascii].concat(); + for fatal in [false, true] { + for ignore_bom in [false, true] { + let expected = if ignore_bom { + "\u{feff}A\u{feff}A" + } else { + "A\u{feff}A" + }; + for first in 0..=input.len() { + for second in first..=input.len() { + let mut state = TextDecoderState::new(encoding, fatal, ignore_bom); + let mut actual = String::new(); + for chunk in [&input[..first], &input[first..second], &input[second..]] + { + actual.push_str(&state.decode(chunk, true).unwrap()); + assert_eq!(state.decode(&[], true).unwrap(), ""); + } + actual.push_str(&state.decode(&[], false).unwrap()); + assert_eq!( + actual, + expected, + "{} cuts={first}/{second}", + encoding.name() + ); + } + } + } + } + } + } }