mirror of
https://github.com/lexmount/moli.git
synced 2026-11-21 00:03:15 +00:00
fix(encoding): preserve decoder and BOM state after stream errors
Preserve codec state across streaming errors, serialize BOM state only for successful output, and reset both at non-stream boundaries. Cover ISO-2022-JP recovery and Unicode BOM handling in native, Window, and Worker tests.
This commit is contained in:
@@ -666,6 +666,31 @@ fn text_decoder_stream_uses_encoding_rs_labels_and_streaming_decode() {
|
||||
assert_eq!(settled, r#"["太平洋:false"]"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_decoder_preserves_fatal_stream_state_and_serialized_bom_state() {
|
||||
let mut vm = new_storage_test_vm("https://text-decoder-fatal-recovery.test/");
|
||||
vm.eval(include_str!(
|
||||
"../../../../tests/fixtures/text-decoder-fatal-recovery.js"
|
||||
))
|
||||
.expect("fatal decoder recovery contract should load");
|
||||
vm.eval("runFatalDecoderRecoveryContract().then(result => globalThis.__fatalDecoderRecoveryResults = result)")
|
||||
.expect("fatal decoder recovery contract should execute");
|
||||
let facts: serde_json::Value = serde_json::from_str(
|
||||
&vm.eval("JSON.stringify(__fatalDecoderRecoveryResults)")
|
||||
.expect("fatal decoder recovery results should serialize"),
|
||||
)
|
||||
.expect("fatal decoder recovery results should be valid JSON");
|
||||
let checks = facts["checks"]
|
||||
.as_array()
|
||||
.expect("fatal decoder recovery checks");
|
||||
let failures = checks
|
||||
.iter()
|
||||
.filter(|check| check["passed"] != true)
|
||||
.collect::<Vec<_>>();
|
||||
assert!(failures.is_empty(), "{failures:#?}");
|
||||
assert_eq!(checks.len(), 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_decoder_preserves_legacy_sequences_across_empty_chunks() {
|
||||
let mut vm = new_storage_test_vm("https://legacy-decoder-empty-chunks.test/");
|
||||
|
||||
+36
@@ -1,5 +1,41 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_text_decoder_preserves_fatal_stream_state_and_serialized_bom_state() {
|
||||
ensure_v8();
|
||||
let source = [
|
||||
include_str!("../../../../../tests/fixtures/text-decoder-fatal-recovery.js"),
|
||||
r#"
|
||||
runFatalDecoderRecoveryContract().then(
|
||||
result => { postMessage(result); close(); },
|
||||
error => { postMessage({ failure: String(error) }); close(); }
|
||||
);
|
||||
"#,
|
||||
]
|
||||
.join("\n");
|
||||
let mut handle = spawn_worker(
|
||||
source,
|
||||
"https://text-decoder-fatal-recovery.test/worker.js".into(),
|
||||
);
|
||||
let message = timeout(TIMEOUT, handle.recv())
|
||||
.await
|
||||
.expect("fatal decoder recovery contract should complete");
|
||||
handle.terminate_and_join();
|
||||
let facts: serde_json::Value = serde_json::from_str(&expect_post_json(
|
||||
message.expect("worker result channel should stay open"),
|
||||
))
|
||||
.expect("fatal decoder recovery results should be valid JSON");
|
||||
let checks = facts["checks"]
|
||||
.as_array()
|
||||
.expect("fatal decoder recovery checks");
|
||||
let failures = checks
|
||||
.iter()
|
||||
.filter(|check| check["passed"] != true)
|
||||
.collect::<Vec<_>>();
|
||||
assert!(failures.is_empty(), "{failures:#?}");
|
||||
assert_eq!(checks.len(), 30);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_text_decoder_preserves_legacy_sequences_across_empty_chunks() {
|
||||
ensure_v8();
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
globalThis.runFatalDecoderRecoveryContract = async function () {
|
||||
const checks = [];
|
||||
function check(name, operation) {
|
||||
try { checks.push({ name, passed: operation() === true }); }
|
||||
catch (error) { checks.push({ name, passed: false, error: String(error) }); }
|
||||
}
|
||||
function fails(decoder, bytes, stream) {
|
||||
try { decoder.decode(Uint8Array.from(bytes), { stream }); return false; }
|
||||
catch (error) { return error instanceof TypeError; }
|
||||
}
|
||||
for (const [encoding, bom, invalid, unit] of [
|
||||
["utf-8", [0xef, 0xbb, 0xbf], [0xff], value => [value]],
|
||||
["utf-16le", [0xff, 0xfe], [0, 0xdc], value => [value, 0]],
|
||||
["utf-16be", [0xfe, 0xff], [0xdc, 0], value => [0, value]],
|
||||
]) {
|
||||
for (const ignoreBOM of [false, true]) {
|
||||
check(`${encoding} first fatal call, ignoreBOM=${ignoreBOM}`, () => {
|
||||
const decoder = new TextDecoder(encoding, { fatal: true, ignoreBOM });
|
||||
if (!fails(decoder, [...bom, ...invalid], true)) return false;
|
||||
const next = decoder.decode(Uint8Array.from([...bom, ...unit(65)]));
|
||||
return next === (ignoreBOM ? "\ufeffA" : "A");
|
||||
});
|
||||
for (const prefix of ["empty", "ASCII", "BOM"]) {
|
||||
check(`${encoding} successful ${prefix} before error, ignoreBOM=${ignoreBOM}`, () => {
|
||||
const decoder = new TextDecoder(encoding, { fatal: true, ignoreBOM });
|
||||
const bytes = prefix === "BOM" ? bom : prefix === "ASCII" ? unit(65) : [];
|
||||
const expected = prefix === "ASCII" ? "A" : prefix === "BOM" && ignoreBOM ? "\ufeff" : "";
|
||||
if (decoder.decode(Uint8Array.from(bytes), { stream: true }) !== expected) return false;
|
||||
if (!fails(decoder, invalid, true)) return false;
|
||||
const next = decoder.decode(Uint8Array.from([...bom, ...unit(66)]));
|
||||
const expectedNext = ignoreBOM || prefix !== "empty" ? "\ufeffB" : "B";
|
||||
const fresh = decoder.decode(Uint8Array.from([...bom, ...unit(67)]));
|
||||
return next === expectedNext && fresh === (ignoreBOM ? "\ufeffC" : "C");
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const [name, escape, byte, expected] of [
|
||||
["Roman", 0x4a, 0x7e, "\u203e"],
|
||||
["Katakana", 0x49, 0x42, "\uff82"],
|
||||
]) {
|
||||
const switchBytes = [0x1b, 0x28, escape];
|
||||
check(`${name} recovery after streaming error`, () => {
|
||||
const decoder = new TextDecoder("iso-2022-jp", { fatal: true });
|
||||
return decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte) &&
|
||||
fails(decoder, [...switchBytes, 0xff], true) &&
|
||||
decoder.decode(Uint8Array.of(byte)) === expected;
|
||||
});
|
||||
check(`${name} reset after successful non-stream decode`, () => {
|
||||
const decoder = new TextDecoder("iso-2022-jp", { fatal: true });
|
||||
return decoder.decode(Uint8Array.from([...switchBytes, byte])) === expected &&
|
||||
decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte);
|
||||
});
|
||||
check(`${name} reset after non-stream error`, () => {
|
||||
const decoder = new TextDecoder("iso-2022-jp", { fatal: true });
|
||||
return fails(decoder, [...switchBytes, 0xff], false) &&
|
||||
decoder.decode(Uint8Array.of(byte)) === String.fromCharCode(byte);
|
||||
});
|
||||
}
|
||||
return { checks };
|
||||
};
|
||||
+210
-38
@@ -73,12 +73,12 @@ struct TextCodecStoreState {
|
||||
decoders: HashMap<u32, TextDecoderState>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct TextDecoderState {
|
||||
encoding: Option<&'static Encoding>,
|
||||
decoder: Option<Decoder>,
|
||||
encoding: &'static Encoding,
|
||||
decoder: Decoder,
|
||||
fatal: bool,
|
||||
ignore_bom: bool,
|
||||
bom_seen: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -203,12 +203,12 @@ fn remove_decoder_state(state: &Weak<Mutex<Option<TextCodecStoreState>>>, decode
|
||||
|
||||
impl TextDecoderState {
|
||||
fn new(encoding: &'static Encoding, fatal: bool, ignore_bom: bool) -> Self {
|
||||
let decoder = Some(new_encoding_decoder(encoding, ignore_bom));
|
||||
Self {
|
||||
encoding: Some(encoding),
|
||||
decoder,
|
||||
encoding,
|
||||
decoder: encoding.new_decoder_without_bom_handling(),
|
||||
fatal,
|
||||
ignore_bom,
|
||||
bom_seen: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,35 +216,49 @@ impl TextDecoderState {
|
||||
let last = !stream;
|
||||
let mut output = String::new();
|
||||
|
||||
if self.fatal {
|
||||
if self.decode_without_replacement(bytes, &mut output, last)? && last {
|
||||
self.reset_decoder();
|
||||
}
|
||||
} else if self.decode_with_replacement(bytes, &mut output, last) && last {
|
||||
let result = if self.fatal {
|
||||
self.decode_without_replacement(bytes, &mut output, last)
|
||||
} else {
|
||||
self.decode_with_replacement(bytes, &mut output, last);
|
||||
Ok(())
|
||||
};
|
||||
// A fatal error discards the output queue without serializing it.
|
||||
if result.is_ok() {
|
||||
self.serialize_output(&mut output);
|
||||
}
|
||||
if last {
|
||||
self.reset_decoder();
|
||||
}
|
||||
|
||||
result?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn decode_with_replacement(&mut self, bytes: &[u8], output: &mut String, last: bool) -> bool {
|
||||
fn serialize_output(&mut self, output: &mut String) {
|
||||
let unicode = self.encoding == encoding_rs::UTF_8
|
||||
|| self.encoding == encoding_rs::UTF_16LE
|
||||
|| self.encoding == encoding_rs::UTF_16BE;
|
||||
if unicode && !self.ignore_bom && !self.bom_seen && !output.is_empty() {
|
||||
self.bom_seen = true;
|
||||
if output.starts_with('\u{feff}') {
|
||||
output.remove(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_with_replacement(&mut self, bytes: &[u8], output: &mut String, last: bool) {
|
||||
let mut total_read = 0usize;
|
||||
loop {
|
||||
let input = &bytes[total_read..];
|
||||
let reserve = self
|
||||
.decoder
|
||||
.as_ref()
|
||||
.and_then(|decoder| decoder.max_utf8_buffer_length(input.len()))
|
||||
.max_utf8_buffer_length(input.len())
|
||||
.unwrap_or_else(|| input.len().saturating_mul(3).saturating_add(16));
|
||||
output.reserve(reserve);
|
||||
let (result, read, _) = self
|
||||
.decoder
|
||||
.as_mut()
|
||||
.expect("TextDecoderState should always own a decoder")
|
||||
.decode_to_string(input, output, last);
|
||||
let (result, read, _) = self.decoder.decode_to_string(input, output, last);
|
||||
total_read += read;
|
||||
match result {
|
||||
CoderResult::InputEmpty => return true,
|
||||
CoderResult::InputEmpty => return,
|
||||
CoderResult::OutputFull => continue,
|
||||
}
|
||||
}
|
||||
@@ -255,27 +269,23 @@ impl TextDecoderState {
|
||||
bytes: &[u8],
|
||||
output: &mut String,
|
||||
last: bool,
|
||||
) -> Result<bool, TextDecodeError> {
|
||||
) -> Result<(), TextDecodeError> {
|
||||
let mut total_read = 0usize;
|
||||
loop {
|
||||
let input = &bytes[total_read..];
|
||||
let reserve = self
|
||||
.decoder
|
||||
.as_ref()
|
||||
.and_then(|decoder| decoder.max_utf8_buffer_length_without_replacement(input.len()))
|
||||
.max_utf8_buffer_length_without_replacement(input.len())
|
||||
.unwrap_or_else(|| input.len().saturating_mul(3).saturating_add(16));
|
||||
output.reserve(reserve);
|
||||
let (result, read) = self
|
||||
.decoder
|
||||
.as_mut()
|
||||
.expect("TextDecoderState should always own a decoder")
|
||||
.decode_to_string_without_replacement(input, output, last);
|
||||
total_read += read;
|
||||
match result {
|
||||
DecoderResult::InputEmpty => return Ok(true),
|
||||
DecoderResult::InputEmpty => return Ok(()),
|
||||
DecoderResult::OutputFull => continue,
|
||||
DecoderResult::Malformed(_, _) => {
|
||||
self.reset_decoder();
|
||||
return Err(TextDecodeError::MalformedInput);
|
||||
}
|
||||
}
|
||||
@@ -283,17 +293,8 @@ impl TextDecoderState {
|
||||
}
|
||||
|
||||
fn reset_decoder(&mut self) {
|
||||
if let Some(encoding) = self.encoding {
|
||||
self.decoder = Some(new_encoding_decoder(encoding, self.ignore_bom));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn new_encoding_decoder(encoding: &'static Encoding, ignore_bom: bool) -> Decoder {
|
||||
if ignore_bom {
|
||||
encoding.new_decoder_without_bom_handling()
|
||||
} else {
|
||||
encoding.new_decoder_with_bom_removal()
|
||||
self.decoder = self.encoding.new_decoder_without_bom_handling();
|
||||
self.bom_seen = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -412,4 +413,175 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn text_decoder_state_preserves_multibyte_sequences_across_empty_chunks() {
|
||||
let cases: &[(&'static Encoding, &[u8], &str)] = &[
|
||||
(encoding_rs::BIG5, &[0xfe, 0x40], "\u{9442}"),
|
||||
(encoding_rs::SHIFT_JIS, &[0x81, 0x87], "\u{221e}"),
|
||||
(encoding_rs::EUC_KR, &[0x81, 0x41], "\u{ac02}"),
|
||||
];
|
||||
for &(encoding, input, expected) in cases {
|
||||
for fatal in [false, true] {
|
||||
for split in 0..=input.len() {
|
||||
let mut state = TextDecoderState::new(encoding, fatal, false);
|
||||
let mut output = state.decode(&input[..split], true).unwrap();
|
||||
for _ in 0..4 {
|
||||
assert_eq!(state.decode(&[], true).unwrap(), "");
|
||||
}
|
||||
output.push_str(&state.decode(&input[split..], true).unwrap());
|
||||
for _ in 0..4 {
|
||||
assert_eq!(state.decode(&[], true).unwrap(), "");
|
||||
}
|
||||
output.push_str(&state.decode(&[], false).unwrap());
|
||||
assert_eq!(
|
||||
output,
|
||||
expected,
|
||||
"{} fatal={fatal} split={split}",
|
||||
encoding.name()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_decoder_preserves_pending_bytes_when_output_is_full() {
|
||||
for (encoding, input, expected) in [
|
||||
(encoding_rs::BIG5, [0xfe, 0x40], "\u{9442}"),
|
||||
(encoding_rs::SHIFT_JIS, [0x81, 0x87], "\u{221e}"),
|
||||
(encoding_rs::EUC_KR, [0x81, 0x41], "\u{ac02}"),
|
||||
] {
|
||||
let mut decoder = encoding.new_decoder_without_bom_handling();
|
||||
let mut output = [0; 8];
|
||||
assert_eq!(
|
||||
decoder.decode_to_utf8_without_replacement(&input[..1], &mut output, false),
|
||||
(DecoderResult::InputEmpty, 1, 0)
|
||||
);
|
||||
assert_eq!(
|
||||
decoder.decode_to_utf8_without_replacement(&input[1..], &mut [], false),
|
||||
(DecoderResult::OutputFull, 0, 0)
|
||||
);
|
||||
let (result, read, written) =
|
||||
decoder.decode_to_utf8_without_replacement(&input[1..], &mut output, true);
|
||||
assert_eq!((result, read), (DecoderResult::InputEmpty, 1));
|
||||
assert_eq!(
|
||||
&output[..written],
|
||||
expected.as_bytes(),
|
||||
"{}",
|
||||
encoding.name()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_decoder_state_preserves_iso2022jp_after_fatal_stream_errors() {
|
||||
for (escape, byte, expected) in [(0x4a, 0x7e, "\u{203e}"), (0x49, 0x42, "\u{ff82}")] {
|
||||
let mut state = TextDecoderState::new(encoding_rs::ISO_2022_JP, true, false);
|
||||
assert_eq!(
|
||||
state.decode(&[byte], false).unwrap(),
|
||||
char::from(byte).to_string()
|
||||
);
|
||||
assert!(matches!(
|
||||
state.decode(&[0x1b, 0x28, escape, 0xff], true),
|
||||
Err(TextDecodeError::MalformedInput)
|
||||
));
|
||||
assert_eq!(state.decode(&[], true).unwrap(), "");
|
||||
assert_eq!(state.decode(&[byte], false).unwrap(), expected);
|
||||
assert_eq!(
|
||||
state.decode(&[byte], false).unwrap(),
|
||||
char::from(byte).to_string()
|
||||
);
|
||||
assert!(matches!(
|
||||
state.decode(&[0x1b, 0x28, escape, 0xff], false),
|
||||
Err(TextDecodeError::MalformedInput)
|
||||
));
|
||||
assert_eq!(
|
||||
state.decode(&[byte], false).unwrap(),
|
||||
char::from(byte).to_string()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_decoder_state_serializes_bom_only_after_successful_decode() {
|
||||
let cases: &[(&'static Encoding, &[u8], &[u8], &[u8])] = &[
|
||||
(encoding_rs::UTF_8, &[0xef, 0xbb, 0xbf], &[65], &[0xff]),
|
||||
(encoding_rs::UTF_16LE, &[0xff, 0xfe], &[65, 0], &[0, 0xdc]),
|
||||
(encoding_rs::UTF_16BE, &[0xfe, 0xff], &[0, 65], &[0xdc, 0]),
|
||||
];
|
||||
for &(encoding, bom, ascii, invalid) in cases {
|
||||
for ignore_bom in [false, true] {
|
||||
let input = [bom, ascii].concat();
|
||||
for prefix in [&[][..], ascii, bom] {
|
||||
let mut state = TextDecoderState::new(encoding, true, ignore_bom);
|
||||
let expected = if prefix == ascii {
|
||||
"A"
|
||||
} else if prefix == bom && ignore_bom {
|
||||
"\u{feff}"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
assert_eq!(state.decode(prefix, true).unwrap(), expected);
|
||||
assert!(matches!(
|
||||
state.decode(invalid, true),
|
||||
Err(TextDecodeError::MalformedInput)
|
||||
));
|
||||
let expected = if ignore_bom || !prefix.is_empty() {
|
||||
"\u{feff}A"
|
||||
} else {
|
||||
"A"
|
||||
};
|
||||
assert_eq!(state.decode(&input, false).unwrap(), expected);
|
||||
let fresh = if ignore_bom { "\u{feff}A" } else { "A" };
|
||||
assert_eq!(state.decode(&input, false).unwrap(), fresh);
|
||||
}
|
||||
let mut state = TextDecoderState::new(encoding, true, ignore_bom);
|
||||
assert!(matches!(
|
||||
state.decode(&[bom, invalid].concat(), true),
|
||||
Err(TextDecodeError::MalformedInput)
|
||||
));
|
||||
let expected = if ignore_bom { "\u{feff}A" } else { "A" };
|
||||
assert_eq!(state.decode(&input, false).unwrap(), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_decoder_state_preserves_bom_serialization_across_all_chunk_partitions() {
|
||||
let cases: &[(&'static Encoding, &[u8], &[u8])] = &[
|
||||
(encoding_rs::UTF_8, &[0xef, 0xbb, 0xbf], &[65]),
|
||||
(encoding_rs::UTF_16LE, &[0xff, 0xfe], &[65, 0]),
|
||||
(encoding_rs::UTF_16BE, &[0xfe, 0xff], &[0, 65]),
|
||||
];
|
||||
for &(encoding, bom, ascii) in cases {
|
||||
let input = [bom, ascii, bom, ascii].concat();
|
||||
for fatal in [false, true] {
|
||||
for ignore_bom in [false, true] {
|
||||
let expected = if ignore_bom {
|
||||
"\u{feff}A\u{feff}A"
|
||||
} else {
|
||||
"A\u{feff}A"
|
||||
};
|
||||
for first in 0..=input.len() {
|
||||
for second in first..=input.len() {
|
||||
let mut state = TextDecoderState::new(encoding, fatal, ignore_bom);
|
||||
let mut actual = String::new();
|
||||
for chunk in [&input[..first], &input[first..second], &input[second..]]
|
||||
{
|
||||
actual.push_str(&state.decode(chunk, true).unwrap());
|
||||
assert_eq!(state.decode(&[], true).unwrap(), "");
|
||||
}
|
||||
actual.push_str(&state.decode(&[], false).unwrap());
|
||||
assert_eq!(
|
||||
actual,
|
||||
expected,
|
||||
"{} cuts={first}/{second}",
|
||||
encoding.name()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user