From 43760d120564ef0bb63d7963eec0fb0b99214dfe Mon Sep 17 00:00:00 2001 From: ldm0 Date: Thu, 27 Aug 2026 00:40:12 +0800 Subject: [PATCH] feat(encoding): detect unlabeled legacy HTML like Blink --- Cargo.lock | 117 ++++++++++++++++++ moli-encoding/Cargo.toml | 1 + moli-encoding/src/detector.rs | 84 +++++++++++++ moli-encoding/src/document.rs | 39 +++++- moli-encoding/src/lib.rs | 1 + moli-encoding/src/tests.rs | 18 ++- .../src/runtime/phase_one/streaming.rs | 6 +- 7 files changed, 257 insertions(+), 9 deletions(-) create mode 100644 moli-encoding/src/detector.rs diff --git a/Cargo.lock b/Cargo.lock index 040ba27b72..f5b54c1c5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,6 +574,17 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + [[package]] name = "color" version = "0.3.3" @@ -595,6 +606,25 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "compact-enc-det" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb43ad03170dee67e3058ac6019ae5018198e6c9e6c7f2d6c3275bfb564aa010" +dependencies = [ + "compact-enc-det-sys", +] + +[[package]] +name = "compact-enc-det-sys" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4e84b53cd2643baf7cd7ec9a0d56afadede16eec55051d305c8af4fce59b6e" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "compact_str" version = "0.9.0" @@ -814,6 +844,68 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "cxx" +version = "1.0.199" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824894a4a85dca76d4c95c2b9098c036f5a29f627b30c12780774f6654e60974" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.199" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ae0b651ea5b0000b19513aef5a03f194d7e3486f2d9258b658da8677fe9036" +dependencies = [ + "cc", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "scratch", + "syn 3.0.3", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.199" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb05f91d3fb8435d9bab6ac5ce6ac1868be774325fb7fb2a91be39393b21388e" +dependencies = [ + "clap", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.199" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf293202e0e3e98495785745389e8d0755b217e66f19194a5c695c25e03282ef" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.199" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca001d746947c7249ed9d332a10f7a59daedbafeb0ec68c5c18a7db7a93f6ccc" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "darling" version = "0.20.11" @@ -2011,6 +2103,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2346,6 +2447,7 @@ dependencies = [ name = "moli-encoding" version = "0.1.0" dependencies = [ + "compact-enc-det", "encoding_rs", "moli-charset-parser", "moli-content-type", @@ -4114,6 +4216,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + [[package]] name = "sec1" version = "0.8.1" @@ -4758,6 +4866,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "textwrap" version = "0.16.2" diff --git a/moli-encoding/Cargo.toml b/moli-encoding/Cargo.toml index 908d897762..5dd413cff5 100644 --- a/moli-encoding/Cargo.toml +++ b/moli-encoding/Cargo.toml @@ -5,6 +5,7 @@ version = "0.1.0" edition = "2024" [dependencies] +compact-enc-det = "0.1" encoding_rs = "0.8" moli-charset-parser = { path = "../moli-charset-parser" } moli-content-type = { path = "../moli-content-type" } diff --git a/moli-encoding/src/detector.rs b/moli-encoding/src/detector.rs new file mode 100644 index 0000000000..af1add771f --- /dev/null +++ b/moli-encoding/src/detector.rs @@ -0,0 +1,84 @@ +use compact_enc_det::{DetectHints, Encoding as CedEncoding, TextCorpusType, detect_encoding}; +use encoding_rs::Encoding; + +pub(crate) fn detect_legacy_html_encoding( + bytes: &[u8], + url_hint: Option<&str>, +) -> Option<&'static Encoding> { + if bytes.iter().all(u8::is_ascii) { + return None; + } + + let detection = detect_encoding( + bytes, + DetectHints { + url_hint: url_hint.unwrap_or_default(), + corpus_type: TextCorpusType::WEB_CORPUS, + // Blink asks CED to consider 7-bit encodings and then disables + // ISO-2022-JP for HTML at the TextResourceDecoder boundary. + ignore_7bit_mail_encodings: false, + ..DetectHints::default() + }, + ); + if matches!( + detection.encoding, + CedEncoding::UNKNOWN_ENCODING + | CedEncoding::ASCII_7BIT + | CedEncoding::UTF8 + | CedEncoding::UTF16BE + | CedEncoding::UTF16LE + | CedEncoding::UTF32BE + | CedEncoding::UTF32LE + | CedEncoding::JAPANESE_JIS + | CedEncoding::KDDI_ISO_2022_JP + | CedEncoding::SOFTBANK_ISO_2022_JP + ) { + return None; + } + + Encoding::for_label(detection.mime_name.as_bytes()).filter(|encoding| { + *encoding != encoding_rs::UTF_8 + && *encoding != encoding_rs::UTF_16BE + && *encoding != encoding_rs::UTF_16LE + && *encoding != encoding_rs::ISO_2022_JP + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_unlabelled_gbk_legacy_web_content() { + let text = "吴姓－姓氏渊源。第一源流，源于姜姓，出自炎帝大臣吴权之后裔。".repeat(8); + let bytes = encoding_rs::GBK.encode(&text).0; + + assert_eq!( + detect_legacy_html_encoding(&bytes, None), + Some(encoding_rs::GBK) + ); + } + + #[test] + fn does_not_auto_detect_utf8_or_ascii() { + assert_eq!( + detect_legacy_html_encoding("吴姓－姓氏渊源".as_bytes(), None), + None + ); + assert_eq!(detect_legacy_html_encoding(b"plain ASCII", None), None); + } + + #[test] + fn url_hint_disambiguates_euc_jp_like_blink() { + let bytes = b"\xA5\xD1\xA5\xEF\xA1\xBC\xA5\xC1\xA5\xE3\xA1\xBC\xA5\xC8\xA1\xC3\xC5\xEA\xBB\xF1\xBE\xF0\xCA\xF3\xA4\xCE\xA5\xD5\xA5\xA3\xA5\xB9\xA5\xB3"; + + assert_eq!( + detect_legacy_html_encoding(bytes, None), + Some(encoding_rs::GBK) + ); + assert_eq!( + detect_legacy_html_encoding(bytes, Some("http://example.co.jp/")), + Some(encoding_rs::EUC_JP) + ); + } +} diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs index 09098542c9..02df2ce05b 100644 --- a/moli-encoding/src/document.rs +++ b/moli-encoding/src/document.rs @@ -1,7 +1,9 @@ use encoding_rs::{CoderResult, Decoder, Encoding}; use moli_charset_parser::{HtmlMetaCharsetParser, HtmlMetaCharsetScanResult}; -use crate::{encoding_for_label, encoding_from_response_headers}; +use crate::{ + detector::detect_legacy_html_encoding, encoding_for_label, encoding_from_response_headers, +}; const DEFAULT_HTML_DOCUMENT_ENCODING: &str = "windows-1252"; const UTF16LE_XML_PREFIX: &[u8; 6] = b"<\0?\0x\0"; @@ -37,18 +39,32 @@ pub struct HtmlDocumentStreamingDecoder { emitted_sniff_len: usize, meta_prescan_fed_len: usize, meta_charset_parser: HtmlMetaCharsetParser, + allow_legacy_content_detection: bool, + url_hint: Option, decoder: Option, selected_encoding: Option<&'static Encoding>, } impl HtmlDocumentStreamingDecoder { pub fn new(headers: &[(String, String)]) -> Self { - Self::new_with_fallback(headers, None) + Self::new_with_fallback_and_url(headers, None, None) + } + + pub fn new_for_url(headers: &[(String, String)], url_hint: &str) -> Self { + Self::new_with_fallback_and_url(headers, None, Some(url_hint)) } pub fn new_with_fallback( headers: &[(String, String)], fallback_encoding: Option<&str>, + ) -> Self { + Self::new_with_fallback_and_url(headers, fallback_encoding, None) + } + + fn new_with_fallback_and_url( + headers: &[(String, String)], + fallback_encoding: Option<&str>, + url_hint: Option<&str>, ) -> Self { Self { transport_encoding: encoding_from_response_headers(headers), @@ -59,6 +75,11 @@ impl HtmlDocumentStreamingDecoder { emitted_sniff_len: 0, meta_prescan_fed_len: 0, meta_charset_parser: HtmlMetaCharsetParser::new(), + // A supplied fallback represents an inherited or otherwise + // authoritative default. Blink only lets content detection replace + // its own default, not an ordinary parent-frame encoding. + allow_legacy_content_detection: fallback_encoding.is_none(), + url_hint: url_hint.map(str::to_owned), decoder: None, selected_encoding: None, } @@ -130,12 +151,22 @@ impl HtmlDocumentStreamingDecoder { return Some(encoding); } match meta_scan { - HtmlMetaCharsetScanResult::NotFound => Some(self.fallback_encoding), - HtmlMetaCharsetScanResult::Pending if finishing => Some(self.fallback_encoding), + HtmlMetaCharsetScanResult::NotFound => self + .detected_legacy_content_encoding() + .or(Some(self.fallback_encoding)), + HtmlMetaCharsetScanResult::Pending if finishing => self + .detected_legacy_content_encoding() + .or(Some(self.fallback_encoding)), HtmlMetaCharsetScanResult::Pending | HtmlMetaCharsetScanResult::Found(_) => None, } } + fn detected_legacy_content_encoding(&self) -> Option<&'static Encoding> { + self.allow_legacy_content_detection + .then(|| detect_legacy_html_encoding(&self.sniff_buffer, self.url_hint.as_deref())) + .flatten() + } + fn feed_meta_charset_prescan(&mut self, finishing: bool) -> HtmlMetaCharsetScanResult { let scan = if self.meta_prescan_fed_len < self.sniff_buffer.len() { let scan = self diff --git a/moli-encoding/src/lib.rs b/moli-encoding/src/lib.rs index b623fb353c..09f1f001ea 100644 --- a/moli-encoding/src/lib.rs +++ b/moli-encoding/src/lib.rs @@ -1,5 +1,6 @@ //! Browser-compatible text, document, form, script, and URL encoding helpers. +mod detector; mod document; mod form; mod labels; diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs index 4be29689c5..33e16f298c 100644 --- a/moli-encoding/src/tests.rs +++ b/moli-encoding/src/tests.rs @@ -114,7 +114,10 @@ fn meta_charset_after_1024_bytes_after_head_is_ignored() { input.extend(vec![b' '; HTML_META_CHARSET_PRESCAN_LIMIT - input.len()]); input.extend_from_slice(b"

"); input.extend_from_slice(&gbk_bytes("家居")); - let mut decoder = HtmlDocumentStreamingDecoder::new(&headers); + // Suppress heuristic detection so this test isolates the meta prescan's + // head boundary rather than CED independently recognizing the GBK body. + let mut decoder = + HtmlDocumentStreamingDecoder::new_with_fallback(&headers, Some("windows-1252")); let decoded = decoder.push(&input).join(""); @@ -219,9 +222,18 @@ fn unknown_charset_falls_back_to_html_default_on_finish() { } #[test] -fn no_label_html_document_falls_back_to_windows_1252() { +fn unlabelled_html_document_uses_legacy_content_detection() { let (text, encoding) = decode_html_document(b"\x80\x80 Hello", &[]); + assert_eq!(encoding, "IBM866"); + assert_eq!(text, "\u{410}\u{410} Hello"); +} + +#[test] +fn inherited_fallback_suppresses_legacy_content_detection() { + let (text, encoding) = + decode_html_document_with_fallback(b"\x80\x80 Hello", &[], Some("windows-1252")); + assert_eq!(encoding, "windows-1252"); assert_eq!(text, "\u{20ac}\u{20ac} Hello"); } @@ -792,7 +804,7 @@ fn header_charset_exact_lookup_rejects_non_http_whitespace() { assert_eq!(encoding_from_response_headers(&headers), None); assert_eq!( decode_html_document(&gbk_bytes("太平洋"), &headers).1, - "windows-1252" + "GBK" ); } } diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs index cf895697ef..e565ba05dd 100644 --- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs +++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs @@ -123,7 +123,8 @@ impl ConcurrentParseTimeRuntime { ) }); state.service_worker_preload_context = service_worker_preload_context.clone(); - let mut decoder = HtmlDocumentStreamingDecoder::new(&response_headers); + let mut decoder = + HtmlDocumentStreamingDecoder::new_for_url(&response_headers, state.final_url.as_str()); // Raw navigation bodies are decoded during prebootstrap scan. The decoder // is carried forward so split multibyte sequences are not decoded twice or // lost between the scan and parser handoff. @@ -357,7 +358,8 @@ impl ConcurrentParseTimeRuntime { ) }); state.service_worker_preload_context = service_worker_preload_context.clone(); - let mut decoder = HtmlDocumentStreamingDecoder::new(&response_headers); + let mut decoder = + HtmlDocumentStreamingDecoder::new_for_url(&response_headers, state.final_url.as_str()); // External raw bodies replay captured chunks. Pre-scan only what is already // buffered before bootstrap so the producer's backpressure boundary still // controls how far ahead the parser can get.