starred/moli
1
0
Fork 0
mirror of https://github.com/lexmount/moli.git synced 2026-10-03 08:00:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(encoding): detect unlabeled legacy HTML like Blink

Browse Source
This commit is contained in:
ldm0
2026-08-27 02:57:24 +08:00
parent 12b6d5af8b
commit 43760d1205
7 changed files with 257 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+117
View File
@@ -574,6 +574,17 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
[[package]]
name = "codespan-reporting"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"
dependencies = [
"serde",
"termcolor",
"unicode-width",
]
[[package]]
name = "color"
version = "0.3.3"
@@ -595,6 +606,25 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]]
name = "compact-enc-det"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb43ad03170dee67e3058ac6019ae5018198e6c9e6c7f2d6c3275bfb564aa010"
dependencies = [
"compact-enc-det-sys",
]
[[package]]
name = "compact-enc-det-sys"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec4e84b53cd2643baf7cd7ec9a0d56afadede16eec55051d305c8af4fce59b6e"
dependencies = [
"cxx",
"cxx-build",
]
[[package]]
name = "compact_str"
version = "0.9.0"
@@ -814,6 +844,68 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "cxx"
version = "1.0.199"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824894a4a85dca76d4c95c2b9098c036f5a29f627b30c12780774f6654e60974"
dependencies = [
"cc",
"cxx-build",
"cxxbridge-cmd",
"cxxbridge-flags",
"cxxbridge-macro",
"foldhash",
"link-cplusplus",
]
[[package]]
name = "cxx-build"
version = "1.0.199"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1ae0b651ea5b0000b19513aef5a03f194d7e3486f2d9258b658da8677fe9036"
dependencies = [
"cc",
"codespan-reporting",
"indexmap",
"proc-macro2",
"quote",
"scratch",
"syn 3.0.3",
]
[[package]]
name = "cxxbridge-cmd"
version = "1.0.199"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb05f91d3fb8435d9bab6ac5ce6ac1868be774325fb7fb2a91be39393b21388e"
dependencies = [
"clap",
"codespan-reporting",
"indexmap",
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "cxxbridge-flags"
version = "1.0.199"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf293202e0e3e98495785745389e8d0755b217e66f19194a5c695c25e03282ef"
[[package]]
name = "cxxbridge-macro"
version = "1.0.199"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca001d746947c7249ed9d332a10f7a59daedbafeb0ec68c5c18a7db7a93f6ccc"
dependencies = [
"indexmap",
"proc-macro2",
"quote",
"syn 3.0.3",
]
[[package]]
name = "darling"
version = "0.20.11"
@@ -2011,6 +2103,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4"
[[package]]
name = "link-cplusplus"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82"
dependencies = [
"cc",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
@@ -2346,6 +2447,7 @@ dependencies = [
name = "moli-encoding"
version = "0.1.0"
dependencies = [
"compact-enc-det",
"encoding_rs",
"moli-charset-parser",
"moli-content-type",
@@ -4114,6 +4216,12 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scratch"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2"
[[package]]
name = "sec1"
version = "0.8.1"
@@ -4758,6 +4866,15 @@ dependencies = [
"utf-8",
]
[[package]]
name = "termcolor"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
dependencies = [
"winapi-util",
]
[[package]]
name = "textwrap"
version = "0.16.2"
moli-encoding/Cargo.toml
+1
View File
@@ -5,6 +5,7 @@ version = "0.1.0"
edition = "2024"
[dependencies]
compact-enc-det = "0.1"
encoding_rs = "0.8"
moli-charset-parser = { path = "../moli-charset-parser" }
moli-content-type = { path = "../moli-content-type" }
moli-encoding/src/detector.rs
+84
View File
@@ -0,0 +1,84 @@
use compact_enc_det::{DetectHints, Encoding as CedEncoding, TextCorpusType, detect_encoding};
use encoding_rs::Encoding;
pub(crate) fn detect_legacy_html_encoding(
bytes: &[u8],
url_hint: Option<&str>,
) -> Option<&'static Encoding> {
if bytes.iter().all(u8::is_ascii) {
return None;
}
let detection = detect_encoding(
bytes,
DetectHints {
url_hint: url_hint.unwrap_or_default(),
corpus_type: TextCorpusType::WEB_CORPUS,
// Blink asks CED to consider 7-bit encodings and then disables
// ISO-2022-JP for HTML at the TextResourceDecoder boundary.
ignore_7bit_mail_encodings: false,
..DetectHints::default()
},
);
if matches!(
detection.encoding,
CedEncoding::UNKNOWN_ENCODING
| CedEncoding::ASCII_7BIT
| CedEncoding::UTF8
| CedEncoding::UTF16BE
| CedEncoding::UTF16LE
| CedEncoding::UTF32BE
| CedEncoding::UTF32LE
| CedEncoding::JAPANESE_JIS
| CedEncoding::KDDI_ISO_2022_JP
| CedEncoding::SOFTBANK_ISO_2022_JP
) {
return None;
}
Encoding::for_label(detection.mime_name.as_bytes()).filter(|encoding| {
*encoding != encoding_rs::UTF_8
&& *encoding != encoding_rs::UTF_16BE
&& *encoding != encoding_rs::UTF_16LE
&& *encoding != encoding_rs::ISO_2022_JP
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_unlabelled_gbk_legacy_web_content() {
let text = "吴姓－姓氏渊源。第一源流，源于姜姓，出自炎帝大臣吴权之后裔。".repeat(8);
let bytes = encoding_rs::GBK.encode(&text).0;
assert_eq!(
detect_legacy_html_encoding(&bytes, None),
Some(encoding_rs::GBK)
);
}
#[test]
fn does_not_auto_detect_utf8_or_ascii() {
assert_eq!(
detect_legacy_html_encoding("吴姓－姓氏渊源".as_bytes(), None),
None
);
assert_eq!(detect_legacy_html_encoding(b"plain ASCII", None), None);
}
#[test]
fn url_hint_disambiguates_euc_jp_like_blink() {
let bytes = b"<TITLE>\xA5\xD1\xA5\xEF\xA1\xBC\xA5\xC1\xA5\xE3\xA1\xBC\xA5\xC8\xA1\xC3\xC5\xEA\xBB\xF1\xBE\xF0\xCA\xF3\xA4\xCE\xA5\xD5\xA5\xA3\xA5\xB9\xA5\xB3</TITLE>";
assert_eq!(
detect_legacy_html_encoding(bytes, None),
Some(encoding_rs::GBK)
);
assert_eq!(
detect_legacy_html_encoding(bytes, Some("http://example.co.jp/")),
Some(encoding_rs::EUC_JP)
);
}
}
moli-encoding/src/document.rs
+35 -4
View File
@@ -1,7 +1,9 @@
use encoding_rs::{CoderResult, Decoder, Encoding};
use moli_charset_parser::{HtmlMetaCharsetParser, HtmlMetaCharsetScanResult};
use crate::{encoding_for_label, encoding_from_response_headers};
use crate::{
detector::detect_legacy_html_encoding, encoding_for_label, encoding_from_response_headers,
};
const DEFAULT_HTML_DOCUMENT_ENCODING: &str = "windows-1252";
const UTF16LE_XML_PREFIX: &[u8; 6] = b"<\0?\0x\0";
@@ -37,18 +39,32 @@ pub struct HtmlDocumentStreamingDecoder {
emitted_sniff_len: usize,
meta_prescan_fed_len: usize,
meta_charset_parser: HtmlMetaCharsetParser,
allow_legacy_content_detection: bool,
url_hint: Option<String>,
decoder: Option<Decoder>,
selected_encoding: Option<&'static Encoding>,
}
impl HtmlDocumentStreamingDecoder {
pub fn new(headers: &[(String, String)]) -> Self {
Self::new_with_fallback(headers, None)
Self::new_with_fallback_and_url(headers, None, None)
}
pub fn new_for_url(headers: &[(String, String)], url_hint: &str) -> Self {
Self::new_with_fallback_and_url(headers, None, Some(url_hint))
}
pub fn new_with_fallback(
headers: &[(String, String)],
fallback_encoding: Option<&str>,
) -> Self {
Self::new_with_fallback_and_url(headers, fallback_encoding, None)
}
fn new_with_fallback_and_url(
headers: &[(String, String)],
fallback_encoding: Option<&str>,
url_hint: Option<&str>,
) -> Self {
Self {
transport_encoding: encoding_from_response_headers(headers),
@@ -59,6 +75,11 @@ impl HtmlDocumentStreamingDecoder {
emitted_sniff_len: 0,
meta_prescan_fed_len: 0,
meta_charset_parser: HtmlMetaCharsetParser::new(),
// A supplied fallback represents an inherited or otherwise
// authoritative default. Blink only lets content detection replace
// its own default, not an ordinary parent-frame encoding.
allow_legacy_content_detection: fallback_encoding.is_none(),
url_hint: url_hint.map(str::to_owned),
decoder: None,
selected_encoding: None,
}
@@ -130,12 +151,22 @@ impl HtmlDocumentStreamingDecoder {
return Some(encoding);
}
match meta_scan {
HtmlMetaCharsetScanResult::NotFound => Some(self.fallback_encoding),
HtmlMetaCharsetScanResult::Pending if finishing => Some(self.fallback_encoding),
HtmlMetaCharsetScanResult::NotFound => self
.detected_legacy_content_encoding()
.or(Some(self.fallback_encoding)),
HtmlMetaCharsetScanResult::Pending if finishing => self
.detected_legacy_content_encoding()
.or(Some(self.fallback_encoding)),
HtmlMetaCharsetScanResult::Pending | HtmlMetaCharsetScanResult::Found(_) => None,
}
}
fn detected_legacy_content_encoding(&self) -> Option<&'static Encoding> {
self.allow_legacy_content_detection
.then(|| detect_legacy_html_encoding(&self.sniff_buffer, self.url_hint.as_deref()))
.flatten()
}
fn feed_meta_charset_prescan(&mut self, finishing: bool) -> HtmlMetaCharsetScanResult {
let scan = if self.meta_prescan_fed_len < self.sniff_buffer.len() {
let scan = self
moli-encoding/src/lib.rs
+1
View File
@@ -1,5 +1,6 @@
//! Browser-compatible text, document, form, script, and URL encoding helpers.
mod detector;
mod document;
mod form;
mod labels;
moli-encoding/src/tests.rs
+15 -3
View File
@@ -114,7 +114,10 @@ fn meta_charset_after_1024_bytes_after_head_is_ignored() {
input.extend(vec![b' '; HTML_META_CHARSET_PRESCAN_LIMIT - input.len()]);
input.extend_from_slice(b"<meta charset=\"gbk\"><p>");
input.extend_from_slice(&gbk_bytes("家居"));
let mut decoder = HtmlDocumentStreamingDecoder::new(&headers);
// Suppress heuristic detection so this test isolates the meta prescan's
// head boundary rather than CED independently recognizing the GBK body.
let mut decoder =
HtmlDocumentStreamingDecoder::new_with_fallback(&headers, Some("windows-1252"));
let decoded = decoder.push(&input).join("");
@@ -219,9 +222,18 @@ fn unknown_charset_falls_back_to_html_default_on_finish() {
}
#[test]
fn no_label_html_document_falls_back_to_windows_1252() {
fn unlabelled_html_document_uses_legacy_content_detection() {
let (text, encoding) = decode_html_document(b"\x80\x80 Hello", &[]);
assert_eq!(encoding, "IBM866");
assert_eq!(text, "\u{410}\u{410} Hello");
}
#[test]
fn inherited_fallback_suppresses_legacy_content_detection() {
let (text, encoding) =
decode_html_document_with_fallback(b"\x80\x80 Hello", &[], Some("windows-1252"));
assert_eq!(encoding, "windows-1252");
assert_eq!(text, "\u{20ac}\u{20ac} Hello");
}
@@ -792,7 +804,7 @@ fn header_charset_exact_lookup_rejects_non_http_whitespace() {
assert_eq!(encoding_from_response_headers(&headers), None);
assert_eq!(
decode_html_document(&gbk_bytes("太平洋"), &headers).1,
"windows-1252"
"GBK"
);
}
}
moli-renderer-v8/src/runtime/phase_one/streaming.rs
+4 -2
View File
@@ -123,7 +123,8 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
let mut decoder = HtmlDocumentStreamingDecoder::new(&response_headers);
let mut decoder =
HtmlDocumentStreamingDecoder::new_for_url(&response_headers, state.final_url.as_str());
// Raw navigation bodies are decoded during prebootstrap scan. The decoder
// is carried forward so split multibyte sequences are not decoded twice or
// lost between the scan and parser handoff.
@@ -357,7 +358,8 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
let mut decoder = HtmlDocumentStreamingDecoder::new(&response_headers);
let mut decoder =
HtmlDocumentStreamingDecoder::new_for_url(&response_headers, state.final_url.as_str());
// External raw bodies replay captured chunks. Pre-scan only what is already
// buffered before bootstrap so the producer's backpressure boundary still
// controls how far ahead the parser can get.