starred/moli
1
0
Fork 0
mirror of https://github.com/lexmount/moli.git synced 2026-09-25 00:01:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(parser): preserve text document MIME and encoding semantics

Browse Source
This commit is contained in:
lanyue-llk
2026-09-22 20:43:46 +08:00
parent cec1edaec3
commit fc42d8ddd9
22 changed files with 625 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
moli-encoding/src/document.rs
+25
View File
@@ -47,9 +47,26 @@ pub struct HtmlDocumentStreamingDecoder {
url_hint: Option<String>,
decoder: Option<Decoder>,
selected_encoding: Option<&'static Encoding>,
sniff_html_declarations: bool,
}
impl HtmlDocumentStreamingDecoder {
/// Decode a text document without interpreting literal HTML/XML declarations.
/// JSON defaults to UTF-8; other text retains the configured legacy detector.
pub fn new_text_document(
headers: &[(String, String)],
url_hint: &str,
detector: LegacyEncodingDetector,
json: bool,
) -> Self {
let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector));
decoder.sniff_html_declarations = false;
if json && decoder.transport_encoding.is_none() {
decoder.transport_encoding = Some(encoding_rs::UTF_8);
}
decoder
}
pub fn new(headers: &[(String, String)]) -> Self {
Self::new_with_options(headers, None, None, None)
}
@@ -88,6 +105,7 @@ impl HtmlDocumentStreamingDecoder {
url_hint: url_hint.map(str::to_owned),
decoder: None,
selected_encoding: None,
sniff_html_declarations: true,
}
}
@@ -97,6 +115,7 @@ impl HtmlDocumentStreamingDecoder {
pub fn document_encoding_name(&self) -> &'static str {
self.selected_encoding_name()
.or_else(|| self.transport_encoding.map(Encoding::name))
.unwrap_or(self.fallback_encoding.name())
}
@@ -149,6 +168,12 @@ impl HtmlDocumentStreamingDecoder {
if let Some(encoding) = self.transport_encoding {
return Some(encoding);
}
if !self.sniff_html_declarations {
return (finishing || self.sniff_buffer.len() >= 1024).then(|| {
self.detected_legacy_content_encoding()
.unwrap_or(self.fallback_encoding)
});
}
if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) {
return Some(encoding);
}
moli-encoding/src/tests.rs
+96
View File
@@ -24,6 +24,102 @@ fn unexpected_legacy_encoding_detector(
panic!("declared encoding must take precedence over heuristic detection")
}
#[test]
fn json_text_document_defaults_to_utf8_across_every_chunk_split() {
let input = "{\"name\":\"Gülçek\"}";
for split in 0..=input.len() {
let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
&[],
"https://example.test/",
unexpected_legacy_encoding_detector,
true,
);
let mut output = decoder.push(&input.as_bytes()[..split]).join("");
output.push_str(&decoder.push(&input.as_bytes()[split..]).join(""));
output.push_str(&decoder.finish().unwrap_or_default());
assert_eq!(output, input, "split {split}");
assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
}
}
#[test]
fn plain_text_document_honors_transport_charset() {
let headers = [(
"Content-Type".to_owned(),
"text/plain; charset=gbk".to_owned(),
)];
let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
&headers,
"https://example.test/",
unexpected_legacy_encoding_detector,
false,
);
let mut output = decoder.push(&gbk_bytes("<b>家居</b>")).join("");
output.push_str(&decoder.finish().unwrap_or_default());
assert_eq!(output, "<b>家居</b>");
assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
}
#[test]
fn text_document_bom_precedes_transport_charset() {
let headers = [(
"Content-Type".to_owned(),
"text/plain; charset=gbk".to_owned(),
)];
let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
&headers,
"https://example.test/",
unexpected_legacy_encoding_detector,
false,
);
assert_eq!(decoder.document_encoding_name(), "GBK");
assert_eq!(decoder.selected_encoding_name(), None);
assert!(decoder.push(&[0xef]).is_empty());
assert!(decoder.push(&[0xbb]).is_empty());
let mut tail = vec![0xbf];
tail.extend_from_slice("Gülçek".as_bytes());
let mut output = decoder.push(&tail).join("");
output.push_str(&decoder.finish().unwrap_or_default());
assert_eq!(output, "Gülçek");
assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
assert_eq!(decoder.document_encoding_name(), "UTF-8");
}
#[test]
fn plain_text_document_retains_legacy_detection_without_a_charset() {
let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
&[],
"https://legacy.example/",
test_legacy_encoding_detector,
false,
);
let mut output = decoder.push(&[0x80]).join("");
output.push_str(&decoder.finish().unwrap_or_default());
assert_eq!(output, "А");
assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));
}
#[test]
fn plain_text_document_ignores_literal_encoding_declarations() {
for prefix in [
"<meta charset=utf-8>",
"<?xml version='1.0' encoding='utf-8'?>",
] {
let mut input = prefix.as_bytes().to_vec();
input.push(0xe9);
let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
&[],
"https://example.test/",
test_legacy_encoding_detector,
false,
);
let mut output = decoder.push(&input).join("");
output.push_str(&decoder.finish().unwrap_or_default());
assert_eq!(output, format!("{prefix}é"));
assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
}
}
#[test]
fn content_type_charset_is_selected() {
let headers = vec![(
moli-parser/src/html.rs
+92 -1
View File
@@ -27,7 +27,7 @@ use super::live_target::{
ParserDomMutationConsumer, ParserDomReadConsumer, ParserElementCreationConsumer,
ParserMutationEffectConsumer, ParserMutationEffectDelivery, ParserRuntimeDomSinks,
ParserStreamHtmlTreeSinkTarget, new_live_document_root_html_tree_sink_stream,
new_parser_stream_html_tree_sink_stream,
new_parser_stream_html_tree_sink_stream, new_parser_stream_html_tree_sink_target,
};
use super::{
ParserSourcePosition, html_chunks,
@@ -438,6 +438,47 @@ impl HtmlParser {
DocumentStream::new_parser_stream(final_url, self.scripting_enabled)
}
/// Start a text document using the HTML tokenizer's plaintext state.
/// Subsequent chunks remain literal text inside the browser-owned `pre`.
pub fn start_text_document(&self, final_url: Url, content_type: &str) -> DocumentStream {
let stream =
DocumentStream::new_text_document(new_parser_stream_html_tree_sink_target(final_url));
stream.inner.initialize_text_document();
let mut host = stream.inner.take_parser_stream_dom_host();
let document_handle = host.document_handle();
host.set_document_content_type_for_handle(document_handle, content_type);
stream.inner.restore_parser_stream_dom_host(host);
stream
}
/// Initialize the same literal-text shell in an existing document. The
/// embedder owns that document's MIME metadata and live DOM callbacks.
pub fn start_live_text_document_root<T>(
&self,
final_url: Url,
document_handle: NativeNodeId,
consumer: &mut T,
) -> DocumentStream
where
T: ParserDomReadConsumer
+ ParserDomMutationConsumer
+ ParserMutationEffectConsumer
+ ParserElementCreationConsumer,
{
let stream = DocumentStream::new_text_document(
ParserStreamHtmlTreeSinkTarget::new_live_document_root(final_url, document_handle),
);
// SAFETY: the consumer is exclusively borrowed until the step guard
// removes the erased callbacks, including when initialization unwinds.
let sinks = unsafe { ParserRuntimeDomSinks::from_consumer(consumer) };
stream.inner.enter_runtime_dom_sinks_parse_step(sinks);
{
let step = RuntimeDomSinksParserStep { stream: &stream };
step.stream.inner.initialize_text_document();
}
stream
}
pub fn start_live_document_root(
&self,
final_url: Url,
@@ -551,6 +592,17 @@ impl Default for ParserInputQueue {
}
impl DocumentStream {
fn new_text_document(target: ParserStreamHtmlTreeSinkTarget) -> Self {
let mut options = html_parse_opts_with_scripting(false);
// The byte decoder has already consumed the transport BOM. In plaintext
// every remaining U+FEFF is content, including at a chunk boundary.
options.tokenizer.discard_bom = false;
Self {
inner: HtmlTreeSinkStream::from_target_with_options(target, options),
input: RefCell::default(),
}
}
fn new_parser_stream(final_url: Url, scripting_enabled: bool) -> Self {
Self {
inner: new_parser_stream_html_tree_sink_stream(final_url, scripting_enabled),
@@ -1843,6 +1895,45 @@ mod tests {
const HTML_NS: &str = "http://www.w3.org/1999/xhtml";
const MATHML_NS: &str = "http://www.w3.org/1998/Math/MathML";
#[test]
fn text_document_stream_preserves_literal_markup_and_leading_newline() {
let payload =
"\u{feff}\n<b>Gülçek</b>&amp;<script>window.executed=1</script></pre>\u{feff}";
for content_type in ["text/plain", "application/json", "application/problem+json"] {
let stream = HtmlParser::SCRIPTING_ENABLED.start_text_document(
Url::parse("https://example.test/data").unwrap(),
content_type,
);
for character in payload.chars() {
stream.feed(&character.to_string());
}
let document = stream.finish();
let pre = first_element_by_ns(&document, HTML_NS, "pre");
assert_eq!(document.text_content(pre).as_deref(), Some(payload));
assert_eq!(document.document().unwrap().content_type(), content_type);
for tag in ["b", "script"] {
assert!(
document
.elements_by_tag_name_ns(
document.document_node_id(),
Some(HTML_NS),
tag,
true
)
.is_empty()
);
}
}
}
#[test]
fn html_document_still_parses_markup_and_entities() {
let document = parse_test_document("<b>Gülçek&amp;</b>");
let bold = first_element_by_ns(&document, HTML_NS, "b");
assert_eq!(document.text_content(bold).as_deref(), Some("Gülçek&"));
assert_eq!(document.document().unwrap().content_type(), "text/html");
}
fn parse_test_document(html: &str) -> NativeDom {
HtmlParser::SCRIPTING_ENABLED.parse(
Url::parse("https://example.test/").expect("test url"),
moli-parser/src/session.rs
+12 -2
View File
@@ -167,6 +167,16 @@ impl TokenSink for EmbedderPausingTreeBuilder {
}
impl HtmlParserSession {
pub(super) fn initialize_text_document(&self) {
// Parse only the browser-owned shell. Response bytes enter the tokenizer
// after it has switched to plaintext, so tags and entities stay literal.
self.process(StrTendril::from(concat!(
"<!doctype html><html><head></head><body>",
"<pre style=\"word-wrap: break-word; white-space: pre-wrap;\">\n"
)));
self.tokenizer.set_plaintext_state();
}
fn new(sink: DocumentSink, opts: ParseOpts) -> Self {
let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);
Self {
@@ -352,10 +362,10 @@ fn feed_with_definitive_encoding(
pub(super) fn new_html_tree_sink_session(
target: ParserStreamHtmlTreeSinkTarget,
scripting_enabled: bool,
options: ParseOpts,
) -> HtmlTreeSinkSession {
let sink = DocumentSink::new(target);
let parser = HtmlParserSession::new(sink, html_parse_opts_with_scripting(scripting_enabled));
let parser = HtmlParserSession::new(sink, options);
let script_input = ParserInputQueue::default();
HtmlTreeSinkSession {
moli-parser/src/stream.rs
+15 -1
View File
@@ -429,11 +429,25 @@ pub(crate) fn prepare_parser_script_handoff_for_static_document(
}
impl HtmlTreeSinkStream {
pub(super) fn initialize_text_document(&self) {
self.parser.initialize_text_document();
}
pub(super) fn from_target_with_scripting(
target: ParserStreamHtmlTreeSinkTarget,
scripting_enabled: bool,
) -> Self {
let session = new_html_tree_sink_session(target, scripting_enabled);
Self::from_target_with_options(
target,
super::session::html_parse_opts_with_scripting(scripting_enabled),
)
}
pub(super) fn from_target_with_options(
target: ParserStreamHtmlTreeSinkTarget,
options: html5ever::ParseOpts,
) -> Self {
let session = new_html_tree_sink_session(target, options);
Self {
parser: session.parser,
script_input: session.script_input,
moli-protocol-server/src/protocol_server/tests/classic.rs
+93
View File
@@ -3,6 +3,99 @@ use axum::http::{HeaderMap, header};
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use serde_json::{Map, Value};
#[tokio::test]
async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {
const PAYLOAD: &str =
"<meta charset='gbk'><script>window.executed=42;</script><b>literal&amp;Gülçek</b>";
let cases = [
(
"plain",
"text/plain; charset=utf-8",
"text/plain",
true,
PAYLOAD,
),
(
"plain-bom",
"text/plain; charset=utf-8",
"text/plain",
true,
"\u{feff}\u{feff}",
),
(
"json",
"application/json",
"application/json",
true,
PAYLOAD,
),
(
"javascript",
"text/javascript; charset=utf-8",
"text/javascript",
true,
PAYLOAD,
),
(
"html",
"text/html; charset=utf-8",
"text/html",
false,
PAYLOAD,
),
];
let mut fixture = axum::Router::new();
for (name, mime, _, _, payload) in cases {
fixture = fixture.route(
&format!("/{name}"),
axum::routing::get(move || async move { ([(header::CONTENT_TYPE, mime)], payload) }),
);
}
fixture = fixture.route(
"/frame/{kind}",
axum::routing::get(
|axum::extract::Path(kind): axum::extract::Path<String>| async move {
(
[(header::CONTENT_TYPE, "text/html")],
format!("<iframe src='/{kind}'></iframe>"),
)
},
),
);
let (addr, _server) = spawn_dedicated_fixture_server(fixture, "document-mime");
let app = build_router(test_state());
let session = classic_request_json(app.clone(), Method::POST, "/session").await;
let session_id = session["value"]["sessionId"].as_str().unwrap();
for (name, _, content_type, literal, payload) in cases {
for prefix in ["", "frame/"] {
let navigated = classic_request_json_with_body(
app.clone(),
Method::POST,
&format!("/session/{session_id}/url"),
json!({"url":format!("http://{addr}/{prefix}{name}")}),
)
.await;
assert_eq!(navigated, json!({"value":null}), "{prefix}{name}");
let observed = classic_request_json_with_body(
app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"),
json!({"script": "const w=document.querySelector('iframe')?.contentWindow ?? window; return [w.document.body.textContent,w.executed??null,w.document.querySelectorAll('script').length,w.document.contentType];", "args": []}),
).await;
let expected = if literal {
json!([
payload.strip_prefix('\u{feff}').unwrap_or(payload),
null,
0,
content_type
])
} else {
json!(["literal&Gülçek", 42, 1, content_type])
};
assert_eq!(observed["value"], expected, "{prefix}{name}");
}
}
classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
}
#[tokio::test]
async fn webdriver_classic_status_session_and_delete_routes_use_value_envelope() {
let app = build_router(test_state());
moli-protocol/src/domains/emulation/tests.rs
+11 -7
View File
@@ -573,7 +573,9 @@ async fn admitted_idle_override_is_visible_to_concurrent_same_site_navigation()
listener,
Router::new().route(
"/",
get(|| async { "<!doctype html><title>idle navigation</title>" }),
get(|| async {
axum::response::Html("<!doctype html><title>idle navigation</title>")
}),
),
)
.await
@@ -644,7 +646,9 @@ async fn idle_override_updates_idle_detector_and_clear_restores_actual_state() {
listener,
Router::new().route(
"/",
get(|| async { "<!doctype html><title>idle detector</title>" }),
get(|| async {
axum::response::Html("<!doctype html><title>idle detector</title>")
}),
),
)
.await
@@ -2219,7 +2223,7 @@ async fn emulation_async_dispatch_updates_live_page_user_agent_and_xhr_header()
#[tokio::test(flavor = "multi_thread")]
async fn emulation_async_dispatch_updates_live_page_surface_without_mutating_accept_language() {
async fn page_handler() -> impl IntoResponse {
"<!doctype html><html><body>ok</body></html>"
axum::response::Html("<!doctype html><html><body>ok</body></html>")
}
async fn xhr_handler(
@@ -2711,9 +2715,9 @@ async fn locale_override_updates_intl_without_mutating_language_surfaces() {
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
format!(
axum::response::Html(format!(
"<!doctype html><html><body data-accept-language=\"{accept_language}\"><script>document.body.textContent = [Intl.DateTimeFormat().resolvedOptions().locale, navigator.language, navigator.languages.join(','), document.body.dataset.acceptLanguage].join('|');</script></body></html>"
)
))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -2767,9 +2771,9 @@ async fn bidi_user_context_locale_composes_with_user_agent_on_all_identity_surfa
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
format!(
axum::response::Html(format!(
"<!doctype html><html><body data-user-agent=\"{user_agent}\" data-accept-language=\"{accept_language}\"><script>document.body.textContent = [navigator.userAgent, Intl.DateTimeFormat().resolvedOptions().locale, navigator.language, navigator.languages.join(','), document.body.dataset.acceptLanguage].join('|');</script></body></html>"
)
))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
moli-protocol/src/domains/page/tests/lifecycle.rs
+12 -4
View File
@@ -2391,11 +2391,15 @@ async fn network_navigations_use_unique_document_loader_ids() {
axum::Router::new()
.route(
"/first",
axum::routing::get(|| async { "<!doctype html><main>first</main>" }),
axum::routing::get(|| async {
axum::response::Html("<!doctype html><main>first</main>")
}),
)
.route(
"/second",
axum::routing::get(|| async { "<!doctype html><main>second</main>" }),
axum::routing::get(|| async {
axum::response::Html("<!doctype html><main>second</main>")
}),
),
)
.await
@@ -2474,11 +2478,15 @@ async fn navigations_without_network_domain_still_use_unique_loader_ids() {
axum::Router::new()
.route(
"/first",
axum::routing::get(|| async { "<!doctype html><main>first</main>" }),
axum::routing::get(|| async {
axum::response::Html("<!doctype html><main>first</main>")
}),
)
.route(
"/second",
axum::routing::get(|| async { "<!doctype html><main>second</main>" }),
axum::routing::get(|| async {
axum::response::Html("<!doctype html><main>second</main>")
}),
),
)
.await
moli-protocol/src/domains/page/tests/runtime.rs
+6 -4
View File
@@ -1269,20 +1269,22 @@ async fn runtime_evaluate_child_history_back_emits_child_frame_navigation_and_li
.route(
"/main",
axum::routing::get(|| async {
"<iframe id='child' name='child-frame' src='/initial'></iframe>"
axum::response::Html(
"<iframe id='child' name='child-frame' src='/initial'></iframe>",
)
}),
)
.route(
"/initial",
axum::routing::get(|| async { "<body>initial</body>" }),
axum::routing::get(|| async { axum::response::Html("<body>initial</body>") }),
)
.route(
"/history-a",
axum::routing::get(|| async { "<body>history-a</body>" }),
axum::routing::get(|| async { axum::response::Html("<body>history-a</body>") }),
)
.route(
"/history-b",
axum::routing::get(|| async { "<body>history-b</body>" }),
axum::routing::get(|| async { axum::response::Html("<body>history-b</body>") }),
),
)
.await
moli-protocol/src/domains/target/tests/tests_background_staging.rs
+5 -3
View File
@@ -3037,9 +3037,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
format!(
axum::response::Html(format!(
"<!doctype html><html><body data-accept-language=\"{accept_language}\"><script>document.body.textContent = [navigator.language, document.body.dataset.acceptLanguage].join('|');</script></body></html>"
)
))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -3204,7 +3204,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat
#[tokio::test(flavor = "multi_thread")]
async fn same_context_background_session_can_clear_its_own_timezone_before_activation() {
async fn handler() -> impl IntoResponse {
"<!doctype html><html><body><script>document.body.textContent = Intl.DateTimeFormat().resolvedOptions().timeZone;</script></body></html>"
axum::response::Html(
"<!doctype html><html><body><script>document.body.textContent = Intl.DateTimeFormat().resolvedOptions().timeZone;</script></body></html>",
)
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
moli-renderer-v8/src/dom_parser.rs
+5 -1
View File
@@ -367,7 +367,11 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>(
source: &'a str,
content_type: Option<&str>,
) -> std::borrow::Cow<'a, str> {
if source == "\u{feff}" && !content_type.is_some_and(is_dom_parser_xml_mime) {
if source == "\u{feff}"
&& !content_type.is_some_and(|mime| {
is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime)
})
{
std::borrow::Cow::Borrowed("<body>\u{feff}</body>")
} else {
std::borrow::Cow::Borrowed(source)
moli-renderer-v8/src/live_document_parser.rs
+23
View File
@@ -716,6 +716,14 @@ impl DocumentParserSession {
)
}
pub(crate) fn start_main_text_document(document_url: Url, content_type: &str) -> Self {
Self::new_html(
HtmlParser::with_scripting_enabled(false)
.start_text_document(document_url, content_type),
DocumentParserLifetime::Finite,
)
}
pub(crate) fn start_finite_live_document(
document_url: Url,
document_handle: NativeNodeId,
@@ -728,6 +736,21 @@ impl DocumentParserSession {
)
}
pub(crate) fn start_finite_live_text_document(
document_url: Url,
document_handle: NativeNodeId,
owner: &mut impl LiveDocumentParserOwner,
) -> Self {
Self::new_html(
HtmlParser::with_scripting_enabled(false).start_live_text_document_root(
document_url,
document_handle,
owner,
),
DocumentParserLifetime::Finite,
)
}
pub(crate) fn start_finite_live_xml_document(
document_url: Url,
document_handle: NativeNodeId,
moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
+1
View File
@@ -239,6 +239,7 @@ impl JsContextHost {
parser_base_url,
source.as_ref(),
is_xml_document,
snapshot.content_type.as_deref(),
);
Some(ChildDocumentInstallResult {
initial_classic_ready_work: parser_start.initial_classic_ready_work,
moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs
+8
View File
@@ -1695,6 +1695,7 @@ impl JsContextHost {
document_base_url: Url,
markup: &str,
is_xml_document: bool,
content_type: Option<&str>,
) -> ChildLiveDocumentParserStartResult {
let owner = FrameDocumentOwner::new(owner_local_window_id, owner_document_id);
self.child_document_parsers.clear(owner);
@@ -1703,6 +1704,13 @@ impl JsContextHost {
document_base_url,
document_handle,
)
} else if content_type.is_some_and(moli_web_mime::is_text_document_mime) {
let mut parser_owner = ChildFrameLiveParserOwner::new(self, scope, document_handle);
DocumentParserSession::start_finite_live_text_document(
document_base_url,
document_handle,
&mut parser_owner,
)
} else {
DocumentParserSession::start_finite_live_document(
document_base_url,
moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
+20 -3
View File
@@ -21,7 +21,8 @@ use crate::{
LoadedChildDocument, SubresourceResponseBody,
},
};
use moli_encoding::decode_html_document_with_fallback;
use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback};
use moli_encoding_detector::detect_legacy_html_encoding;
pub(crate) struct AppliedChildDocumentLoadCompletion {
/// Initial parser-classic work produced by the committed child document.
@@ -757,8 +758,24 @@ fn child_document_load_outcome_from_response(
let body_bytes = response_body
.try_bytes()
.map_err(|error| format!("failed to read child document response body: {error}"))?;
let (markup, character_set) =
decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback));
let (markup, character_set) = if let Some(mime) = content_type
.as_deref()
.filter(|mime| moli_web_mime::is_text_document_mime(mime))
{
let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
&head.headers,
head.final_url.as_str(),
detect_legacy_html_encoding,
moli_web_mime::is_json_module_mime(mime) || mime == "text/json",
);
let mut markup = decoder.push(&body_bytes).concat();
if let Some(tail) = decoder.finish() {
markup.push_str(&tail);
}
(markup, decoder.document_encoding_name())
} else {
decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback))
};
(markup, character_set.to_owned())
};
let policy_container =
moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
+2
View File
@@ -25,6 +25,7 @@ impl ConcurrentParseTimeRuntime {
let (page_vm, triggered_navigation) = {
let buffered_document_preloads = &mut state.buffered_document_preloads;
let service_worker_preload_context = state.service_worker_preload_context.as_ref();
let document_character_set = &state.document_character_set;
PageVm::new_from_parser_stream_and_run_document_start(
page_id,
local_executor,
@@ -34,6 +35,7 @@ impl ConcurrentParseTimeRuntime {
&mut state.parser_session,
started,
|page_vm| {
page_vm.set_document_character_set(document_character_set.clone());
admit_pending_preloads(
page_vm,
buffered_document_preloads,
moli-renderer-v8/src/runtime/phase_one/state.rs
+20
View File
@@ -11,6 +11,7 @@ pub(super) struct ParseTimeDriverState {
pub(super) buffered_document_preloads: Box<BufferedDocumentPreloadState>,
pub(super) service_worker_preload_context: Option<ServiceWorkerScriptPreloadContext>,
pub(super) input_closed: bool,
pub(super) is_text_document: bool,
}
impl ParseTimeDriverState {
@@ -32,6 +33,24 @@ impl ParseTimeDriverState {
buffered_document_preloads: Box::default(),
service_worker_preload_context: None,
input_closed: false,
is_text_document: false,
}
}
pub(super) fn new_text(final_url: Url, mime: &str) -> Self {
Self {
parser_session: DocumentParserSession::start_main_text_document(
final_url.clone(),
mime,
),
final_url,
document_character_set: "UTF-8".to_owned(),
scheduler: DocumentScriptScheduler::new(),
pending_parsing_blocking_script: PendingParsingBlockingClassicScriptRunner::empty(),
buffered_document_preloads: Box::default(),
service_worker_preload_context: None,
input_closed: false,
is_text_document: true,
}
}
@@ -45,6 +64,7 @@ impl ParseTimeDriverState {
buffered_document_preloads: Box::default(),
service_worker_preload_context: None,
input_closed: false,
is_text_document: false,
}
}
moli-renderer-v8/src/runtime/phase_one/streaming.rs
+144 -18
View File
@@ -94,10 +94,8 @@ impl ConcurrentParseTimeRuntime {
env.apply_navigation_response_headers(&response_final_url, &response_headers);
let mut body_source = RawDocumentBodySource::fetch_response(response);
let mut state = ParseTimeDriverState::new_with_scripting_enabled(
response_final_url,
main_document_parser_scripting_enabled(&env),
);
let (mut state, mut decoder) =
response_document_parser(response_final_url, &response_headers, &env);
state
.buffered_document_preloads
.set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission(
@@ -124,11 +122,6 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
&response_headers,
state.final_url.as_str(),
detect_legacy_html_encoding,
);
// Raw navigation bodies are decoded during prebootstrap scan. The decoder
// is carried forward so split multibyte sequences are not decoded twice or
// lost between the scan and parser handoff.
@@ -332,10 +325,7 @@ impl ConcurrentParseTimeRuntime {
},
)));
}
let mut state = ParseTimeDriverState::new_with_scripting_enabled(
final_url,
main_document_parser_scripting_enabled(&env),
);
let (mut state, mut decoder) = response_document_parser(final_url, &response_headers, &env);
state
.buffered_document_preloads
.set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission(
@@ -362,11 +352,6 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
&response_headers,
state.final_url.as_str(),
detect_legacy_html_encoding,
);
// External raw bodies replay captured chunks. Pre-scan only what is already
// buffered before bootstrap so the producer's backpressure boundary still
// controls how far ahead the parser can get.
@@ -451,6 +436,10 @@ impl ConcurrentParseTimeRuntime {
chunk: String,
service_worker_context: Option<&ServiceWorkerScriptPreloadContext>,
) {
if self.state.is_text_document {
self.state.parser_session.queue_arrived_chunk(chunk);
return;
}
self.state
.buffered_document_preloads
.append_to_main_document_scan_with_service_worker_context(
@@ -607,6 +596,9 @@ fn scan_prebootstrap_html_chunk_into_state(
chunk: &str,
service_worker_context: Option<&ServiceWorkerScriptPreloadContext>,
) {
if state.is_text_document {
return;
}
state
.buffered_document_preloads
.append_to_main_document_prebootstrap_scan_with_service_worker_context(
@@ -617,6 +609,41 @@ fn scan_prebootstrap_html_chunk_into_state(
);
}
fn response_document_parser(
final_url: Url,
headers: &[(String, String)],
env: &PageVmEnvConfig,
) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) {
let content_type = moli_web_mime::response_document_content_type(headers);
let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime));
let mut state = if let Some(mime) = &text_type {
ParseTimeDriverState::new_text(final_url, mime)
} else {
ParseTimeDriverState::new_with_scripting_enabled(
final_url,
main_document_parser_scripting_enabled(env),
)
};
let decoder = if let Some(mime) = text_type {
HtmlDocumentStreamingDecoder::new_text_document(
headers,
state.final_url.as_str(),
detect_legacy_html_encoding,
moli_web_mime::is_json_module_mime(&mime) || mime == "text/json",
)
} else {
HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
headers,
state.final_url.as_str(),
detect_legacy_html_encoding,
)
};
// Headers/defaults are observable even when no body chunk is ready yet.
// A later BOM or decoder decision can still replace this tentative value.
sync_state_document_character_set_from_decoder(&mut state, &decoder);
(state, decoder)
}
pub(super) fn enqueue_streaming_raw_chunk(
runtime: &mut ConcurrentParseTimeRuntime,
decoder: &mut HtmlDocumentStreamingDecoder,
@@ -908,6 +935,105 @@ mod tests {
use crate::parser::ScriptSource;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
#[tokio::test]
async fn text_document_charset_is_visible_to_document_start_before_body_arrives() {
let _js_runtime_owner = crate::JsRuntime::initialize();
tokio::task::LocalSet::new().run_until(async {
for (mime, chunk, expected) in [
("text/plain; charset=gbk", None, "GBK"),
("application/json", None, "UTF-8"),
("text/plain; charset=gbk", Some(b"\xef\xbb\xbfready".as_slice()), "UTF-8"),
] {
let loader_owner = ResourceRequestClient::new(&FetchConfig::default()).unwrap();
let loader = loader_owner.handle();
let mut env = default_test_page_vm_env_config();
env.document_start_scripts.push(crate::DocumentStartScript {
registry_key: None,
devtools_session: None,
source: "document.title = document.characterSet".to_owned(),
world_name: None,
has_bidi_channel_argument: false,
bidi_channel_handoffs: Vec::new(),
});
let (mut state, mut decoder) = response_document_parser(
Url::parse("https://example.test/data").unwrap(),
&[("Content-Type".to_owned(), mime.to_owned())], &env,
);
if let Some(chunk) = chunk {
let chunks = decoder.push(chunk);
sync_state_document_character_set_from_decoder(&mut state, &decoder);
for chunk in chunks {
state.parser_session.queue_arrived_chunk(chunk);
}
}
assert_eq!(state.document_character_set, expected);
let (_, page_vm, navigated) = ConcurrentParseTimeRuntime::bootstrap_page_vm_from_state_on_fresh_local_task(
PageId::new_for_testing(1), JsLocalExecutor::new(), loader, env,
PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(), state,
Instant::now(), "charset test bootstrap closed",
).await.unwrap();
assert!(!navigated);
assert_eq!(page_vm.vm().snapshot_live_document().document_title(), expected, "{mime}");
}
}).await;
}
#[test]
fn response_document_dispatch_distinguishes_text_from_html_and_xml() {
for (mime, text) in [
("application/json", true),
("application/problem+json", true),
("text/plain; charset=gbk", true),
("text/javascript", true),
("text/html", false),
("application/xhtml+xml", false),
("application/xml", false),
("text/xml", false),
] {
let (state, _) = response_document_parser(
Url::parse("https://example.test/data").unwrap(),
&[("Content-Type".to_owned(), mime.to_owned())],
&default_test_page_vm_env_config(),
);
assert_eq!(state.is_text_document, text, "{mime}");
}
}
#[tokio::test]
async fn text_document_does_not_preload_literal_script_tags() {
let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default())
.expect("default loader");
let runtime_hooks = PageVmRuntimeHooks::standalone_without_owner_reservation_for_test();
for (mime, expected_preload) in [
("text/plain", false),
("application/json", false),
("text/html", true),
] {
let (mut state, _) = response_document_parser(
Url::parse("https://example.test/data").unwrap(),
&[("Content-Type".to_owned(), mime.to_owned())],
&default_test_page_vm_env_config(),
);
state.buffered_document_preloads.bind_resource_runtime(
runtime_hooks.owner_wake(),
runtime_hooks.resource_task_runner(),
);
scan_prebootstrap_html_chunk_into_state(
&mut state,
&loader,
"<script defer src='/literal.js'></script>",
None,
);
assert_eq!(
state.buffered_document_preloads.entries.contains_key(
&classic_preload_key_for_streaming_test("https://example.test/literal.js")
),
expected_preload,
"{mime}",
);
}
}
#[tokio::test]
async fn external_raw_document_body_source_collects_chunks_and_completion() {
let (completion_tx, completion_rx) = oneshot::channel();
moli-web-mime/src/classification.rs
+8
View File
@@ -51,6 +51,14 @@ pub fn is_text_mime_essence(input: &str) -> bool {
input.starts_with("text/")
}
/// Navigation responses rendered as literal text rather than HTML or XML.
/// This also includes JSON and JavaScript resources opened as documents.
pub fn is_text_document_mime(input: &str) -> bool {
!is_html_document_mime(input)
&& !is_dom_parser_xml_mime(input)
&& (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input))
}
/// Whether a `style` element's raw `type` attribute selects classic CSS.
///
/// HTML's update-a-style-block algorithm compares the untrimmed attribute
moli-web-mime/src/lib.rs
+3 -3
View File
@@ -14,9 +14,9 @@ pub use classification::{
is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime,
is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence,
is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence,
is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, is_text_mime,
is_text_mime_essence, is_video_mime, is_video_mime_essence, is_webassembly_mime,
multipart_form_data_boundary,
is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence,
is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime,
is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary,
};
pub use data_url::{
data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type,
moli-web-mime/src/tests.rs
+22
View File
@@ -648,3 +648,25 @@ fn script_like_mime_type_block_matches_fetch_response_rule() {
));
assert!(!should_script_like_response_be_blocked_due_to_mime_type(&[]));
}
#[test]
fn document_text_classification_excludes_html_and_xml() {
for mime in [
"text/plain",
"Text/Css; charset=utf-8",
"application/json",
"application/ld+json",
"application/javascript",
] {
assert!(crate::is_text_document_mime(mime), "{mime}");
}
for mime in [
"text/html",
"text/xml",
"application/xhtml+xml",
"image/svg+xml",
"image/png",
"application/octet-stream",
] {
assert!(!crate::is_text_document_mime(mime), "{mime}");
}
}
moli/src/fetch_dump/tests.rs
+2 -2
View File
@@ -1,7 +1,7 @@
use std::sync::Arc;
use anyhow::Result;
use axum::{Router, routing::get};
use axum::{Router, response::Html, routing::get};
use moli_core::{
LayoutPolicy,
page::Page,
@@ -36,7 +36,7 @@ async fn load_page_with_config(
"/",
get(move || {
let body = Arc::clone(&server_body);
async move { (*body).clone() }
async move { Html((*body).clone()) }
}),
);
axum::serve(listener, app).await.unwrap();