diff --git a/moli-core/tests/fetch_behaviors.rs b/moli-core/tests/fetch_behaviors.rs index ab6e690dd1b..7f7c62be64f 100644 --- a/moli-core/tests/fetch_behaviors.rs +++ b/moli-core/tests/fetch_behaviors.rs @@ -12,6 +12,8 @@ use tokio::time::Duration; #[path = "fetch_behaviors/cache_timing.rs"] mod cache_timing; +#[path = "fetch_behaviors/document_encoding.rs"] +mod document_encoding; #[path = "fetch_behaviors/document_mime.rs"] mod document_mime; #[path = "fetch_behaviors/preload_as.rs"] diff --git a/moli-core/tests/fetch_behaviors/document_encoding.rs b/moli-core/tests/fetch_behaviors/document_encoding.rs new file mode 100644 index 00000000000..fa15223c595 --- /dev/null +++ b/moli-core/tests/fetch_behaviors/document_encoding.rs @@ -0,0 +1,137 @@ +use super::*; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +async fn serve(mut socket: tokio::net::TcpStream) -> Result<()> { + let mut request = Vec::new(); + while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { + let mut buffer = [0; 4096]; + let count = socket.read(&mut buffer).await?; + if count == 0 { + return Ok(()); + } + request.extend_from_slice(&buffer[..count]); + } + let request = String::from_utf8(request)?; + let path = request + .split_whitespace() + .nth(1) + .context("request target")?; + let url = url::Url::parse(&format!("http://fixture{path}"))?; + let parameter = |name: &str| { + url.query_pairs() + .find(|(key, _)| key == name) + .map(|(_, value)| value.into_owned()) + }; + let mut extra = String::new(); + let (mime, body) = match url.path() { + "/encoding-parent" => { + let charset = parameter("charset").unwrap_or_else(|| "windows-1253".to_owned()); + let source = "
"; + let body = match charset.as_str() { + "UTF-16LE" => source.encode_utf16().flat_map(u16::to_le_bytes).collect(), + "UTF-16BE" => source.encode_utf16().flat_map(u16::to_be_bytes).collect(), + _ => source.as_bytes().to_vec(), + }; + (format!("text/html;charset={charset}"), body) + } + "/encoding-parent.js" => ( + "text/javascript;charset=utf-8".to_owned(), + include_bytes!("../fixtures/document-encoding-parent.js").to_vec(), + ), + "/encoding-child" => { + let mime = parameter("header").map_or_else( + || "text/html".to_owned(), + |charset| format!("text/html;charset={charset}"), + ); + let meta = parameter("meta") + .map_or_else(String::new, |charset| format!("")); + let token = serde_json::to_string(¶meter("token").unwrap_or_default())?; + let report = format!( + "" + ); + let mut body = format!("{meta}").into_bytes(); + body.push(0xa2); + body.extend_from_slice(format!("
{report}").as_bytes()); + if parameter("bom").as_deref() == Some("utf8") { + body.splice(0..0, [0xef, 0xbb, 0xbf]); + } + if let Some(csp) = parameter("csp") { + extra.push_str(&format!("Content-Security-Policy: {csp}\r\n")); + } + (mime, body) + } + "/redirect" => { + extra.push_str(&format!( + "Location: {}\r\n", + parameter("to").context("redirect target")? + )); + ("text/html".to_owned(), Vec::new()) + } + _ => ( + "text/html;charset=utf-8".to_owned(), + b"root".to_vec(), + ), + }; + let status = if url.path() == "/redirect" { + "302 Found" + } else { + "200 OK" + }; + let head = format!( + "HTTP/1.1 {status}\r\nContent-Type: {mime}\r\nContent-Length: {}\r\nCache-Control: no-store\r\n{extra}Connection: close\r\n\r\n", + body.len() + ); + socket.write_all(head.as_bytes()).await?; + socket.write_all(&body).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn document_encoding_inherits_only_from_the_eligible_container() -> Result<()> { + let local = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let remote = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let origins = [ + format!("http://{}", local.local_addr()?), + format!("http://{}", remote.local_addr()?), + ]; + let (shutdown, mut shutting_down) = tokio::sync::oneshot::channel::<()>(); + let server = tokio::spawn(async move { + let mut requests = tokio::task::JoinSet::new(); + loop { + tokio::select! { + _ = &mut shutting_down => break, + accepted = async { tokio::select! { + accepted = local.accept() => accepted, + accepted = remote.accept() => accepted, + }} => { + let (socket, _) = accepted?; + requests.spawn(serve(socket)); + } + Some(result) = requests.join_next(), if !requests.is_empty() => { result??; } + } + } + requests.shutdown().await; + Ok::<_, anyhow::Error>(()) + }); + let result = tokio::time::timeout(Duration::from_secs(30), async { + let browser = Browser::new(AppConfig::default())?; + let mut page = browser.fetch(&origins[0]).await?; + let script = format!( + "(globalThis.encodingOrigins={}, {})", + serde_json::to_string(&origins)?, + include_str!("../fixtures/document-encoding-inheritance.js") + ); + let result = page + .evaluate_runtime_expression_with_await_async(&script, true) + .await?; + let observed: serde_json::Value = + serde_json::from_str(result["value"].as_str().context("encoding result")?)?; + assert_eq!(observed["errors"], serde_json::json!([])); + assert_eq!(observed["observations"].as_array().unwrap().len(), 27); + Ok::<_, anyhow::Error>(()) + }) + .await; + let _ = shutdown.send(()); + server.await??; + result? +} diff --git a/moli-core/tests/fixtures/document-encoding-inheritance.js b/moli-core/tests/fixtures/document-encoding-inheritance.js new file mode 100644 index 00000000000..cdb27017b1c --- /dev/null +++ b/moli-core/tests/fixtures/document-encoding-inheritance.js @@ -0,0 +1,69 @@ +(async () => { + const [origin, other] = globalThis.encodingOrigins || globalThis.__fetchOrigins; + const cases = [ + ['same-origin container', {}, 'windows-1253', 'Ά'], + ['different nested encoding', {charset: 'windows-1251'}, 'windows-1251', 'ў'], + ['cross-origin child', {childOrigin: other}, 'windows-1252', '¢'], + ['cross-origin parent with same-origin child', {parentOrigin: other, childOrigin: other}, 'windows-1253', 'Ά'], + ['cross-origin parent with top-origin child', {parentOrigin: other, childOrigin: origin}, 'windows-1252', '¢'], + ['redirect to other origin', {childOrigin: other, redirectOrigin: origin}, 'windows-1252', '¢'], + ['redirect to parent origin', {redirectOrigin: other}, 'windows-1253', 'Ά'], + ['opaque child', {sandbox: 'allow-scripts'}, 'windows-1252', '¢'], + ['same-origin sandbox child', {sandbox: 'allow-scripts allow-same-origin'}, 'windows-1253', 'Ά'], + ['CSP opaque child', {csp: 'sandbox allow-scripts'}, 'windows-1252', '¢'], + ['opaque parent', {parentSandbox: 'allow-scripts'}, 'windows-1252', '¢'], + ['UTF-16LE parent', {charset: 'UTF-16LE'}, 'windows-1252', '¢'], + ['UTF-16BE parent', {charset: 'UTF-16BE'}, 'windows-1252', '¢'], + ['transport wins', {header: 'windows-1251'}, 'windows-1251', 'ў'], + ['meta wins', {meta: 'windows-1251'}, 'windows-1251', 'ў'], + ['invalid transport inherits', {header: 'not-an-encoding'}, 'windows-1253', 'Ά'], + ['cross-origin invalid transport', {header: 'not-an-encoding', childOrigin: other}, 'windows-1252', '¢'], + ['BOM wins', {bom: 'utf8'}, 'UTF-8', '�'], + ['data URL is opaque', {mode: 'data'}, 'windows-1252', '¢'], + ['blob URL shares parent origin', {mode: 'blob'}, 'windows-1253', 'Ά'], + ['sandboxed blob', {mode: 'blob', sandbox: 'allow-scripts'}, 'windows-1252', '¢'], + ['srcdoc uses UTF-8', {mode: 'srcdoc'}, 'UTF-8', '¢'], + ['sandboxed srcdoc uses UTF-8', {mode: 'srcdoc', sandbox: 'allow-scripts'}, 'UTF-8', '¢'], + ['srcdoc from UTF-16 parent uses UTF-8', {mode: 'srcdoc', charset: 'UTF-16LE'}, 'UTF-8', '¢'], + ['popup has no container', {popup: '1'}, 'windows-1252', '¢'], + ['popup transport wins', {popup: '1', header: 'windows-1251'}, 'windows-1251', 'ў'], + ['popup blob has no container', {popup: '1', mode: 'blob'}, 'windows-1252', '¢'], + ]; + const errors = [], observations = []; + for (const [name, options, charset, text] of cases) { + const token = 'encoding-' + observations.length; + const frame = document.createElement('iframe'); + const url = new URL('/encoding-parent', options.parentOrigin || origin); + url.searchParams.set('token', token); + url.searchParams.set('charset', options.charset || 'windows-1253'); + for (const [key, value] of Object.entries(options)) url.searchParams.set(key, value); + if (options.parentSandbox) frame.setAttribute('sandbox', options.parentSandbox); + let listener, timer, parent; + try { + const observed = await new Promise((resolve, reject) => { + listener = event => { + if (event.data && event.data.token === token + ':parent') parent = event.data.charset; + if (event.data && event.data.token === token) resolve(event.data); + }; + addEventListener('message', listener); + timer = setTimeout(() => reject(new Error('child did not report')), 5000); + frame.src = url.href; + document.body.append(frame); + }); + observations.push({name, parent, observed}); + if (parent !== (options.charset || 'windows-1253') || observed.charset !== charset || observed.text !== text) { + errors.push({name, parent, observed, expected: {charset, text}}); + } + } catch (error) { + observations.push({name, parent, error: String(error)}); + errors.push({name, parent, error: String(error)}); + } finally { + clearTimeout(timer); + removeEventListener('message', listener); + if (frame.contentWindow) frame.contentWindow.postMessage('cleanup', '*'); + await new Promise(resolve => setTimeout(resolve, 0)); + frame.remove(); + } + } + return JSON.stringify({errors, observations}); +})() diff --git a/moli-core/tests/fixtures/document-encoding-parent.js b/moli-core/tests/fixtures/document-encoding-parent.js new file mode 100644 index 00000000000..09fa31fd286 --- /dev/null +++ b/moli-core/tests/fixtures/document-encoding-parent.js @@ -0,0 +1,38 @@ +(() => { + const q = new URLSearchParams(location.search); + const token = q.get('token'); + addEventListener('error', event => top.postMessage({token, error: event.message}, '*')); + top.postMessage({token: token + ':parent', charset: document.characterSet}, '*'); + const child = new URL('/encoding-child', q.get('childOrigin') || location.href); + for (const name of ['token', 'header', 'meta', 'csp', 'bom']) { + if (q.has(name)) child.searchParams.set(name, q.get(name)); + } + const report = ''; + const markup = '\xa2
' + report; + let target = child.href; + if (q.has('redirectOrigin')) { + target = q.get('redirectOrigin') + '/redirect?to=' + encodeURIComponent(target); + } + const mode = q.get('mode'); + if (mode === 'data') { + target = 'data:text/html,' + Array.from(markup, c => '%' + c.charCodeAt(0).toString(16).padStart(2, '0')).join(''); + } else if (mode === 'blob') { + target = URL.createObjectURL(new Blob([Uint8Array.from(markup, c => c.charCodeAt(0))], {type: 'text/html'})); + } + if (q.has('popup')) { + const popup = open(target, '_blank'); + addEventListener('message', event => { if (event.data === 'cleanup') popup.close(); }); + } else { + const frame = document.createElement('iframe'); + if (q.has('sandbox')) frame.setAttribute('sandbox', q.get('sandbox')); + if (mode === 'srcdoc') frame.srcdoc = markup; + else frame.src = target; + document.body.append(frame); + } + addEventListener('message', event => { + if (event.data === 'cleanup' && mode === 'blob') URL.revokeObjectURL(target); + }); +})() diff --git a/moli-core/tests/fixtures/document-response-mime.js b/moli-core/tests/fixtures/document-response-mime.js index 73fe791a0a5..23597b50218 100644 --- a/moli-core/tests/fixtures/document-response-mime.js +++ b/moli-core/tests/fixtures/document-response-mime.js @@ -1,21 +1,23 @@ (async () => { + // A missing/invalid charset inherits from an iframe's UTF-8 container; + // a top-level popup uses the browser default instead. const cases = [ - [['', 'text/plain'], 'text/plain', 'UTF-8', 'hi'], - [['text/plain', ''], 'text/plain', 'UTF-8', 'hi'], - [['text/html', 'text/plain'], 'text/plain', 'UTF-8', 'hi'], - [['text/plain;charset=gbk', 'text/html'], 'text/html', 'UTF-8', 'hi'], + [['', 'text/plain'], 'text/plain', null, 'hi'], + [['text/plain', ''], 'text/plain', null, 'hi'], + [['text/html', 'text/plain'], 'text/plain', null, 'hi'], + [['text/plain;charset=gbk', 'text/html'], 'text/html', null, 'hi'], [['text/plain;charset=gbk', 'text/html;charset=windows-1254'], 'text/html', 'windows-1254', 'Ğ'], [['text/plain;charset=gbk', 'text/plain'], 'text/plain', 'GBK', '家居'], [['text/html;charset=gbk', 'text/html;charset=utf-8', 'text/html'], 'text/html', 'GBK', '家居'], - [['text/html;charset=gbk', 'text/plain', 'text/html'], 'text/html', 'UTF-8', 'hi'], - [['text/plain', '*/*;charset=gbk'], 'text/plain', 'UTF-8', 'hi'], - [['text/html', '*/*'], 'text/html', 'UTF-8', 'hi'], - [['text/html;x="', 'text/plain'], 'text/html', 'UTF-8', 'hi'], + [['text/html;charset=gbk', 'text/plain', 'text/html'], 'text/html', null, 'hi'], + [['text/plain', '*/*;charset=gbk'], 'text/plain', null, 'hi'], + [['text/html', '*/*'], 'text/html', null, 'hi'], + [['text/html;x="', 'text/plain'], 'text/html', null, 'hi'], [['text/html;"', '\\"', 'text/plain', '";charset=GBK'], 'text/html', 'GBK', '家居'], - [['text/html;"', '"', 'text/plain'], 'text/plain', 'UTF-8', 'hi'], - [['application/octet-stream', 'text/html', 'invalid'], 'text/html', 'UTF-8', 'hi'], - [['text/html;charset=gbk', 'text/html;charset=unknown'], 'text/html', 'UTF-8', 'hi'], - [['text/html;charset=gbk', 'text/html;charset=""'], 'text/html', 'UTF-8', 'hi'], + [['text/html;"', '"', 'text/plain'], 'text/plain', null, 'hi'], + [['application/octet-stream', 'text/html', 'invalid'], 'text/html', null, 'hi'], + [['text/html;charset=gbk', 'text/html;charset=unknown'], 'text/html', null, 'hi'], + [['text/html;charset=gbk', 'text/html;charset=""'], 'text/html', null, 'hi'], ]; const errors = []; let checked = 0; @@ -24,7 +26,7 @@ for (const fields of [values, [values.join(',')]]) { const url = new URL('/mime', location.href); for (const field of fields) url.searchParams.append('value', field); - url.searchParams.set('encoding', charset.toLowerCase()); + url.searchParams.set('encoding', (charset || 'UTF-8').toLowerCase()); let frame, popup; try { const doc = await new Promise((resolve, reject) => { @@ -45,7 +47,7 @@ }); const observed = [doc.contentType, doc.characterSet.toUpperCase(), doc.body.firstChild.localName, doc.body.textContent]; - const expected = [type, charset.toUpperCase(), type === 'text/plain' ? 'pre' : 'b', + const expected = [type, (charset || (kind === 'iframe' ? 'UTF-8' : 'windows-1252')).toUpperCase(), type === 'text/plain' ? 'pre' : 'b', type === 'text/plain' ? `${text}\n` : `${text}\n`]; if (JSON.stringify(observed) !== JSON.stringify(expected)) { errors.push({kind, fields, observed, expected}); diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents.rs index efeeb34dbf5..055717ec1f8 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents.rs @@ -1,6 +1,7 @@ mod async_policy; mod bootstrap_commit; mod commit; +mod encoding; mod frame_owner_resource_timing; mod initial_empty; mod lifecycle; diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/encoding.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/encoding.rs new file mode 100644 index 00000000000..527ef0c8410 --- /dev/null +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/encoding.rs @@ -0,0 +1,67 @@ +use super::super::PageScriptHost; +use crate::document_runtime::DomHandle; +use moli_url::WebOrigin; +use url::Url; + +/// The container Document's encoding and origin, captured for this navigation. +/// An opener is not a container, and a response can introduce another origin +/// through a redirect or CSP sandboxing. +#[derive(Default)] +pub(super) struct ChildDocumentEncodingContext { + parent: Option<(WebOrigin, String)>, + sandbox_forces_opaque_origin: bool, +} + +impl ChildDocumentEncodingContext { + pub(super) fn fallback_encoding( + &self, + response_url: &Url, + content_type: Option<&str>, + response_forces_opaque_origin: bool, + ) -> Option<&str> { + if content_type.is_some_and(moli_web_mime::is_dom_parser_xml_mime) { + return Some("UTF-8"); + } + if self.sandbox_forces_opaque_origin || response_forces_opaque_origin { + return None; + } + let (origin, charset) = self.parent.as_ref()?; + if !origin.same_origin_url(response_url) { + return None; + } + let encoding = moli_encoding::encoding_for_label(charset)?; + // HTML permits inheritance from the same-origin container except for + // UTF-16, which would reinterpret the child's markup byte structure. + (!matches!(encoding.name(), "UTF-16LE" | "UTF-16BE")).then_some(charset.as_str()) + } +} + +impl PageScriptHost { + pub(super) fn child_document_container_character_set(&self, handle: DomHandle) -> Option<&str> { + let parent = self.dom_host().node(handle)?.owner_document()?; + // SAFETY: PageScriptHost is owned by the ScriptVm that owns this DocumentRuntime. + unsafe { &*self.document_runtime }.document_character_set_for_handle(parent) + } + + pub(super) fn child_document_encoding_context( + &self, + handle: DomHandle, + ) -> ChildDocumentEncodingContext { + let parent = self + .owner_dispatch_scope_for_node(handle) + .and_then(|owner| self.window_access_origin_for_dispatch_scope(owner)) + .zip(self.child_document_container_character_set(handle)) + .map(|(origin, charset)| { + ( + WebOrigin::from_serialized(&origin.serialized_origin()), + charset.to_owned(), + ) + }); + ChildDocumentEncodingContext { + parent, + sandbox_forces_opaque_origin: self + .child_browsing_context_sandbox_policy_from_owner(handle) + .forces_opaque_origin, + } + } +} diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs index b95afd1df02..6ea54ff1f1f 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs @@ -260,7 +260,7 @@ impl PageScriptHost { browser_context_runtime, resource_loader, completion_tx, - parent_character_set: self.document_character_set().to_owned(), + encoding_context: self.child_document_encoding_context(handle), }; let paused_fetch = self.start_child_document_fetch(handle, initiating_owner, &initiating_loader, fetch); @@ -1129,7 +1129,7 @@ pub(crate) struct ChildDocumentFetch { browser_context_runtime: crate::runtime::RendererBrowserContextRuntime, resource_loader: crate::network::navigation::NavigationResourceLoader, completion_tx: crate::page_task_queue::RendererResourceCompletionSender, - parent_character_set: String, + encoding_context: super::encoding::ChildDocumentEncodingContext, } impl std::fmt::Debug for ChildDocumentFetch { @@ -1460,7 +1460,7 @@ impl ChildDocumentFetch { head, body, navigation_url, - &self.parent_character_set, + &self.encoding_context, self.download_policy, self.response_body.take(), resource_timing_status, @@ -1526,7 +1526,7 @@ fn child_document_load_outcome_from_response( head: moli_fetch::ResponseHead, body: moli_fetch::ResponseBody, navigation_url: url::Url, - parent_character_set: &str, + encoding_context: &super::encoding::ChildDocumentEncodingContext, download_policy: ChildNavigationDownloadPolicy, download_body: Option