diff --git a/Cargo.lock b/Cargo.lock index e16d6ebe4d..527cb6aff6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -685,7 +685,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf 0.13.1", + "phf", "serde", "smallvec", ] @@ -1142,16 +1142,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "futf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" -dependencies = [ - "mac", - "new_debug_unreachable", -] - [[package]] name = "futures-channel" version = "0.3.32" @@ -1379,13 +1369,12 @@ dependencies = [ [[package]] name = "html5ever" -version = "0.35.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55d958c2f74b664487a2035fe1dadb032c48718a03b63f3ab0b8537db8549ed4" +checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8" dependencies = [ "log", - "markup5ever 0.35.0", - "match_token", + "markup5ever", ] [[package]] @@ -1875,12 +1864,6 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" -[[package]] -name = "mac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" - [[package]] name = "malloc_size_of_derive" version = "0.1.3" @@ -1892,17 +1875,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "markup5ever" -version = "0.35.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "311fe69c934650f8f19652b3946075f0fc41ad8757dbb68f1ca14e7900ecc1c3" -dependencies = [ - "log", - "tendril 0.4.3", - "web_atoms 0.1.3", -] - [[package]] name = "markup5ever" version = "0.39.0" @@ -1910,19 +1882,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de" dependencies = [ "log", - "tendril 0.5.0", - "web_atoms 0.2.3", -] - -[[package]] -name = "match_token" -version = "0.35.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac84fd3f360fcc43dc5f5d186f02a94192761a080e8bc58621ad4d12296a58cf" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", + "tendril", + "web_atoms", ] [[package]] @@ -2415,7 +2376,7 @@ version = "0.1.0" dependencies = [ "anyhow", "html5ever", - "markup5ever 0.35.0", + "markup5ever", "moli-dom", "moli-fetch", "moli-page-types", @@ -2562,7 +2523,7 @@ dependencies = [ "http", "httpdate", "indexmap", - "markup5ever 0.35.0", + "markup5ever", "moli-broadcast-channel", "moli-browser-profile", "moli-canvas", @@ -2942,7 +2903,7 @@ name = "moli-xpath" version = "0.1.0" dependencies = [ "log", - "markup5ever 0.35.0", + "markup5ever", ] [[package]] @@ -3234,7 +3195,7 @@ version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c50246449a5fa669debd2debeb90be4c30f0a3a2e954f852ec40e5ef49701285" dependencies = [ - "phf 0.13.1", + "phf", "proc-macro2", "quote", "syn 2.0.117", @@ -3337,7 +3298,7 @@ dependencies = [ "oxc_diagnostics", "oxc_span", "oxc_str", - "phf 0.13.1", + "phf", "rustc-hash", "unicode-id-start", ] @@ -3384,7 +3345,7 @@ dependencies = [ "oxc_index", "oxc_span", "oxc_str", - "phf 0.13.1", + "phf", "unicode-id-start", ] @@ -3469,15 +3430,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "phf" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_shared 0.11.3", -] - [[package]] name = "phf" version = "0.13.1" @@ -3485,38 +3437,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ "phf_macros", - "phf_shared 0.13.1", + "phf_shared", "serde", ] -[[package]] -name = "phf_codegen" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" -dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", -] - [[package]] name = "phf_codegen" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" dependencies = [ - "phf_generator 0.13.1", - "phf_shared 0.13.1", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared 0.11.3", - "rand 0.8.5", + "phf_generator", + "phf_shared", ] [[package]] @@ -3526,7 +3458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" dependencies = [ "fastrand", - "phf_shared 0.13.1", + "phf_shared", ] [[package]] @@ -3535,22 +3467,13 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" dependencies = [ - "phf_generator 0.13.1", - "phf_shared 0.13.1", + "phf_generator", + "phf_shared", "proc-macro2", "quote", "syn 2.0.117", ] -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher", -] - [[package]] name = "phf_shared" version = "0.13.1" @@ -3999,8 +3922,8 @@ dependencies = [ "derive_more", "log", "new_debug_unreachable", - "phf 0.13.1", - "phf_codegen 0.13.1", + "phf", + "phf_codegen", "precomputed-hash", "rustc-hash", "servo_arc", @@ -4270,19 +4193,6 @@ dependencies = [ "float-cmp", ] -[[package]] -name = "string_cache" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" -dependencies = [ - "new_debug_unreachable", - "parking_lot", - "phf_shared 0.11.3", - "precomputed-hash", - "serde", -] - [[package]] name = "string_cache" version = "0.9.0" @@ -4291,31 +4201,19 @@ checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" dependencies = [ "new_debug_unreachable", "parking_lot", - "phf_shared 0.13.1", + "phf_shared", "precomputed-hash", "serde", ] -[[package]] -name = "string_cache_codegen" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" -dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", - "proc-macro2", - "quote", -] - [[package]] name = "string_cache_codegen" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" dependencies = [ - "phf_generator 0.13.1", - "phf_shared 0.13.1", + "phf_generator", + "phf_shared", "proc-macro2", "quote", ] @@ -4393,7 +4291,7 @@ dependencies = [ "smallbitvec", "smallvec", "static_assertions", - "string_cache 0.9.0", + "string_cache", "strum", "strum_macros", "stylo_atoms", @@ -4409,7 +4307,7 @@ dependencies = [ "url", "void", "walkdir", - "web_atoms 0.2.3", + "web_atoms", ] [[package]] @@ -4417,8 +4315,8 @@ name = "stylo_atoms" version = "0.20.0" source = "git+https://github.com/ldm0/stylo?rev=74cd014af14d84db1fc8faed85086895012fc687#74cd014af14d84db1fc8faed85086895012fc687" dependencies = [ - "string_cache 0.9.0", - "string_cache_codegen 0.6.1", + "string_cache", + "string_cache_codegen", ] [[package]] @@ -4454,7 +4352,7 @@ dependencies = [ "servo_arc", "smallbitvec", "smallvec", - "string_cache 0.9.0", + "string_cache", "thin-vec", "void", ] @@ -4597,17 +4495,6 @@ dependencies = [ "writeable", ] -[[package]] -name = "tendril" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" -dependencies = [ - "futf", - "mac", - "utf-8", -] - [[package]] name = "tendril" version = "0.5.0" @@ -4774,7 +4661,7 @@ dependencies = [ "servo_arc", "smallbitvec", "smallvec", - "string_cache 0.9.0", + "string_cache", "thin-vec", ] @@ -5313,28 +5200,16 @@ dependencies = [ "thiserror", ] -[[package]] -name = "web_atoms" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ffde1dc01240bdf9992e3205668b235e59421fd085e8a317ed98da0178d414" -dependencies = [ - "phf 0.11.3", - "phf_codegen 0.11.3", - "string_cache 0.8.9", - "string_cache_codegen 0.5.4", -] - [[package]] name = "web_atoms" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576" dependencies = [ - "phf 0.13.1", - "phf_codegen 0.13.1", - "string_cache 0.9.0", - "string_cache_codegen 0.6.1", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", ] [[package]] @@ -5661,7 +5536,7 @@ name = "xml5ever" version = "0.39.0" dependencies = [ "log", - "markup5ever 0.39.0", + "markup5ever", ] [[package]] diff --git a/moli-charset-parser/Cargo.toml b/moli-charset-parser/Cargo.toml index 900664b4b5..fd6ecc292c 100644 --- a/moli-charset-parser/Cargo.toml +++ b/moli-charset-parser/Cargo.toml @@ -6,7 +6,7 @@ edition = "2024" [dependencies] encoding_rs = "0.8" -html5ever = "0.35.0" +html5ever = "0.39.0" [lints] workspace = true diff --git a/moli-dom/Cargo.toml b/moli-dom/Cargo.toml index f97ecd4fd4..3b6879268a 100644 --- a/moli-dom/Cargo.toml +++ b/moli-dom/Cargo.toml @@ -5,7 +5,7 @@ version = "0.1.0" edition = "2024" [dependencies] -html5ever = "0.35.0" +html5ever = "0.39.0" indexmap = "2.6" moli-css-parse = { path = "../moli-css-parse" } moli-html-input-temporal = { path = "../moli-html-input-temporal" } diff --git a/moli-parser/Cargo.toml b/moli-parser/Cargo.toml index 1662fa8895..f7ef6abcd5 100644 --- a/moli-parser/Cargo.toml +++ b/moli-parser/Cargo.toml @@ -6,13 +6,13 @@ edition = "2024" [dependencies] anyhow = "1.0.100" -html5ever = "0.35.0" +html5ever = "0.39.0" moli-dom = { path = "../moli-dom" } moli-fetch = { path = "../moli-fetch" } moli-page-types = { path = "../moli-page-types" } moli-script = { path = "../moli-script" } moli-stylesheet-blocking = { path = "../moli-stylesheet-blocking" } -markup5ever = "0.35.0" +markup5ever = "0.39.0" url = "2.5.7" xml5ever = "0.39.0" xmlparser = "0.13.6" diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rs index c55e32bca4..94e86f14af 100644 --- a/moli-parser/src/session.rs +++ b/moli-parser/src/session.rs @@ -27,6 +27,11 @@ pub(super) struct HtmlParserSession { suspended_input_buffers: Vec, } +pub(super) enum HtmlParserSessionResult { + InputDrained, + Script(ParseHandle), +} + struct EmbedderPausingTreeBuilder { inner: TreeBuilder, } @@ -93,7 +98,7 @@ impl TokenSink for EmbedderPausingTreeBuilder { None }; if let Some(script) = svg_script_handoff { - // html5ever 0.35 has an explicit FIXME for in SVG and + // html5ever 0.39 has an explicit FIXME for in SVG and // does not return TokenSinkResult::Script for it. Preserve the // ordinary tokenizer pause contract at this narrow adapter. return TokenSinkResult::Script(ParseHandle::new(script, None)); @@ -103,9 +108,9 @@ impl TokenSink for EmbedderPausingTreeBuilder { .sink .pending_custom_element_construction_handoff_placeholder() { - // html5ever 0.35 only exposes tokenizer pauses through the script handoff - // result. Moli interprets this handle as a custom-element handoff when - // the parser sink has a matching pending construction record. + // html5ever has no custom-element pause result. Moli interprets the + // script handoff handle as a custom-element handoff when the parser + // sink has a matching pending construction record. return TokenSinkResult::Script(ParseHandle::new(placeholder, None)); } if let Some(stylesheet) = self.inner.sink.pending_blocking_stylesheet_pause() { @@ -162,7 +167,9 @@ impl HtmlParserSession { pub(super) fn process(&mut self, input: StrTendril) { self.input_buffer.push_back(input); - while let TokenizerResult::Script(_) = self.tokenizer.feed(&self.input_buffer) { + while let HtmlParserSessionResult::Script(_) = + feed_with_definitive_encoding(&self.tokenizer, &self.input_buffer) + { // Non-pump callers intentionally parse through embedder pauses. They have no // runtime owner to notify, so parser-side custom-element handoffs and // blocking-stylesheet pauses must be discarded before continuing. @@ -222,9 +229,9 @@ impl HtmlParserSession { buffered } - pub(super) fn feed(&mut self) -> TokenizerResult { - let result = self.tokenizer.feed(&self.input_buffer); - if matches!(result, TokenizerResult::Done) + pub(super) fn feed(&mut self) -> HtmlParserSessionResult { + let result = feed_with_definitive_encoding(&self.tokenizer, &self.input_buffer); + if matches!(result, HtmlParserSessionResult::InputDrained) && self.input_buffer.is_empty() && let Some(parent) = self.suspended_input_buffers.pop() { @@ -244,7 +251,9 @@ impl HtmlParserSession { mut suspended_input_buffers, } = self; restore_all_suspended_input(&mut input_buffer, &mut suspended_input_buffers); - while let TokenizerResult::Script(_) = tokenizer.feed(&input_buffer) { + while let HtmlParserSessionResult::Script(_) = + feed_with_definitive_encoding(&tokenizer, &input_buffer) + { tokenizer .sink .sink() @@ -267,7 +276,9 @@ impl HtmlParserSession { mut suspended_input_buffers, } = self; restore_all_suspended_input(&mut input_buffer, &mut suspended_input_buffers); - while let TokenizerResult::Script(_) = tokenizer.feed(&input_buffer) { + while let HtmlParserSessionResult::Script(_) = + feed_with_definitive_encoding(&tokenizer, &input_buffer) + { tokenizer .sink .sink() @@ -305,6 +316,25 @@ impl HtmlParserSession { } } +fn feed_with_definitive_encoding( + tokenizer: &Tokenizer, + input_buffer: &BufferQueue, +) -> HtmlParserSessionResult { + loop { + match tokenizer.feed(input_buffer) { + // Moli resolves the document encoding from the response and byte-level + // meta prescan before creating this Unicode parser session. The tree + // builder cannot change that definitive decoding, so continue past its + // advisory notification without exposing a false parser pause. + TokenizerResult::EncodingIndicator(_) => {} + TokenizerResult::Done => return HtmlParserSessionResult::InputDrained, + TokenizerResult::Script(handle) => { + return HtmlParserSessionResult::Script(handle); + } + } + } +} + fn restore_all_suspended_input( input_buffer: &mut BufferQueue, suspended_input_buffers: &mut Vec, diff --git a/moli-parser/src/stream.rs b/moli-parser/src/stream.rs index 26a0f1ffc4..6988b77590 100644 --- a/moli-parser/src/stream.rs +++ b/moli-parser/src/stream.rs @@ -1,6 +1,4 @@ use html5ever::tendril::StrTendril; -use markup5ever::TokenizerResult; - use moli_dom::{ NodeId, native::{DomHost, NativeDom, NativeNodeId}, @@ -27,7 +25,10 @@ use super::{ ParserYield, }, live_target::{ParserRuntimeDomSinks, ParserStreamHtmlTreeSinkTarget}, - session::{HtmlParserSession, new_fragment_html_tree_sink_session, new_html_tree_sink_session}, + session::{ + HtmlParserSession, HtmlParserSessionResult, new_fragment_html_tree_sink_session, + new_html_tree_sink_session, + }, }; pub(super) struct HtmlTreeSinkStream { @@ -532,28 +533,29 @@ impl HtmlTreeSinkStream { } let tokenizer_result = self.parser.feed(); - let paused_for_custom_element = matches!(tokenizer_result, TokenizerResult::Script(_)) - && self.has_pending_custom_element_construction_handoff(); - let paused_for_stylesheet = matches!(tokenizer_result, TokenizerResult::Script(_)) + let paused_for_custom_element = + matches!(tokenizer_result, HtmlParserSessionResult::Script(_)) + && self.has_pending_custom_element_construction_handoff(); + let paused_for_stylesheet = matches!(tokenizer_result, HtmlParserSessionResult::Script(_)) && !paused_for_custom_element && self.peek_pending_blocking_stylesheet_pause().is_some(); let result = match tokenizer_result { - TokenizerResult::Script(handle) if paused_for_custom_element => { + HtmlParserSessionResult::Script(handle) if paused_for_custom_element => { debug_assert_eq!( self.peek_pending_custom_element_construction_handoff_placeholder(), Some(handle.node_id()) ); RawParserStep::CustomElementConstruction } - TokenizerResult::Script(handle) if paused_for_stylesheet => { + HtmlParserSessionResult::Script(handle) if paused_for_stylesheet => { debug_assert_eq!( self.peek_pending_blocking_stylesheet_pause(), Some(handle.node_id()) ); RawParserStep::BlockingStylesheet(handle.node_id()) } - TokenizerResult::Script(handle) => RawParserStep::Script(handle.node_id()), - TokenizerResult::Done => RawParserStep::InputDrained, + HtmlParserSessionResult::Script(handle) => RawParserStep::Script(handle.node_id()), + HtmlParserSessionResult::InputDrained => RawParserStep::InputDrained, }; let discovered_async_prefetch_candidate_node_ids = self.drain_discovered_async_prefetch_candidates(); @@ -1234,6 +1236,68 @@ mod tests { } } + #[test] + fn parser_stream_feed_continues_past_definitive_encoding_indicator() { + let mut stream = DocumentStream::new_parser_stream_for_testing( + Url::parse("https://example.test/page.html").expect("test url"), + ); + + stream.feed("

after meta

"); + + let document = stream.snapshot_parser_stream_document(); + assert_eq!( + document + .elements_by_tag_name(document.document_node_id(), "p", false) + .len(), + 1, + "the advisory encoding indicator must not leave decoded input buffered" + ); + } + + #[test] + fn parser_stream_pump_continues_past_definitive_encoding_indicator() { + let mut stream = DocumentStream::new_parser_stream_for_testing( + Url::parse("https://example.test/page.html").expect("test url"), + ); + + let outcome = + stream.pump_parser_step("

after meta

"); + + assert!(matches!(outcome.result, ParserPumpStep::InputDrained)); + let document = stream.snapshot_parser_stream_document(); + assert_eq!( + document + .elements_by_tag_name(document.document_node_id(), "p", false) + .len(), + 1, + "the parser pump must consume the tail after an encoding indicator" + ); + } + + #[test] + fn parser_stream_finish_continues_past_definitive_encoding_indicator() { + let mut stream = DocumentStream::new_parser_stream_for_testing( + Url::parse("https://example.test/page.html").expect("test url"), + ); + let outcome = stream.pump_parser_step(concat!( + "", + "

after meta

" + )); + assert!(matches!( + outcome.result, + ParserPumpStep::Yield(ParserYield::Script(_)) + )); + + let document = stream.finish(); + assert_eq!( + document + .elements_by_tag_name(document.document_node_id(), "p", false) + .len(), + 1, + "finishing must consume the buffered tail after an encoding indicator" + ); + } + #[test] fn prepare_parser_blocking_classic_script_accepts_inline_classic() { let (document, handle) = first_script_handle(""); diff --git a/moli-renderer-v8/Cargo.toml b/moli-renderer-v8/Cargo.toml index e814829d50..5f489a72fd 100644 --- a/moli-renderer-v8/Cargo.toml +++ b/moli-renderer-v8/Cargo.toml @@ -85,10 +85,10 @@ curl = "0.4.49" http = "1" httpdate = "1.0.3" html-escape = "0.2.13" -html5ever = "0.35.0" +html5ever = "0.39.0" indexmap = "2.6" parking_lot = "0.12" -markup5ever = "0.35.0" +markup5ever = "0.39.0" xml5ever = "0.39.0" v8 = "146.8.0" serde = { version = "1.0.228", features = ["derive"] } diff --git a/moli-wpt-compat/Cargo.toml b/moli-wpt-compat/Cargo.toml index 867e5264c3..73922736cc 100644 --- a/moli-wpt-compat/Cargo.toml +++ b/moli-wpt-compat/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" [dependencies] anyhow = "1.0.100" axum = { version = "0.8.6", features = ["ws"] } -html5ever = "0.35.0" +html5ever = "0.39.0" parking_lot = "0.12" rcgen = { version = "0.13", default-features = false, features = ["aws_lc_rs"] } serde = { version = "1.0.228", features = ["derive"] } diff --git a/moli-xpath/Cargo.toml b/moli-xpath/Cargo.toml index d92e659abb..f8c2ffc125 100644 --- a/moli-xpath/Cargo.toml +++ b/moli-xpath/Cargo.toml @@ -6,7 +6,7 @@ license = "MPL-2.0" [dependencies] log = "0.4" -markup5ever = "0.35.0" +markup5ever = "0.39.0" [lints] workspace = true