diff --git a/.gitignore b/.gitignore index 984fa217d6..4bf854fa55 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,6 @@ !/vendor/v8-152.2.0/** !/vendor/deno_v8-0.3.0/ !/vendor/deno_v8-0.3.0/** -!/vendor/xml5ever-0.39.0/ -!/vendor/xml5ever-0.39.0/** /.moli __pycache__ *.pyc diff --git a/Cargo.lock b/Cargo.lock index 52de8cefc9..0d25b9502a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3159,7 +3159,7 @@ dependencies = [ "moli-script", "moli-stylesheet-blocking", "url", - "xml5ever 0.40.0", + "xml5ever 0.40.1", "xmlparser", ] @@ -6863,7 +6863,8 @@ dependencies = [ [[package]] name = "xml5ever" -version = "0.40.0" +version = "0.40.1" +source = "git+https://github.com/lexmount/html5ever?rev=4574311c63e79bba7d4ddb98cb0562d8dad499ee#4574311c63e79bba7d4ddb98cb0562d8dad499ee" dependencies = [ "log", "markup5ever 0.40.0", diff --git a/Cargo.toml b/Cargo.toml index ab6f8951cc..a5c0984399 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,7 +92,6 @@ members = [ "moli-http-cache", "moli-wpt-compat" ] -exclude = ["vendor/xml5ever-0.40.0"] [workspace.package] license = "MIT OR Apache-2.0" @@ -103,7 +102,7 @@ fontique = { git = "https://github.com/lexmount/parley", rev = "a18cfc3ba6638d44 # Share fontations between shaping, raster rendering and PDF font subsetting. subsetter = { git = "https://github.com/lexmount/subsetter", rev = "8d3815245835af309559c3b4a872d6a0a54913a9" } url = { git = "https://github.com/lexmount/rust-url", rev = "ca9fcad2b99f20c30c28a618ad78410f6fd31161" } -xml5ever = { path = "vendor/xml5ever-0.40.0" } +xml5ever = { git = "https://github.com/lexmount/html5ever", rev = "4574311c63e79bba7d4ddb98cb0562d8dad499ee" } curl = { git = "https://github.com/lexmount/curl-rust", rev = "9d40ca90d8f19ccfd11afe7dd49bc32aab79435c" } curl-sys = { git = "https://github.com/lexmount/curl-rust", rev = "9d40ca90d8f19ccfd11afe7dd49bc32aab79435c" } cookie = { git = "https://github.com/ldm0/cookie-rs", branch = "priority" } diff --git a/vendor/xml5ever-0.40.0/Cargo.toml b/vendor/xml5ever-0.40.0/Cargo.toml deleted file mode 100644 index 0c0e4e7381..0000000000 --- a/vendor/xml5ever-0.40.0/Cargo.toml +++ /dev/null @@ -1,70 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2021" -rust-version = "1.85" -name = "xml5ever" -version = "0.40.0" -authors = ["The xml5ever project developers"] -build = false -exclude = ["xml5lib-tests/*"] -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Push based streaming parser for XML." -homepage = "https://github.com/servo/html5ever/blob/main/xml5ever/README.md" -documentation = "https://docs.rs/xml5ever" -readme = "README.md" -keywords = [ - "xml", - "xml5", - "parser", - "parsing", -] -categories = [ - "parser-implementations", - "web-programming", -] -license = "MIT OR Apache-2.0" -repository = "https://github.com/servo/html5ever" - -[features] -serde = ["markup5ever/serde"] -trace_tokenizer = [] - -[lib] -name = "xml5ever" -path = "src/lib.rs" - -[[example]] -name = "simple_xml_tokenizer" -path = "examples/simple_xml_tokenizer.rs" - -[[example]] -name = "xml_tokenizer" -path = "examples/xml_tokenizer.rs" - -[[bench]] -name = "xml5ever" -path = "benches/xml5ever.rs" -harness = false - -[dependencies.log] -version = "0.4" - -[dependencies.markup5ever] -version = "0.40" - -[dev-dependencies.criterion] -version = "0.8" diff --git a/vendor/xml5ever-0.40.0/LICENSE-APACHE b/vendor/xml5ever-0.40.0/LICENSE-APACHE deleted file mode 100644 index 16fe87b06e..0000000000 --- a/vendor/xml5ever-0.40.0/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/vendor/xml5ever-0.40.0/LICENSE-MIT b/vendor/xml5ever-0.40.0/LICENSE-MIT deleted file mode 100644 index 6e4510217d..0000000000 --- a/vendor/xml5ever-0.40.0/LICENSE-MIT +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2014 The html5ever Project Developers - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/vendor/xml5ever-0.40.0/MOLI_VENDORING.md b/vendor/xml5ever-0.40.0/MOLI_VENDORING.md deleted file mode 100644 index 5dfb596c55..0000000000 --- a/vendor/xml5ever-0.40.0/MOLI_VENDORING.md +++ /dev/null @@ -1,14 +0,0 @@ -# xml5ever 0.40.0 - -This directory contains the source of the crates.io `xml5ever` 0.40.0 release. -It is patched locally because Moli needs the tokenizer to preserve CDATA as a -distinct token, the XML tree builder to surface CDATA nodes to its sink, and -namespace declarations to remain available as DOM attributes after binding. -Numeric character references use the XML 1.0 Char production, preserving -referenced carriage returns and C1 values without HTML's Windows-1252 mapping. - -The upstream source is licensed under MIT or Apache-2.0; see `LICENSE-MIT` and -`LICENSE-APACHE` in this directory. - -Updated with the crates.io 0.40.0 release. The CDATA, namespace attribute, -and XML character-reference patches above are retained on the new source. diff --git a/vendor/xml5ever-0.40.0/README.md b/vendor/xml5ever-0.40.0/README.md deleted file mode 100644 index e5cb1f757e..0000000000 --- a/vendor/xml5ever-0.40.0/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# xml5ever - -![http://www.apache.org/licenses/LICENSE-2.0](https://img.shields.io/badge/license-Apache-blue.svg) ![https://opensource.org/licenses/MIT](https://img.shields.io/badge/license-MIT-blue.svg) -[![Docs.rs](https://docs.rs/xml5ever/badge.svg)](https://docs.rs/xml5ever) -[![](https://img.shields.io/crates/v/xml5ever.svg)](https://crates.io/crates/xml5ever) - -[API documentation](https://docs.rs/xml5ever) - -**Warning:** This library is alpha quality, so no guarantees are given. - -This crate provides a push based XML parser library that trades well-formedness for error recovery. - -xml5ever is based largely on [html5ever](https://github.com/servo/html5ever) parser, so if you have experience with html5ever you will be familiar with xml5ever. - -The library is dual licensed under MIT and Apache license. - -# Why you should use xml5ever - -Main use case for this library is when XML is badly formatted, usually from bad XML -templates. XML5 tries to handle most common errors, in a manner similar to HTML5. - -## When you should use it? - - - You aren't interested in well-formed documents. - - You need to get some info from your data even if it has errors (although not all possible errors are handled). - - You want to features like character references or XML namespaces. - -## When you shouldn't use it - - - You need to have your document validated. - - You require DTD support. - - You require an easy to use parser, with lots of extensions (e.g. XPath, XQuery). - - You require a battle tested, industry proven solution. - -# Installation - -Add xml5ever as a dependency: - -```bash -cargo add xml5ever -``` - -# Getting started - -Here is a very simple RcDom backed parser: - -```rust - - let input = "".to_tendril(); - - // To parse XML into a tree form, we need a TreeSink - // luckily xml5ever comes with a static RC backed tree represetation. - let dom: RcDom = parse(std::iter::once(input), Default::default()); - - // Do something with dom - -``` -The thing that does actual parsing is the `parse` function. It expects an iterator that can be converted into `StrTendril`, so you can use `std::iter::once(input)` or `Some(input).into_iter()` (where `input` is `StrTendril` like structure). - -# Working on xml5ever - -To build examples and tests you need to do something along the lines of: - -```rust - git submodule update --init # to fetch xml5lib-tests - cargo build - cargo test -``` - -This will fetch tests from outside repository and it will invoke cargo to -build and test the crate. If you need docs checkout either [API docs](https://docs.rs/xml5ever) or run `cargo docs` -to generate documentation. diff --git a/vendor/xml5ever-0.40.0/benches/xml5ever.rs b/vendor/xml5ever-0.40.0/benches/xml5ever.rs deleted file mode 100644 index 8380ab323a..0000000000 --- a/vendor/xml5ever-0.40.0/benches/xml5ever.rs +++ /dev/null @@ -1,77 +0,0 @@ -#[macro_use] -extern crate criterion; -extern crate markup5ever; -extern crate xml5ever; - -use std::fs; -use std::path::PathBuf; - -use criterion::Criterion; - -use markup5ever::buffer_queue::BufferQueue; -use xml5ever::tendril::*; -use xml5ever::tokenizer::{ProcessResult, Token, TokenSink, XmlTokenizer}; - -struct Sink; - -impl TokenSink for Sink { - type Handle = (); - - fn process_token(&self, token: Token) -> ProcessResult<()> { - // Don't use the token, but make sure we don't get - // optimized out entirely. - std::hint::black_box(token); - ProcessResult::Continue - } -} - -fn run_bench(c: &mut Criterion, name: &str) { - let mut path = PathBuf::from("./"); - path.push("data/bench/"); - path.push(name); - let mut file = fs::File::open(&path).expect("can't open file"); - - // Read the file and treat it as an infinitely repeating sequence of characters. - let mut file_input = ByteTendril::new(); - file.read_to_tendril(&mut file_input) - .expect("can't read file"); - let file_input: StrTendril = file_input.try_reinterpret().unwrap(); - let size = file_input.len(); - let mut stream = file_input.chars().cycle(); - - // Break the input into chunks of 1024 chars (= a few kB). - // This simulates reading from the network. - let mut input = vec![]; - let mut total = 0usize; - while total < size { - // The by_ref() call is important, otherwise we get wrong results! - // See rust-lang/rust#18045. - let sz = std::cmp::min(1024, size - total); - input.push(stream.by_ref().take(sz).collect::().to_tendril()); - total += sz; - } - - let test_name = format!("xml tokenizing {name}"); - - c.bench_function(&test_name, move |b| { - b.iter(|| { - let tok = XmlTokenizer::new(Sink, Default::default()); - let buffer = BufferQueue::default(); - // We are doing clone inside the bench function, this is not ideal, but possibly - // necessary since our iterator consumes the underlying buffer. - for buf in input.clone().into_iter() { - buffer.push_back(buf); - let _ = tok.feed(&buffer); - } - let _ = tok.feed(&buffer); - tok.end(); - }) - }); -} - -fn xml5ever_benchmarks(c: &mut Criterion) { - run_bench(c, "strong.xml"); -} - -criterion_group!(benches, xml5ever_benchmarks); -criterion_main!(benches); diff --git a/vendor/xml5ever-0.40.0/data/bench/strong.xml b/vendor/xml5ever-0.40.0/data/bench/strong.xml deleted file mode 100644 index 0ef665e5d7..0000000000 --- a/vendor/xml5ever-0.40.0/data/bench/strong.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/vendor/xml5ever-0.40.0/examples/README.md b/vendor/xml5ever-0.40.0/examples/README.md deleted file mode 100644 index b1763fc7ed..0000000000 --- a/vendor/xml5ever-0.40.0/examples/README.md +++ /dev/null @@ -1,223 +0,0 @@ -# Examples - -The examples have been designed with [`cargo-script`](https://github.com/DanielKeep/cargo-script) in mind. - -Here I'll just give broad overview how to install [`cargo script`] for Rust 1.5. For more details, check out [cargo-script repository](https://github.com/DanielKeep/cargo-script). - - cargo install cargo-script - - -# Token printer - -The basis of xml5ever is its tokenizer and tree builder. Roughly speaking tokenizer -takes input and returns a set of tokens like comment, processing instruction, start -tag, end tag, etc. - -First let's define our dependencies: - -```toml - [dependencies] - xml5ever = "0.2.0" - tendril = "0.1.3" -``` - -With dependencies declared, we can now make a simple tokenizer sink. First step is to -define a [`TokenSink`](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/trait.TokenSink.html). [`TokenSink`](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/trait.TokenSink.html) are traits that received stream of [`Tokens`](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/enum.Token.html). - -In our case we'll define a unit struct (i.e. a struct without any fields). - -```rust - struct SimpleTokenPrinter; -``` - -To make `SimpleTokenPrinter` a [`TokenSink`](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/trait.TokenSink.html), we need to implement [process_token](https://docs.rs/xml5ever/latest/xml5ever/tokenizer/trait.TokenSink.html#tymethod.process_token) method. - -```rust - impl TokenSink for SimpleTokenPrinter { - fn process_token(&mut self, token: Token) { - match token { - CharacterTokens(b) => { - println!("TEXT: {}", &*b); - }, - NullCharacterToken => print!("NULL"), - TagToken(tag) => { - println!("{:?} {} ", tag.kind, &*tag.name.local); - }, - ParseError(err) => { - println!("ERROR: {}", err); - }, - PIToken(Pi{ref target, ref data}) => { - println!("PI : ", &*target, &*data); - }, - CommentToken(ref comment) => { - println!("", &*comment); - }, - EOFToken => { - println!("EOF"); - }, - DoctypeToken(Doctype{ref name, ref public_id, ..}) => { - println!("", &*name, &*public_id); - } - } - } - } -``` - -Now, we need some input to process. For input we'll use `stdin`. However, xml5ever `tokenize_to` method only takes `StrTendril`. So we need to construct a -[`ByteTendril`](https://docs.rs/tendril/latest/tendril/type.ByteTendril.html) using `ByteTendril::new()`, then read the `stdin` using [`read_to_tendril`](https://docs.rs/tendril/latest/tendril/trait.ReadExt.html#tymethod.read_to_tendril) extension. - -Once that is set, to make `SimpleTokenPrinter` parse the input, call, -`tokenize_to` with it as the first parameter, input wrapped in Option for second parameter and XmlToke. - -```rust - fn main() { - let sink = SimpleTokenPrinter; - - // We need a ByteTendril to read a file - let mut input = ByteTendril::new(); - // Using SliceExt.read_to_tendril we read stdin - io::stdin().read_to_tendril(&mut input).unwrap(); - // For xml5ever we need StrTendril, so we reinterpret it - // into StrTendril. - // - // You might wonder, how does `try_reinterpret` know we - // need StrTendril and the answer is type inference based - // on `tokenize_xml_to` signature. - let input = input.try_reinterpret().unwrap(); - // Here we create and run tokenizer - let mut tok = XmlTokenizer::new(sink, Default::default()); - // We pass input to parsed. - tok.feed(input); - - // tok.end must be invoked for final bytes to be processed. - tok.end(); - } -``` - -NOTE: `unwrap` causes panic, it's only OK to use in simple examples. - -For full source code check out: [`examples/simple_xml_tokenizer.rs`](https://github.com/servo/html5ever/blob/main/xml5ever/examples/simple_xml_tokenizer.rs) - -Once we have successfully compiled the example we run the example with inline -xml - -```bash - cargo script simple_xml_tokenizer.rs <<< "Text with bold words!" -``` - -or by sending an [`examples/example.xml`](https://github.com/servo/html5ever/blob/main/xml5ever/examples/example.xml) located in same folder as examples. - -```bash - cargo script simple_xml_tokenizer.rs < example.xml -``` - -# Tree printer - -To actually get an XML document tree from the xml5ever, you need to use a `TreeSink`. -`TreeSink` is in many way similar to the TokenSink. Basically, TokenSink takes data -and returns list of tokens, while TreeSink takes tokens and returns a tree of parsed -XML document. Do note, that this is a simplified explanation and consult -documentation for more info. - -Ok, with that in mind, let's build us a TreePrinter. For example if we get an XML -file like: - -```xml - - Bobby - Tables - -``` - -We'd want a structure similar to this: - -``` -#document - student - first-name - #text Bobby - last-name - #text Tables - -``` -We won't print anything other than element names and text fields. So comments, -doctypes and other such elements are ignored. - -First part is similar to making SimpleTokenPrinter: - -```rust - // We need to allocate an input tendril for xml5ever - let mut input = ByteTendril::new(); - // Using SliceExt.read_to_tendril functions we can read stdin - io::stdin().read_to_tendril(&mut input).unwrap(); - let input = input.try_reinterpret().unwrap(); -``` - -This time, we need an implementation of [`TreeSink`](https://docs.rs/xml5ever/latest/xml5ever/tree_builder/trait.TreeSink.html). xml5ever comes with a -built-in `TreeSink` implementation called [`RcDom`](https://docs.rs/markup5ever_rcdom/latest/markup5ever_rcdom/struct.RcDom.html). To process input into -a `TreeSink` we use the following line: - -```rust - let dom: RcDom = parse(one_input(input), Default::default()); -``` - -Let's analyze it a bit. First there is `let dom: RcDom`. We need this part, -because the type inferencer can't infer which TreeSink implementation we mean -in this scenario. - -Function [`one_input`](https://ygg01.github.io/docs/xml5ever/xml5ever/fn.one_input.html) is a convenience function that turns any value into an iterator. In this case -it converts a StrTendril into an Iterator over itself. - -Ok, so now that we parsed our tree what with it? Well, for that we might need some -kind of function that will help us traverse it. We shall call that function `walk`. - -```rust - fn walk(prefix: &str, handle: Handle) { - let node = handle.borrow(); - - // We print out the prefix before we start - print!("{}", prefix); - // We are only interested in following nodes: - // Document, Text and Element, so our match - // reflects that. - match node.node { - Document - => println!("#document"), - - Text(ref text) => { - println!("#text {}", text.escape_default()) - }, - - Element(ref name, _) => { - println!("{}", name.local); - }, - - _ => {}, - - } - - // We increase indent in child nodes - let new_indent = { - let mut temp = String::new(); - temp.push_str(prefix); - temp.push_str(" "); - temp - }; - - for child in node.children.iter() - // In order to avoid weird indentation, we filter - // only Text/Element nodes. - // We don't need to filter Document since its guaranteed - // child elements don't contain documents - .filter(|child| match child.borrow().node { - Text(_) | Element (_, _) => true, - _ => false, - } - ) { - // Recursion - Yay! - walk(&new_indent, child.clone()); - } - } -``` - -For full source code check out: [`examples/xml_tree_printer.rs`](https://github.com/servo/html5ever/blob/main/rcdom/examples/xml_tree_printer.rs) diff --git a/vendor/xml5ever-0.40.0/examples/example.xml b/vendor/xml5ever-0.40.0/examples/example.xml deleted file mode 100644 index 834be5b92f..0000000000 --- a/vendor/xml5ever-0.40.0/examples/example.xml +++ /dev/null @@ -1,3 +0,0 @@ - - -BobbyTables diff --git a/vendor/xml5ever-0.40.0/examples/simple_xml_tokenizer.rs b/vendor/xml5ever-0.40.0/examples/simple_xml_tokenizer.rs deleted file mode 100644 index 20893a8c5b..0000000000 --- a/vendor/xml5ever-0.40.0/examples/simple_xml_tokenizer.rs +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env run-cargo-script -//! This is a regular crate doc comment, but it also contains a partial -//! Cargo manifest. Note the use of a *fenced* code block, and the -//! `cargo` "language". -//! -//! ```cargo -//! [dependencies] -//! xml5ever = "0.1.1" -//! tendril = "0.1.3" -//! markup5ever = "0.7.4" -//! ``` -extern crate markup5ever; -extern crate xml5ever; - -use std::io; - -use markup5ever::buffer_queue::BufferQueue; -use xml5ever::tendril::{ByteTendril, ReadExt}; -use xml5ever::tokenizer::{Doctype, Pi, ProcessResult, Token, TokenSink, XmlTokenizer}; - -struct SimpleTokenPrinter; - -impl TokenSink for SimpleTokenPrinter { - type Handle = (); - - fn process_token(&self, token: Token) -> ProcessResult<()> { - match token { - Token::Characters(b) => { - println!("TEXT: {}", &*b); - }, - Token::Cdata(b) => { - println!("CDATA: {}", &*b); - }, - Token::NullCharacter => print!("NULL"), - Token::Tag(tag) => { - println!("{:?} {} ", tag.kind, &*tag.name.local); - }, - Token::ParseError(err) => { - println!("ERROR: {err}"); - }, - Token::ProcessingInstruction(Pi { - ref target, - ref data, - }) => { - println!("PI : "); - }, - Token::Comment(ref comment) => { - println!(""); - }, - Token::EndOfFile => { - println!("EOF"); - }, - Token::Doctype(Doctype { - ref name, - ref public_id, - .. - }) => { - println!(""); - }, - }; - ProcessResult::Continue - } -} - -fn main() { - // Our implementation of TokenSink - let sink = SimpleTokenPrinter; - - // We need a ByteTendril to read a file - let mut input = ByteTendril::new(); - - // Using SliceExt.read_to_tendril we can read stdin - io::stdin().read_to_tendril(&mut input).unwrap(); - // For xml5ever we need StrTendril, so we reinterpret it - // into StrTendril. - - // Load input into BufferQueue - let input_buffer = BufferQueue::default(); - input_buffer.push_back(input.try_reinterpret().unwrap()); - // Here we create and run tokenizer - let tok = XmlTokenizer::new(sink, Default::default()); - let _ = tok.feed(&input_buffer); - tok.end(); -} diff --git a/vendor/xml5ever-0.40.0/examples/xml_tokenizer.rs b/vendor/xml5ever-0.40.0/examples/xml_tokenizer.rs deleted file mode 100644 index e3f1d8a897..0000000000 --- a/vendor/xml5ever-0.40.0/examples/xml_tokenizer.rs +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env run-cargo-script -//! This is a regular crate doc comment, but it also contains a partial -//! Cargo manifest. Note the use of a *fenced* code block, and the -//! `cargo` "language". -//! -//! ```cargo -//! [dependencies] -//! xml5ever = "0.2.0" -//! tendril = "0.1.3" -//! markup5ever = "0.7.4" -//! ``` -extern crate markup5ever; -extern crate xml5ever; - -use std::cell::Cell; -use std::io; - -use markup5ever::buffer_queue::BufferQueue; -use xml5ever::tendril::{ByteTendril, ReadExt}; -use xml5ever::tokenizer::{ - EmptyTag, EndTag, Pi, ProcessResult, ShortTag, StartTag, Token, TokenSink, XmlTokenizer, - XmlTokenizerOpts, -}; - -#[derive(Clone)] -struct TokenPrinter { - in_char_run: Cell, -} - -impl TokenPrinter { - fn is_char(&self, is_char: bool) { - match (self.in_char_run.get(), is_char) { - (false, true) => print!("CHAR : \""), - (true, false) => println!("\""), - _ => (), - } - self.in_char_run.set(is_char); - } - - fn do_char(&self, c: char) { - self.is_char(true); - print!("{}", c.escape_default().collect::()); - } -} - -impl TokenSink for TokenPrinter { - type Handle = (); - - fn process_token(&self, token: Token) -> ProcessResult<()> { - match token { - Token::Characters(b) => { - for c in b.chars() { - self.do_char(c); - } - }, - Token::Cdata(b) => { - self.is_char(false); - println!("CDATA: {}", &*b); - }, - Token::NullCharacter => self.do_char('\0'), - Token::Tag(tag) => { - self.is_char(false); - // This is not proper HTML serialization, of course. - match tag.kind { - StartTag => print!("TAG : <\x1b[32m{}\x1b[0m", tag.name.local), - EndTag => print!("END TAG : <\x1b[31m/{}\x1b[0m", tag.name.local), - ShortTag => print!("Short TAG : <\x1b[31m/{}\x1b[0m", tag.name.local), - EmptyTag => print!("Empty TAG : <\x1b[31m{}\x1b[0m", tag.name.local), - } - for attr in tag.attrs.iter() { - print!( - " \x1b[36m{}\x1b[0m='\x1b[34m{}\x1b[0m'", - attr.name.local, attr.value - ); - } - if tag.kind == EmptyTag { - print!("/"); - } - println!(">"); - }, - Token::ParseError(err) => { - self.is_char(false); - println!("ERROR: {err}"); - }, - Token::ProcessingInstruction(Pi { target, data }) => { - self.is_char(false); - println!("PI : "); - }, - _ => { - self.is_char(false); - println!("OTHER: {token:?}"); - }, - }; - - ProcessResult::Continue - } -} - -fn main() { - let sink = TokenPrinter { - in_char_run: Cell::new(false), - }; - let mut input = ByteTendril::new(); - io::stdin().read_to_tendril(&mut input).unwrap(); - let input_buffer = BufferQueue::default(); - input_buffer.push_back(input.try_reinterpret().unwrap()); - - let tok = XmlTokenizer::new( - sink, - XmlTokenizerOpts { - profile: true, - exact_errors: true, - ..Default::default() - }, - ); - let _ = tok.feed(&input_buffer); - tok.end(); - tok.sink.is_char(false); -} diff --git a/vendor/xml5ever-0.40.0/src/driver.rs b/vendor/xml5ever-0.40.0/src/driver.rs deleted file mode 100644 index 083ce43c66..0000000000 --- a/vendor/xml5ever-0.40.0/src/driver.rs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use crate::tokenizer::{XmlTokenizer, XmlTokenizerOpts}; -use crate::tree_builder::{XmlTreeBuilder, XmlTreeBuilderOpts, XmlTreeSink}; - -use std::borrow::Cow; - -use crate::tendril; -use crate::tendril::stream::{TendrilSink, Utf8LossyDecoder}; -use crate::tendril::StrTendril; -use markup5ever::buffer_queue::BufferQueue; -use markup5ever::TokenizerResult; - -/// All-encompasing parser setting structure. -#[derive(Clone, Default)] -pub struct XmlParseOpts { - /// Xml tokenizer options. - pub tokenizer: XmlTokenizerOpts, - /// Xml tree builder . - pub tree_builder: XmlTreeBuilderOpts, -} - -/// Parse and send results to a `TreeSink`. -/// -/// ## Example -/// -/// ```ignore -/// let mut sink = MySink; -/// parse_document(&mut sink, iter::once(my_str), Default::default()); -/// ``` -pub fn parse_document(sink: Sink, opts: XmlParseOpts) -> XmlParser -where - Sink: XmlTreeSink, -{ - let tb = XmlTreeBuilder::new(sink, opts.tree_builder); - let tok = XmlTokenizer::new(tb, opts.tokenizer); - XmlParser { - tokenizer: tok, - input_buffer: BufferQueue::default(), - } -} - -/// An XML parser, -/// ready to receive Unicode input through the `tendril::TendrilSink` trait’s methods. -pub struct XmlParser -where - Sink: XmlTreeSink, -{ - /// Tokenizer used by XmlParser. - pub tokenizer: XmlTokenizer>, - /// Input used by XmlParser. - pub input_buffer: BufferQueue, -} - -impl TendrilSink for XmlParser { - type Output = Sink::Output; - - fn process(&mut self, t: StrTendril) { - self.input_buffer.push_back(t); - // FIXME: Properly support somehow. - while let TokenizerResult::Script(_) = self.tokenizer.feed(&self.input_buffer) {} - } - - // FIXME: Is it too noisy to report every character decoding error? - fn error(&mut self, desc: Cow<'static, str>) { - self.tokenizer.sink.sink.parse_error(desc) - } - - fn finish(self) -> Self::Output { - self.tokenizer.end(); - self.tokenizer.sink.sink.finish() - } -} - -impl XmlParser { - /// Wrap this parser into a `TendrilSink` that accepts UTF-8 bytes. - /// - /// Use this when your input is bytes that are known to be in the UTF-8 encoding. - /// Decoding is lossy, like `String::from_utf8_lossy`. - #[allow(clippy::wrong_self_convention)] - pub fn from_utf8(self) -> Utf8LossyDecoder { - Utf8LossyDecoder::new(self) - } -} diff --git a/vendor/xml5ever-0.40.0/src/lib.rs b/vendor/xml5ever-0.40.0/src/lib.rs deleted file mode 100644 index c9e557ff1d..0000000000 --- a/vendor/xml5ever-0.40.0/src/lib.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! This crate provides a push based XML parser library that -//! adheres to XML5 specification. In other words this library -//! trades well-formedness for error recovery. -//! -//! The idea behind this, was to minimize number of errors from -//! tools that generate XML (e.g. `S` won't just return `S` -//! as text, but will parse it into `S` ). -//! You can check out full specification [here](https://ygg01.github.io/xml5_draft/). -//! -//! What this library provides is a solid XML parser that can: -//! -//! * Parse somewhat erroneous XML input -//! * Provide support for [Numeric character references](https://en.wikipedia.org/wiki/Numeric_character_reference). -//! * Provide partial [XML namespace](http://www.w3.org/TR/xml-names11/) support. -//! * Provide full set of SVG/MathML entities -//! -//! What isn't in scope for this library: -//! -//! * Document Type Definition parsing - this is pretty hard to do right and nowadays, its used -//! - -#![crate_name = "xml5ever"] -#![crate_type = "dylib"] -#![allow(unexpected_cfgs)] -#![deny(missing_docs)] - -pub use markup5ever::*; - -pub(crate) mod macros; - -/// Driver -pub mod driver; -/// Serializer for XML5. -pub mod serialize; -/// XML5 tokenizer - converts input into tokens -pub mod tokenizer; -/// XML5 tree builder - converts tokens into a tree like structure -pub mod tree_builder; diff --git a/vendor/xml5ever-0.40.0/src/macros.rs b/vendor/xml5ever-0.40.0/src/macros.rs deleted file mode 100644 index 0b786c38a8..0000000000 --- a/vendor/xml5ever-0.40.0/src/macros.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -macro_rules! time { - ($e:expr) => {{ - let t0 = ::std::time::Instant::now(); - let result = $e; - let dt = t0.elapsed().as_nanos() as u64; - (result, dt) - }}; -} -pub(crate) use time; diff --git a/vendor/xml5ever-0.40.0/src/serialize/mod.rs b/vendor/xml5ever-0.40.0/src/serialize/mod.rs deleted file mode 100644 index a2d2671026..0000000000 --- a/vendor/xml5ever-0.40.0/src/serialize/mod.rs +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use crate::tree_builder::NamespaceMap; -use crate::QualName; -pub use markup5ever::serialize::{AttrRef, Serialize, Serializer, TraversalScope}; -use std::io::{self, Write}; - -#[derive(Clone)] -/// Struct for setting serializer options. -pub struct SerializeOpts { - /// Serialize the root node? Default: ChildrenOnly - pub traversal_scope: TraversalScope, -} - -impl Default for SerializeOpts { - fn default() -> SerializeOpts { - SerializeOpts { - traversal_scope: TraversalScope::ChildrenOnly(None), - } - } -} - -/// Method for serializing generic node to a given writer. -pub fn serialize(writer: Wr, node: &T, opts: SerializeOpts) -> io::Result<()> -where - Wr: Write, - T: Serialize, -{ - let mut ser = XmlSerializer::new(writer); - node.serialize(&mut ser, opts.traversal_scope) -} - -/// Struct used for serializing nodes into a text that other XML -/// parses can read. -/// -/// Serializer contains a set of functions (start_elem, end_elem...) -/// that make parsing nodes easier. -pub struct XmlSerializer { - writer: Wr, - namespace_stack: NamespaceMapStack, -} - -#[derive(Debug)] -struct NamespaceMapStack(Vec); - -impl NamespaceMapStack { - fn new() -> NamespaceMapStack { - NamespaceMapStack(vec![]) - } - - fn push(&mut self, namespace: NamespaceMap) { - self.0.push(namespace); - } - - fn pop(&mut self) { - self.0.pop(); - } -} - -/// Writes given text into the Serializer, escaping it, -/// depending on where the text is written inside the tag or attribute value. -/// -/// For example -///```text -/// '&-quotes' becomes '&-quotes' -/// becomes (writer: &mut W, text: &str, attr_mode: bool) -> io::Result<()> { - for c in text.chars() { - match c { - '&' => writer.write_all(b"&"), - '\'' if attr_mode => writer.write_all(b"'"), - '"' if attr_mode => writer.write_all(b"""), - '<' if !attr_mode => writer.write_all(b"<"), - '>' if !attr_mode => writer.write_all(b">"), - c => writer.write_fmt(format_args!("{c}")), - }?; - } - Ok(()) -} - -#[inline] -fn write_qual_name(writer: &mut W, name: &QualName) -> io::Result<()> { - if let Some(ref prefix) = name.prefix { - writer.write_all(prefix.as_bytes())?; - writer.write_all(b":")?; - } - - writer.write_all(name.local.as_bytes())?; - Ok(()) -} - -impl XmlSerializer { - /// Creates a new Serializier from a writer and given serialization options. - pub fn new(writer: Wr) -> Self { - XmlSerializer { - writer, - namespace_stack: NamespaceMapStack::new(), - } - } - - #[inline(always)] - fn qual_name(&mut self, name: &QualName) -> io::Result<()> { - self.find_or_insert_ns(name); - write_qual_name(&mut self.writer, name) - } - - #[inline(always)] - fn qual_attr_name(&mut self, name: &QualName) -> io::Result<()> { - self.find_or_insert_ns(name); - write_qual_name(&mut self.writer, name) - } - - fn find_uri(&self, name: &QualName) -> bool { - let mut found = false; - for stack in self.namespace_stack.0.iter().rev() { - if let Some(Some(el)) = stack.get(&name.prefix) { - found = *el == name.ns; - break; - } - } - found - } - - fn find_or_insert_ns(&mut self, name: &QualName) { - if (name.prefix.is_some() || !name.ns.is_empty()) && !self.find_uri(name) { - if let Some(last_ns) = self.namespace_stack.0.last_mut() { - last_ns.insert(name); - } - } - } -} - -impl Serializer for XmlSerializer { - /// Serializes given start element into text. Start element contains - /// qualified name and an attributes iterator. - fn start_elem<'a, AttrIter>(&mut self, name: QualName, attrs: AttrIter) -> io::Result<()> - where - AttrIter: Iterator>, - { - self.namespace_stack.push(NamespaceMap::empty()); - - self.writer.write_all(b"<")?; - self.qual_name(&name)?; - if let Some(current_namespace) = self.namespace_stack.0.last() { - for (prefix, url_opt) in current_namespace.get_scope_iter() { - self.writer.write_all(b" xmlns")?; - if let Some(ref prefix) = *prefix { - self.writer.write_all(b":")?; - self.writer.write_all(prefix.as_bytes())?; - } - - self.writer.write_all(b"=\"")?; - let url = if let Some(ref a) = *url_opt { - a.as_bytes() - } else { - b"" - }; - self.writer.write_all(url)?; - self.writer.write_all(b"\"")?; - } - } - for (name, value) in attrs { - self.writer.write_all(b" ")?; - self.qual_attr_name(name)?; - self.writer.write_all(b"=\"")?; - write_to_buf_escaped(&mut self.writer, value, true)?; - self.writer.write_all(b"\"")?; - } - self.writer.write_all(b">")?; - Ok(()) - } - - /// Serializes given end element into text. - fn end_elem(&mut self, name: QualName) -> io::Result<()> { - self.namespace_stack.pop(); - self.writer.write_all(b"") - } - - /// Serializes comment into text. - fn write_comment(&mut self, text: &str) -> io::Result<()> { - self.writer.write_all(b"") - } - - /// Serializes given doctype - fn write_doctype(&mut self, name: &str) -> io::Result<()> { - self.writer.write_all(b"") - } - - /// Serializes text for a node or an attributes. - fn write_text(&mut self, text: &str) -> io::Result<()> { - write_to_buf_escaped(&mut self.writer, text, false) - } - - /// Serializes given processing instruction. - fn write_processing_instruction(&mut self, target: &str, data: &str) -> io::Result<()> { - self.writer.write_all(b"") - } -} diff --git a/vendor/xml5ever-0.40.0/src/tokenizer/char_ref/mod.rs b/vendor/xml5ever-0.40.0/src/tokenizer/char_ref/mod.rs deleted file mode 100644 index 5d1bd1063e..0000000000 --- a/vendor/xml5ever-0.40.0/src/tokenizer/char_ref/mod.rs +++ /dev/null @@ -1,451 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use super::{TokenSink, XmlTokenizer}; -use crate::data; -use crate::tendril::StrTendril; -use log::debug; -use markup5ever::buffer_queue::BufferQueue; -use std::borrow::Cow::{self, Borrowed}; -use std::char::from_u32; - -use self::State::*; -pub use self::Status::*; - -//§ tokenizing-character-references -pub struct CharRef { - /// The resulting character(s) - pub chars: [char; 2], - - /// How many slots in `chars` are valid? - pub num_chars: u8, -} - -pub enum Status { - Stuck, - Progress, - Done, -} - -#[derive(Debug)] -enum State { - Begin, - Octothorpe, - Numeric(u32), // base - NumericSemicolon, - Named, - BogusName, -} - -pub struct CharRefTokenizer { - state: State, - addnl_allowed: Option, - result: Option, - - num: u32, - num_too_big: bool, - seen_digit: bool, - hex_marker: Option, - - name_buf_opt: Option, - name_match: Option<(u32, u32)>, - name_len: usize, -} - -impl CharRefTokenizer { - // NB: We assume that we have an additional allowed character iff we're - // tokenizing in an attribute value. - pub fn new(addnl_allowed: Option) -> CharRefTokenizer { - CharRefTokenizer { - state: Begin, - addnl_allowed, - result: None, - num: 0, - num_too_big: false, - seen_digit: false, - hex_marker: None, - name_buf_opt: None, - name_match: None, - name_len: 0, - } - } - - // A CharRefTokenizer can only tokenize one character reference, - // so this method consumes the tokenizer. - pub fn get_result(self) -> CharRef { - self.result.expect("get_result called before done") - } - - fn name_buf(&self) -> &StrTendril { - self.name_buf_opt - .as_ref() - .expect("name_buf missing in named character reference") - } - - fn name_buf_mut(&mut self) -> &mut StrTendril { - self.name_buf_opt - .as_mut() - .expect("name_buf missing in named character reference") - } - - fn finish_none(&mut self) -> Status { - self.result = Some(CharRef { - chars: ['\0', '\0'], - num_chars: 0, - }); - Done - } - - fn finish_one(&mut self, c: char) -> Status { - self.result = Some(CharRef { - chars: [c, '\0'], - num_chars: 1, - }); - Done - } -} - -impl CharRefTokenizer { - pub fn step( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) -> Status { - if self.result.is_some() { - return Done; - } - - debug!("char ref tokenizer stepping in state {:?}", self.state); - match self.state { - Begin => self.do_begin(tokenizer, input), - Octothorpe => self.do_octothorpe(tokenizer, input), - Numeric(base) => self.do_numeric(tokenizer, base, input), - NumericSemicolon => self.do_numeric_semicolon(tokenizer, input), - Named => self.do_named(tokenizer, input), - BogusName => self.do_bogus_name(tokenizer, input), - } - } - - fn do_begin( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) -> Status { - match tokenizer.peek(input) { - Some('\t' | '\n' | '\x0C' | ' ' | '<' | '&') => self.finish_none(), - Some(c) if Some(c) == self.addnl_allowed => self.finish_none(), - Some('#') => { - tokenizer.discard_char(input); - self.state = Octothorpe; - Progress - }, - Some(_) => { - self.state = Named; - self.name_buf_opt = Some(StrTendril::new()); - Progress - }, - None => Stuck, - } - } - - fn do_octothorpe( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) -> Status { - match tokenizer.peek(input) { - Some(c @ ('x' | 'X')) => { - tokenizer.discard_char(input); - self.hex_marker = Some(c); - self.state = Numeric(16); - }, - Some(_) => { - self.hex_marker = None; - self.state = Numeric(10); - }, - None => return Stuck, - } - Progress - } - - fn do_numeric( - &mut self, - tokenizer: &XmlTokenizer, - base: u32, - input: &BufferQueue, - ) -> Status { - let Some(c) = tokenizer.peek(input) else { - return Stuck; - }; - match c.to_digit(base) { - Some(n) => { - tokenizer.discard_char(input); - self.num = self.num.wrapping_mul(base); - if self.num > 0x10FFFF { - // We might overflow, and the character is definitely invalid. - // We still parse digits and semicolon, but don't use the result. - self.num_too_big = true; - } - self.num = self.num.wrapping_add(n); - self.seen_digit = true; - Progress - }, - - None if !self.seen_digit => self.unconsume_numeric(tokenizer, input), - - None => { - self.state = NumericSemicolon; - Progress - }, - } - } - - fn do_numeric_semicolon( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) -> Status { - match tokenizer.peek(input) { - Some(';') => tokenizer.discard_char(input), - Some(_) => tokenizer.emit_error(Borrowed( - "Semicolon missing after numeric character reference", - )), - None => return Stuck, - }; - self.finish_numeric(tokenizer) - } - - fn unconsume_numeric( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) -> Status { - let mut unconsume = StrTendril::from_char('#'); - if let Some(c) = self.hex_marker { - unconsume.push_char(c); - } - - tokenizer.unconsume(input, unconsume); - tokenizer.emit_error(Borrowed("Numeric character reference without digits")); - self.finish_none() - } - - fn finish_numeric(&mut self, tokenizer: &XmlTokenizer) -> Status { - let (c, error) = match self.num { - n if !self.num_too_big - && matches!( - n, - 0x9 | 0xa | 0xd | 0x20..=0xd7ff | 0xe000..=0xfffd | 0x10000..=0x10ffff - ) => - { - ( - from_u32(n).expect("the XML Char production contains only Unicode scalars"), - false, - ) - }, - _ => ('\u{fffd}', true), - }; - - if error { - let msg = if tokenizer.opts.exact_errors { - Cow::from(format!( - "Invalid numeric character reference value 0x{:06X}", - self.num - )) - } else { - Cow::from("Invalid numeric character reference") - }; - tokenizer.emit_error(msg); - } - - self.finish_one(c) - } - - fn do_named( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) -> Status { - let Some(c) = tokenizer.get_char(input) else { - return Stuck; - }; - self.name_buf_mut().push_char(c); - match data::NAMED_ENTITIES.get(&self.name_buf()[..]) { - // We have either a full match or a prefix of one. - Some(&m) => { - if m.0 != 0 { - // We have a full match, but there might be a longer one to come. - self.name_match = Some(m); - self.name_len = self.name_buf().len(); - } - // Otherwise we just have a prefix match. - Progress - }, - - // Can't continue the match. - None => self.finish_named(tokenizer, Some(c), input), - } - } - - fn emit_name_error(&mut self, tokenizer: &XmlTokenizer) { - let msg = if tokenizer.opts.exact_errors { - Cow::from(format!("Invalid character reference &{}", self.name_buf())) - } else { - Cow::from("Invalid character reference") - }; - tokenizer.emit_error(msg); - } - - fn unconsume_name( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) { - tokenizer.unconsume(input, self.name_buf_opt.take().unwrap()); - } - - fn finish_named( - &mut self, - tokenizer: &XmlTokenizer, - end_char: Option, - input: &BufferQueue, - ) -> Status { - match self.name_match { - None => { - match end_char { - Some(c) if c.is_ascii_alphanumeric() => { - // Keep looking for a semicolon, to determine whether - // we emit a parse error. - self.state = BogusName; - return Progress; - }, - - // Check length because &; is not a parse error. - Some(';') if self.name_buf().len() > 1 => self.emit_name_error(tokenizer), - - _ => (), - } - self.unconsume_name(tokenizer, input); - self.finish_none() - }, - - Some((c1, c2)) => { - // We have a complete match, but we may have consumed - // additional characters into self.name_buf. Usually - // at least one, but several in cases like - // - // ¬ => match for U+00AC - // ¬i => valid prefix for ¬in - // ¬it => can't continue match - - let name_len = self.name_len; - assert!(name_len > 0); - let last_matched = self.name_buf()[name_len - 1..].chars().next().unwrap(); - - // There might not be a next character after the match, if - // we had a full match and then hit EOF. - let next_after = if name_len == self.name_buf().len() { - None - } else { - Some(self.name_buf()[name_len..].chars().next().unwrap()) - }; - - // "If the character reference is being consumed as part of an - // attribute, and the last character matched is not a U+003B - // SEMICOLON character (;), and the next character is either a - // U+003D EQUALS SIGN character (=) or an alphanumeric ASCII - // character, then, for historical reasons, all the characters - // that were matched after the U+0026 AMPERSAND character (&) - // must be unconsumed, and nothing is returned. However, if - // this next character is in fact a U+003D EQUALS SIGN - // character (=), then this is a parse error" - - let unconsume_all = match (self.addnl_allowed, last_matched, next_after) { - (_, ';', _) => false, - (Some(_), _, Some('=')) => { - tokenizer.emit_error(Borrowed( - "Equals sign after character reference in attribute", - )); - true - }, - (Some(_), _, Some(c)) if c.is_ascii_alphanumeric() => true, - _ => { - tokenizer.emit_error(Borrowed( - "Character reference does not end with semicolon", - )); - false - }, - }; - - if unconsume_all { - self.unconsume_name(tokenizer, input); - self.finish_none() - } else { - tokenizer - .unconsume(input, StrTendril::from_slice(&self.name_buf()[name_len..])); - self.result = Some(CharRef { - chars: [from_u32(c1).unwrap(), from_u32(c2).unwrap()], - num_chars: if c2 == 0 { 1 } else { 2 }, - }); - Done - } - }, - } - } - - fn do_bogus_name( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) -> Status { - let Some(c) = tokenizer.get_char(input) else { - return Stuck; - }; - self.name_buf_mut().push_char(c); - match c { - _ if c.is_ascii_alphanumeric() => return Progress, - ';' => self.emit_name_error(tokenizer), - _ => (), - } - self.unconsume_name(tokenizer, input); - self.finish_none() - } - - pub fn end_of_file( - &mut self, - tokenizer: &XmlTokenizer, - input: &BufferQueue, - ) { - while self.result.is_none() { - match self.state { - Begin => drop(self.finish_none()), - - Numeric(_) if !self.seen_digit => drop(self.unconsume_numeric(tokenizer, input)), - - Numeric(_) | NumericSemicolon => { - tokenizer.emit_error(Borrowed("EOF in numeric character reference")); - self.finish_numeric(tokenizer); - }, - - Named => drop(self.finish_named(tokenizer, None, input)), - - BogusName => { - self.unconsume_name(tokenizer, input); - self.finish_none(); - }, - - Octothorpe => { - tokenizer.unconsume(input, StrTendril::from_slice("#")); - tokenizer.emit_error(Borrowed("EOF after '#' in character reference")); - self.finish_none(); - }, - } - } - } -} diff --git a/vendor/xml5ever-0.40.0/src/tokenizer/interface.rs b/vendor/xml5ever-0.40.0/src/tokenizer/interface.rs deleted file mode 100644 index 15cc569a85..0000000000 --- a/vendor/xml5ever-0.40.0/src/tokenizer/interface.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use std::borrow::Cow; - -use crate::tendril::StrTendril; -use crate::tokenizer::ProcessResult; -use crate::{Attribute, QualName}; - -pub use self::TagKind::{EmptyTag, EndTag, ShortTag, StartTag}; - -/// Tag kind denotes which kind of tag did we encounter. -#[derive(PartialEq, Eq, Hash, Copy, Clone, Debug)] -pub enum TagKind { - /// Beginning of a tag (e.g. ``). - StartTag, - /// End of a tag (e.g. ``). - EndTag, - /// Empty tag (e.g. ``). - EmptyTag, - /// Short tag (e.g. ``). - ShortTag, -} - -/// XML 5 Tag Token -#[derive(PartialEq, Eq, Debug, Clone)] -pub struct Tag { - /// Token kind denotes which type of token was encountered. - /// E.g. if parser parsed `` the token kind would be `EndTag`. - pub kind: TagKind, - /// Qualified name of the tag. - pub name: QualName, - /// List of attributes attached to this tag. - /// Only valid in start and empty tag. - pub attrs: Vec, -} - -impl Tag { - /// Sorts attributes in a tag. - pub fn equiv_modulo_attr_order(&self, other: &Tag) -> bool { - if (self.kind != other.kind) || (self.name != other.name) { - return false; - } - - let mut self_attrs = self.attrs.clone(); - let mut other_attrs = other.attrs.clone(); - self_attrs.sort(); - other_attrs.sort(); - - self_attrs == other_attrs - } -} - -/// A `DOCTYPE` token. -/// Doctype token in XML5 is rather limited for reasons, such as: -/// security and simplicity. XML5 only supports declaring DTD with -/// name, public identifier and system identifier -#[derive(PartialEq, Eq, Clone, Debug, Default)] -pub struct Doctype { - /// Name of DOCTYPE declared - pub name: Option, - /// Public identifier of this DOCTYPE. - pub public_id: Option, - /// System identifier of this DOCTYPE. - pub system_id: Option, -} - -/// A ProcessingInstruction token. -#[derive(PartialEq, Eq, Clone, Debug)] -pub struct Pi { - /// What is the name of processing instruction. - pub target: StrTendril, - - /// Text of processing instruction. - pub data: StrTendril, -} - -/// Describes tokens encountered during parsing of input. -#[derive(PartialEq, Eq, Debug)] -pub enum Token { - /// Doctype token - Doctype(Doctype), - /// Token tag founds. This token applies to all - /// possible kinds of tags (like start, end, empty tag, etc.). - Tag(Tag), - /// Processing Instruction token - ProcessingInstruction(Pi), - /// Comment token. - Comment(StrTendril), - /// Token that represents a series of characters. - Characters(StrTendril), - /// Token that represents a CDATA section. - Cdata(StrTendril), - /// End of File found. - EndOfFile, - /// NullCharacter encountered. - NullCharacter, - /// Error happened - ParseError(Cow<'static, str>), -} - -/// Types which can receive tokens from the tokenizer. -pub trait TokenSink { - /// Handle to a DOM script element - type Handle; - - /// Process a token. - fn process_token(&self, token: Token) -> ProcessResult; - - /// Signal to the sink that parsing has ended. - fn end(&self) {} -} diff --git a/vendor/xml5ever-0.40.0/src/tokenizer/mod.rs b/vendor/xml5ever-0.40.0/src/tokenizer/mod.rs deleted file mode 100644 index aa7ecd6c2e..0000000000 --- a/vendor/xml5ever-0.40.0/src/tokenizer/mod.rs +++ /dev/null @@ -1,1536 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -mod char_ref; -mod interface; -mod qname; -pub mod states; - -pub use self::interface::{ - Doctype, EmptyTag, EndTag, Pi, ShortTag, StartTag, Tag, TagKind, Token, TokenSink, -}; -pub use crate::{LocalName, Namespace, Prefix}; - -use crate::macros::time; -use crate::tendril::StrTendril; -use crate::{buffer_queue, Attribute, QualName, SmallCharSet}; -use log::debug; -use markup5ever::{ns, small_char_set, TokenizerResult}; -use std::borrow::Cow::{self, Borrowed}; -use std::cell::{Cell, RefCell, RefMut}; -use std::cmp::Reverse; -use std::collections::BTreeMap; -use std::mem::replace; - -use buffer_queue::{BufferQueue, FromSet, NotFromSet, SetResult}; -use char_ref::{CharRef, CharRefTokenizer}; -use qname::QualNameTokenizer; -use states::{AttrValueKind::*, DoctypeKind, DoctypeKind::*, XmlState}; - -/// Copy of Tokenizer options, with an impl for `Default`. -#[derive(Copy, Clone)] -pub struct XmlTokenizerOpts { - /// Report all parse errors described in the spec, at some - /// performance penalty? Default: false - pub exact_errors: bool, - - /// Discard a `U+FEFF BYTE ORDER MARK` if we see one at the beginning - /// of the stream? Default: true - pub discard_bom: bool, - - /// Keep a record of how long we spent in each state? Printed - /// when `end()` is called. Default: false - pub profile: bool, - - /// Initial state override. Only the test runner should use - /// a non-`None` value! - pub initial_state: Option, -} - -fn process_qname(tag_name: StrTendril) -> QualName { - // If tag name can't possibly contain full namespace, skip qualified name - // parsing altogether. For a tag to have namespace it must look like: - // a:b - // Since StrTendril are UTF-8, we know that minimal size in bytes must be - // three bytes minimum. - let split = if (*tag_name).len() < 3 { - None - } else { - QualNameTokenizer::new((*tag_name).as_bytes()).run() - }; - - match split { - None => QualName::new(None, ns!(), LocalName::from(&*tag_name)), - Some(col) => { - let len = (*tag_name).len() as u32; - let prefix = tag_name.subtendril(0, col); - let local = tag_name.subtendril(col + 1, len - col - 1); - let ns = ns!(); // Actual namespace URL set in XmlTreeBuilder::bind_qname - QualName::new(Some(Prefix::from(&*prefix)), ns, LocalName::from(&*local)) - }, - } -} - -fn option_push(opt_str: &mut Option, c: char) { - match *opt_str { - Some(ref mut s) => s.push_char(c), - None => *opt_str = Some(StrTendril::from_char(c)), - } -} - -impl Default for XmlTokenizerOpts { - fn default() -> XmlTokenizerOpts { - XmlTokenizerOpts { - exact_errors: false, - discard_bom: true, - profile: false, - initial_state: None, - } - } -} -/// The Xml tokenizer. -pub struct XmlTokenizer { - /// Options controlling the behavior of the tokenizer. - opts: XmlTokenizerOpts, - - /// Destination for tokens we emit. - pub sink: Sink, - - /// The abstract machine state as described in the spec. - state: Cell, - - /// Are we at the end of the file, once buffers have been processed - /// completely? This affects whether we will wait for lookahead or not. - at_eof: Cell, - - /// Tokenizer for character references, if we're tokenizing - /// one at the moment. - char_ref_tokenizer: RefCell>>, - - /// Current input character. Just consumed, may reconsume. - current_char: Cell, - - /// Should we reconsume the current input character? - reconsume: Cell, - - /// Did we just consume \r, translating it to \n? In that case we need - /// to ignore the next character if it's \n. - ignore_lf: Cell, - - /// Discard a U+FEFF BYTE ORDER MARK if we see one? Only done at the - /// beginning of the stream. - discard_bom: Cell, - - /// Temporary buffer - temp_buf: RefCell, - - /// Current tag kind. - current_tag_kind: Cell, - - /// Current tag name. - current_tag_name: RefCell, - - /// Current tag attributes. - current_tag_attrs: RefCell>, - - /// Current attribute name. - current_attr_name: RefCell, - - /// Current attribute value. - current_attr_value: RefCell, - - current_doctype: RefCell, - - /// Current comment. - current_comment: RefCell, - - /// Current CDATA section. - current_cdata: RefCell, - - /// Current processing instruction target. - current_pi_target: RefCell, - - /// Current processing instruction value. - current_pi_data: RefCell, - - /// Record of how many ns we spent in each state, if profiling is enabled. - state_profile: RefCell>, - - /// Record of how many ns we spent in the token sink. - time_in_sink: Cell, -} - -impl XmlTokenizer { - /// Create a new tokenizer which feeds tokens to a particular `TokenSink`. - pub fn new(sink: Sink, opts: XmlTokenizerOpts) -> XmlTokenizer { - if opts.profile && cfg!(for_c) { - panic!("Can't profile tokenizer when built as a C library"); - } - - let state = *opts.initial_state.as_ref().unwrap_or(&XmlState::Data); - let discard_bom = opts.discard_bom; - XmlTokenizer { - opts, - sink, - state: Cell::new(state), - char_ref_tokenizer: RefCell::new(None), - at_eof: Cell::new(false), - current_char: Cell::new('\0'), - reconsume: Cell::new(false), - ignore_lf: Cell::new(false), - temp_buf: RefCell::new(StrTendril::new()), - discard_bom: Cell::new(discard_bom), - current_tag_kind: Cell::new(StartTag), - current_tag_name: RefCell::new(StrTendril::new()), - current_tag_attrs: RefCell::new(vec![]), - current_attr_name: RefCell::new(StrTendril::new()), - current_attr_value: RefCell::new(StrTendril::new()), - current_comment: RefCell::new(StrTendril::new()), - current_cdata: RefCell::new(StrTendril::new()), - current_pi_data: RefCell::new(StrTendril::new()), - current_pi_target: RefCell::new(StrTendril::new()), - current_doctype: RefCell::new(Doctype::default()), - state_profile: RefCell::new(BTreeMap::new()), - time_in_sink: Cell::new(0), - } - } - - /// Feed an input string into the tokenizer. - pub fn feed(&self, input: &BufferQueue) -> TokenizerResult { - if input.is_empty() { - return TokenizerResult::Done; - } - - if self.discard_bom.get() { - if let Some(c) = input.peek() { - if c == '\u{feff}' { - input.next(); - } - } else { - return TokenizerResult::Done; - } - }; - - self.run(input) - } - - fn process_token(&self, token: Token) -> ProcessResult { - if self.opts.profile { - let (result, dt) = time!(self.sink.process_token(token)); - self.time_in_sink.set(self.time_in_sink.get() + dt); - result - } else { - self.sink.process_token(token) - } - } - - // Get the next input character, which might be the character - // 'c' that we already consumed from the buffers. - fn get_preprocessed_char(&self, mut c: char, input: &BufferQueue) -> Option { - if self.ignore_lf.get() { - self.ignore_lf.set(false); - if c == '\n' { - c = input.next()?; - } - } - - if c == '\r' { - self.ignore_lf.set(true); - c = '\n'; - } - - // Normalize \x00 into \uFFFD - if c == '\x00' { - c = '\u{FFFD}' - } - - // Exclude forbidden Unicode characters - if self.opts.exact_errors - && match c as u32 { - 0x01..=0x08 | 0x0B | 0x0E..=0x1F | 0x7F..=0x9F | 0xFDD0..=0xFDEF => true, - n if (n & 0xFFFE) == 0xFFFE => true, - _ => false, - } - { - let msg = format!("Bad character {c}"); - self.emit_error(Cow::Owned(msg)); - } - - debug!("got character {c}"); - self.current_char.set(c); - Some(c) - } - - fn bad_eof_error(&self) { - let msg = if self.opts.exact_errors { - Cow::from(format!("Saw EOF in state {:?}", self.state)) - } else { - Cow::from("Unexpected EOF") - }; - self.emit_error(msg); - } - - fn pop_except_from(&self, input: &BufferQueue, set: SmallCharSet) -> Option { - // Bail to the slow path for various corner cases. - // This means that `FromSet` can contain characters not in the set! - // It shouldn't matter because the fallback `FromSet` case should - // always do the same thing as the `NotFromSet` case. - if self.opts.exact_errors || self.reconsume.get() || self.ignore_lf.get() { - return self.get_char(input).map(FromSet); - } - - let d = input.pop_except_from(set); - debug!("got characters {d:?}"); - match d { - Some(FromSet(c)) => self.get_preprocessed_char(c, input).map(FromSet), - - // NB: We don't set self.current_char for a run of characters not - // in the set. It shouldn't matter for the codepaths that use - // this. - _ => d, - } - } - - // Check if the next characters are an ASCII case-insensitive match. See - // BufferQueue::eat. - // - // NB: this doesn't do input stream preprocessing or set the current input - // character. - fn eat(&self, input: &BufferQueue, pat: &str) -> Option { - input.push_front(replace(&mut *self.temp_buf.borrow_mut(), StrTendril::new())); - match input.eat(pat, u8::eq_ignore_ascii_case) { - None if self.at_eof.get() => Some(false), - None => { - let mut temp_buf = self.temp_buf.borrow_mut(); - while let Some(data) = input.next() { - temp_buf.push_char(data); - } - None - }, - Some(matched) => Some(matched), - } - } - - /// Run the state machine for as long as we can. - pub fn run(&self, input: &BufferQueue) -> TokenizerResult { - if self.opts.profile { - loop { - let state = self.state.get(); - let old_sink = self.time_in_sink.get(); - let (run, mut dt) = time!(self.step(input)); - dt -= self.time_in_sink.get() - old_sink; - let new = match self.state_profile.borrow_mut().get_mut(&state) { - Some(x) => { - *x += dt; - false - }, - None => true, - }; - if new { - // do this here because of borrow shenanigans - self.state_profile.borrow_mut().insert(state, dt); - } - match run { - ProcessResult::Continue => continue, - ProcessResult::Done => return TokenizerResult::Done, - ProcessResult::Script(handle) => return TokenizerResult::Script(handle), - } - } - } else { - loop { - match self.step(input) { - ProcessResult::Continue => continue, - ProcessResult::Done => return TokenizerResult::Done, - ProcessResult::Script(handle) => return TokenizerResult::Script(handle), - } - } - } - } - - //§ tokenization - // Get the next input character, if one is available. - fn get_char(&self, input: &BufferQueue) -> Option { - if self.reconsume.get() { - self.reconsume.set(false); - Some(self.current_char.get()) - } else { - input - .next() - .and_then(|c| self.get_preprocessed_char(c, input)) - } - } - - fn bad_char_error(&self) { - let msg = if self.opts.exact_errors { - let c = self.current_char.get(); - let state = self.state.get(); - Cow::from(format!("Saw {c} in state {state:?}")) - } else { - Cow::from("Bad character") - }; - self.emit_error(msg); - } - - fn discard_tag(&self) { - *self.current_tag_name.borrow_mut() = StrTendril::new(); - *self.current_tag_attrs.borrow_mut() = Vec::new(); - } - - fn create_tag(&self, kind: TagKind, c: char) { - self.discard_tag(); - self.current_tag_name.borrow_mut().push_char(c); - self.current_tag_kind.set(kind); - } - - // This method creates a PI token and - // sets its target to given char - fn create_pi(&self, c: char) { - *self.current_pi_target.borrow_mut() = StrTendril::new(); - *self.current_pi_data.borrow_mut() = StrTendril::new(); - self.current_pi_target.borrow_mut().push_char(c); - } - - fn emit_char(&self, c: char) { - self.process_token(Token::Characters(StrTendril::from_char(match c { - '\0' => '\u{FFFD}', - c => c, - }))); - } - - fn emit_short_tag(&self) -> ProcessResult { - self.current_tag_kind.set(ShortTag); - *self.current_tag_name.borrow_mut() = StrTendril::new(); - self.emit_current_tag() - } - - fn emit_empty_tag(&self) -> ProcessResult { - self.current_tag_kind.set(EmptyTag); - self.emit_current_tag() - } - - fn set_empty_tag(&self) { - self.current_tag_kind.set(EmptyTag); - } - - fn emit_start_tag(&self) -> ProcessResult { - self.current_tag_kind.set(StartTag); - self.emit_current_tag() - } - - fn emit_current_tag(&self) -> ProcessResult { - self.finish_attribute(); - - let qname = process_qname(replace( - &mut *self.current_tag_name.borrow_mut(), - StrTendril::new(), - )); - - match self.current_tag_kind.get() { - StartTag | EmptyTag => {}, - EndTag => { - if !self.current_tag_attrs.borrow().is_empty() { - self.emit_error(Borrowed("Attributes on an end tag")); - } - }, - ShortTag => { - if !self.current_tag_attrs.borrow().is_empty() { - self.emit_error(Borrowed("Attributes on a short tag")); - } - }, - } - - let token = Token::Tag(Tag { - kind: self.current_tag_kind.get(), - name: qname, - attrs: self.current_tag_attrs.take(), - }); - - self.process_token(token) - } - - // The string must not contain '\0'! - fn emit_chars(&self, b: StrTendril) { - self.process_token(Token::Characters(b)); - } - - // Emits the current Processing Instruction - fn emit_pi(&self) -> ProcessResult<::Handle> { - let token = Token::ProcessingInstruction(Pi { - target: replace(&mut *self.current_pi_target.borrow_mut(), StrTendril::new()), - data: replace(&mut *self.current_pi_data.borrow_mut(), StrTendril::new()), - }); - self.process_token(token) - } - - fn consume_char_ref(&self, addnl_allowed: Option) { - // NB: The char ref tokenizer assumes we have an additional allowed - // character iff we're tokenizing in an attribute value. - *self.char_ref_tokenizer.borrow_mut() = - Some(Box::new(CharRefTokenizer::new(addnl_allowed))); - } - - fn emit_eof(&self) { - self.process_token(Token::EndOfFile); - } - - fn emit_error(&self, error: Cow<'static, str>) { - self.process_token(Token::ParseError(error)); - } - - fn emit_current_comment(&self) { - let comment = self.current_comment.take(); - self.process_token(Token::Comment(comment)); - } - - fn emit_current_cdata(&self) -> ProcessResult<::Handle> { - let cdata = self.current_cdata.take(); - self.process_token(Token::Cdata(cdata)) - } - - fn emit_incomplete_cdata_as_characters(&self, trailing_brackets: usize) { - let mut cdata = self.current_cdata.take(); - for _ in 0..trailing_brackets { - cdata.push_char(']'); - } - self.process_token(Token::Characters(cdata)); - } - - fn emit_current_doctype(&self) { - let doctype = self.current_doctype.take(); - self.process_token(Token::Doctype(doctype)); - } - - fn doctype_id(&self, kind: DoctypeKind) -> RefMut<'_, Option> { - let current_doctype = self.current_doctype.borrow_mut(); - match kind { - DoctypeKind::Public => RefMut::map(current_doctype, |d| &mut d.public_id), - DoctypeKind::System => RefMut::map(current_doctype, |d| &mut d.system_id), - } - } - - fn clear_doctype_id(&self, kind: DoctypeKind) { - let mut id = self.doctype_id(kind); - match *id { - Some(ref mut s) => s.clear(), - None => *id = Some(StrTendril::new()), - } - } - - fn peek(&self, input: &BufferQueue) -> Option { - if self.reconsume.get() { - Some(self.current_char.get()) - } else { - input.peek() - } - } - - fn discard_char(&self, input: &BufferQueue) { - let c = self.get_char(input); - assert!(c.is_some()); - } - - fn unconsume(&self, input: &BufferQueue, buf: StrTendril) { - input.push_front(buf); - } -} - -// Shorthand for common state machine behaviors. -macro_rules! shorthand ( - ( $me:ident : emit $c:expr ) => ( $me.emit_char($c) ); - ( $me:ident : create_tag $kind:ident $c:expr ) => ( $me.create_tag($kind, $c) ); - ( $me:ident : push_tag $c:expr ) => ( $me.current_tag_name.borrow_mut().push_char($c) ); - ( $me:ident : discard_tag $input:expr ) => ( $me.discard_tag($input) ); - ( $me:ident : discard_char ) => ( $me.discard_char() ); - ( $me:ident : push_temp $c:expr ) => ( $me.temp_buf.borrow_mut().push_char($c) ); - ( $me:ident : emit_temp ) => ( $me.emit_temp_buf() ); - ( $me:ident : clear_temp ) => ( $me.clear_temp_buf() ); - ( $me:ident : create_attr $c:expr ) => ( $me.create_attribute($c) ); - ( $me:ident : push_name $c:expr ) => ( $me.current_attr_name.borrow_mut().push_char($c) ); - ( $me:ident : push_value $c:expr ) => ( $me.current_attr_value.borrow_mut().push_char($c) ); - ( $me:ident : append_value $c:expr ) => ( $me.current_attr_value.borrow_mut().push_tendril($c)); - ( $me:ident : push_comment $c:expr ) => ( $me.current_comment.borrow_mut().push_char($c) ); - ( $me:ident : append_comment $c:expr ) => ( $me.current_comment.borrow_mut().push_slice($c) ); - ( $me:ident : emit_comment ) => ( $me.emit_current_comment() ); - ( $me:ident : clear_comment ) => ( $me.current_comment.borrow_mut().clear() ); - ( $me:ident : push_cdata $c:expr ) => ( $me.current_cdata.borrow_mut().push_char($c) ); - ( $me:ident : clear_cdata ) => ( $me.current_cdata.borrow_mut().clear() ); - ( $me:ident : emit_incomplete_cdata $n:expr ) => ( $me.emit_incomplete_cdata_as_characters($n) ); - ( $me:ident : create_doctype ) => ( *$me.current_doctype.borrow_mut() = Doctype::default() ); - ( $me:ident : push_doctype_name $c:expr ) => ( option_push(&mut $me.current_doctype.borrow_mut().name, $c) ); - ( $me:ident : push_doctype_id $k:ident $c:expr ) => ( option_push(&mut $me.doctype_id($k), $c) ); - ( $me:ident : clear_doctype_id $k:ident ) => ( $me.clear_doctype_id($k) ); - ( $me:ident : emit_doctype ) => ( $me.emit_current_doctype() ); - ( $me:ident : error ) => ( $me.bad_char_error() ); - ( $me:ident : error_eof ) => ( $me.bad_eof_error() ); - ( $me:ident : create_pi $c:expr ) => ( $me.create_pi($c) ); - ( $me:ident : push_pi_target $c:expr ) => ( $me.current_pi_target.borrow_mut().push_char($c) ); - ( $me:ident : push_pi_data $c:expr ) => ( $me.current_pi_data.borrow_mut().push_char($c) ); - ( $me:ident : set_empty_tag ) => ( $me.set_empty_tag() ); -); - -// Tracing of tokenizer actions. This adds significant bloat and compile time, -// so it's behind a cfg flag. -#[cfg(feature = "trace_tokenizer")] -macro_rules! sh_trace ( ( $me:ident : $($cmds:tt)* ) => ({ - debug!(" {:?}", stringify!($($cmds)*)); - shorthand!($me : $($cmds)*); -})); - -#[cfg(not(feature = "trace_tokenizer"))] -macro_rules! sh_trace ( ( $me:ident : $($cmds:tt)* ) => ( shorthand!($me: $($cmds)*) ) ); - -// A little DSL for sequencing shorthand actions. -macro_rules! go ( - // A pattern like $($cmd:tt)* ; $($rest:tt)* causes parse ambiguity. - // We have to tell the parser how much lookahead we need. - - ( $me:ident : $a:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a); go!($me: $($rest)*); }); - ( $me:ident : $a:tt $b:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b); go!($me: $($rest)*); }); - ( $me:ident : $a:tt $b:tt $c:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b $c); go!($me: $($rest)*); }); - ( $me:ident : $a:tt $b:tt $c:tt $d:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b $c $d); go!($me: $($rest)*); }); - - // These can only come at the end. - - ( $me:ident : to $s:expr ) => ({ $me.state.set($s); return ProcessResult::Continue; }); - ( $me:ident : reconsume $s:expr ) => ({ $me.reconsume.set(true); go!($me: to $s); }); - ( $me:ident : consume_char_ref ) => ({ $me.consume_char_ref(None); return ProcessResult::Continue; }); - ( $me:ident : consume_char_ref $addnl:expr ) => ({ $me.consume_char_ref(Some($addnl)); return ProcessResult::Continue; }); - - // We have a default next state after emitting a tag, but the sink can override. - ( $me:ident : emit_tag $s:expr ) => ({ - $me.state.set($s); - return $me.emit_current_tag(); - }); - - // We have a special when dealing with empty and short tags in Xml - ( $me:ident : emit_short_tag $s:expr ) => ({ - $me.state.set($s); - return $me.emit_short_tag(); - }); - - ( $me:ident : emit_empty_tag $s:ident ) => ({ - $me.state.set(XmlState::$s); - return $me.emit_empty_tag(); - }); - - ( $me:ident : emit_start_tag $s:ident ) => ({ - $me.state.set(XmlState::$s); - return $me.emit_start_tag(); - }); - - ( $me:ident : emit_pi $s:ident ) => ({ - $me.state.set(XmlState::$s); - return $me.emit_pi(); - }); - - ( $me:ident : emit_cdata $s:ident ) => ({ - $me.state.set(XmlState::$s); - return $me.emit_current_cdata(); - }); - - ( $me:ident : eof ) => ({ $me.emit_eof(); return ProcessResult::Done; }); - - // If nothing else matched, it's a single command - ( $me:ident : $($cmd:tt)+ ) => ( sh_trace!($me: $($cmd)+) ); - - // or nothing. - ( $me:ident : ) => (()); -); - -// This is a macro because it can cause early return -// from the function where it is used. -macro_rules! get_char ( ($me:expr, $input:expr) => {{ - let Some(character) = $me.get_char($input) else { - return ProcessResult::Done; - }; - character -}}); - -macro_rules! eat ( ($me:expr, $input:expr, $pat:expr) => {{ - let Some(value) = $me.eat($input, $pat) else { - return ProcessResult::Done; - }; - value -}}); - -/// The result of a single tokenization operation -pub enum ProcessResult { - /// The tokenizer needs more input before it can continue - Done, - /// The tokenizer can be invoked again immediately - Continue, - /// The tokenizer encountered a script element that must be executed - /// before tokenization can continue - Script(Handle), -} - -impl XmlTokenizer { - // Run the state machine for a while. - #[allow(clippy::never_loop)] - fn step(&self, input: &BufferQueue) -> ProcessResult { - if self.char_ref_tokenizer.borrow().is_some() { - return self.step_char_ref_tokenizer(input); - } - - debug!("processing in state {:?}", self.state); - match self.state.get() { - //§ data-state - XmlState::Data => loop { - let Some(popped_element) = - self.pop_except_from(input, small_char_set!('\r' '&' '<')) - else { - return ProcessResult::Done; - }; - - match popped_element { - FromSet('&') => go!(self: consume_char_ref), - FromSet('<') => go!(self: to XmlState::TagState), - FromSet(c) => go!(self: emit c), - NotFromSet(b) => self.emit_chars(b), - } - }, - //§ tag-state - XmlState::TagState => loop { - match get_char!(self, input) { - '!' => go!(self: to XmlState::MarkupDecl), - '/' => go!(self: to XmlState::EndTagState), - '?' => go!(self: to XmlState::Pi), - '\t' | '\n' | ' ' | ':' | '<' | '>' => { - go!(self: error; emit '<'; reconsume XmlState::Data) - }, - cl => go!(self: create_tag StartTag cl; to XmlState::TagName), - } - }, - //§ end-tag-state - XmlState::EndTagState => loop { - match get_char!(self, input) { - '>' => go!(self: emit_short_tag XmlState::Data), - '\t' | '\n' | ' ' | '<' | ':' => { - go!(self: error; emit '<'; emit '/'; reconsume XmlState::Data) - }, - cl => go!(self: create_tag EndTag cl; to XmlState::EndTagName), - } - }, - //§ end-tag-name-state - XmlState::EndTagName => loop { - match get_char!(self, input) { - '\t' | '\n' | ' ' => go!(self: to XmlState::EndTagNameAfter), - '/' => go!(self: error; to XmlState::EndTagNameAfter), - '>' => go!(self: emit_tag XmlState::Data), - cl => go!(self: push_tag cl), - } - }, - //§ end-tag-name-after-state - XmlState::EndTagNameAfter => loop { - match get_char!(self, input) { - '>' => go!(self: emit_tag XmlState::Data), - '\t' | '\n' | ' ' => (), - _ => self.emit_error(Borrowed("Unexpected element in tag name")), - } - }, - //§ pi-state - XmlState::Pi => loop { - match get_char!(self, input) { - '\t' | '\n' | ' ' => go!(self: error; reconsume XmlState::BogusComment), - cl => go!(self: create_pi cl; to XmlState::PiTarget), - } - }, - //§ pi-target-state - XmlState::PiTarget => loop { - match get_char!(self, input) { - '\t' | '\n' | ' ' => go!(self: to XmlState::PiTargetAfter), - '?' => go!(self: to XmlState::PiAfter), - cl => go!(self: push_pi_target cl), - } - }, - //§ pi-target-after-state - XmlState::PiTargetAfter => loop { - match get_char!(self, input) { - '\t' | '\n' | ' ' => (), - _ => go!(self: reconsume XmlState::PiData), - } - }, - //§ pi-data-state - XmlState::PiData => loop { - match get_char!(self, input) { - '?' => go!(self: to XmlState::PiAfter), - cl => go!(self: push_pi_data cl), - } - }, - //§ pi-after-state - XmlState::PiAfter => loop { - match get_char!(self, input) { - '>' => go!(self: emit_pi Data), - // The '?' that got us here is not part of a '?>', so it is - // ordinary data. The one we just read may still close the - // processing instruction, so stay in this state. - '?' => go!(self: push_pi_data '?'; to XmlState::PiAfter), - cl => go!(self: push_pi_data '?'; push_pi_data cl; to XmlState::PiData), - } - }, - //§ markup-declaration-state - XmlState::MarkupDecl => loop { - if eat!(self, input, "--") { - go!(self: clear_comment; to XmlState::CommentStart); - } else if eat!(self, input, "[CDATA[") { - go!(self: clear_cdata; to XmlState::Cdata); - } else if eat!(self, input, "DOCTYPE") { - go!(self: to XmlState::Doctype); - } else { - // FIXME: 'error' gives wrong message - go!(self: error; to XmlState::BogusComment); - } - }, - //§ comment-start-state - XmlState::CommentStart => loop { - match get_char!(self, input) { - '-' => go!(self: to XmlState::CommentStartDash), - '>' => go!(self: error; emit_comment; to XmlState::Data), - _ => go!(self: reconsume XmlState::Comment), - } - }, - //§ comment-start-dash-state - XmlState::CommentStartDash => loop { - match get_char!(self, input) { - '-' => go!(self: to XmlState::CommentEnd), - '>' => go!(self: error; emit_comment; to XmlState::Data), - _ => go!(self: push_comment '-'; reconsume XmlState::Comment), - } - }, - //§ comment-state - XmlState::Comment => loop { - match get_char!(self, input) { - '<' => go!(self: push_comment '<'; to XmlState::CommentLessThan), - '-' => go!(self: to XmlState::CommentEndDash), - c => go!(self: push_comment c), - } - }, - //§ comment-less-than-sign-state - XmlState::CommentLessThan => loop { - match get_char!(self, input) { - '!' => go!(self: push_comment '!';to XmlState::CommentLessThanBang), - '<' => go!(self: push_comment '<'), - _ => go!(self: reconsume XmlState::Comment), - } - }, - //§ comment-less-than-sign-bang-state - XmlState::CommentLessThanBang => loop { - match get_char!(self, input) { - '-' => go!(self: to XmlState::CommentLessThanBangDash), - _ => go!(self: reconsume XmlState::Comment), - } - }, - //§ comment-less-than-sign-bang-dash-state - XmlState::CommentLessThanBangDash => loop { - match get_char!(self, input) { - '-' => go!(self: to XmlState::CommentLessThanBangDashDash), - _ => go!(self: reconsume XmlState::CommentEndDash), - } - }, - //§ comment-less-than-sign-bang-dash-dash-state - XmlState::CommentLessThanBangDashDash => loop { - match get_char!(self, input) { - '>' => go!(self: reconsume XmlState::CommentEnd), - _ => go!(self: error; reconsume XmlState::CommentEnd), - } - }, - //§ comment-end-dash-state - XmlState::CommentEndDash => loop { - match get_char!(self, input) { - '-' => go!(self: to XmlState::CommentEnd), - _ => go!(self: push_comment '-'; reconsume XmlState::Comment), - } - }, - //§ comment-end-state - XmlState::CommentEnd => loop { - match get_char!(self, input) { - '>' => go!(self: emit_comment; to XmlState::Data), - '!' => go!(self: to XmlState::CommentEndBang), - '-' => go!(self: push_comment '-'), - _ => go!(self: append_comment "--"; reconsume XmlState::Comment), - } - }, - //§ comment-end-bang-state - XmlState::CommentEndBang => loop { - match get_char!(self, input) { - '-' => go!(self: append_comment "--!"; to XmlState::CommentEndDash), - '>' => go!(self: error; emit_comment; to XmlState::Data), - _ => go!(self: append_comment "--!"; reconsume XmlState::Comment), - } - }, - //§ bogus-comment-state - XmlState::BogusComment => loop { - match get_char!(self, input) { - '>' => go!(self: emit_comment; to XmlState::Data), - c => go!(self: push_comment c), - } - }, - //§ cdata-state - XmlState::Cdata => loop { - match get_char!(self, input) { - ']' => go!(self: to XmlState::CdataBracket), - cl => go!(self: push_cdata cl), - } - }, - //§ cdata-bracket-state - XmlState::CdataBracket => loop { - match get_char!(self, input) { - ']' => go!(self: to XmlState::CdataEnd), - cl => go!(self: push_cdata ']'; push_cdata cl; to XmlState::Cdata), - } - }, - //§ cdata-end-state - XmlState::CdataEnd => loop { - match get_char!(self, input) { - '>' => go!(self: emit_cdata Data), - ']' => go!(self: push_cdata ']'), - cl => { - go!(self: push_cdata ']'; push_cdata ']'; push_cdata cl; to XmlState::Cdata) - }, - } - }, - //§ tag-name-state - XmlState::TagName => loop { - match get_char!(self, input) { - '\t' | '\n' | ' ' => go!(self: to XmlState::TagAttrNameBefore), - '>' => go!(self: emit_tag XmlState::Data), - '/' => go!(self: set_empty_tag; to XmlState::TagEmpty), - cl => go!(self: push_tag cl), - } - }, - //§ empty-tag-state - XmlState::TagEmpty => loop { - match get_char!(self, input) { - '>' => go!(self: emit_empty_tag Data), - _ => go!(self: reconsume XmlState::TagAttrValueBefore), - } - }, - //§ tag-attribute-name-before-state - XmlState::TagAttrNameBefore => loop { - match get_char!(self, input) { - '\t' | '\n' | ' ' => (), - '>' => go!(self: emit_tag XmlState::Data), - '/' => go!(self: set_empty_tag; to XmlState::TagEmpty), - ':' => go!(self: error), - cl => go!(self: create_attr cl; to XmlState::TagAttrName), - } - }, - //§ tag-attribute-name-state - XmlState::TagAttrName => loop { - match get_char!(self, input) { - '=' => go!(self: to XmlState::TagAttrValueBefore), - '>' => go!(self: emit_tag XmlState::Data), - '\t' | '\n' | ' ' => go!(self: to XmlState::TagAttrNameAfter), - '/' => go!(self: set_empty_tag; to XmlState::TagEmpty), - cl => go!(self: push_name cl), - } - }, - //§ tag-attribute-name-after-state - XmlState::TagAttrNameAfter => loop { - match get_char!(self, input) { - '\t' | '\n' | ' ' => (), - '=' => go!(self: to XmlState::TagAttrValueBefore), - '>' => go!(self: emit_tag XmlState::Data), - '/' => go!(self: set_empty_tag; to XmlState::TagEmpty), - cl => go!(self: create_attr cl; to XmlState::TagAttrName), - } - }, - //§ tag-attribute-value-before-state - XmlState::TagAttrValueBefore => loop { - match get_char!(self, input) { - '\t' | '\n' | ' ' => (), - '"' => go!(self: to XmlState::TagAttrValue(DoubleQuoted)), - '\'' => go!(self: to XmlState::TagAttrValue(SingleQuoted)), - '&' => go!(self: reconsume XmlState::TagAttrValue(Unquoted)), - '>' => go!(self: emit_tag XmlState::Data), - cl => go!(self: push_value cl; to XmlState::TagAttrValue(Unquoted)), - } - }, - //§ tag-attribute-value-double-quoted-state - XmlState::TagAttrValue(DoubleQuoted) => loop { - let Some(popped_element) = - self.pop_except_from(input, small_char_set!('\n' '"' '&')) - else { - return ProcessResult::Done; - }; - - match popped_element { - FromSet('"') => go!(self: to XmlState::TagAttrNameBefore), - FromSet('&') => go!(self: consume_char_ref '"' ), - FromSet(c) => go!(self: push_value c), - NotFromSet(ref b) => go!(self: append_value b), - } - }, - //§ tag-attribute-value-single-quoted-state - XmlState::TagAttrValue(SingleQuoted) => loop { - let Some(popped_element) = - self.pop_except_from(input, small_char_set!('\n' '\'' '&')) - else { - return ProcessResult::Done; - }; - - match popped_element { - FromSet('\'') => go!(self: to XmlState::TagAttrNameBefore), - FromSet('&') => go!(self: consume_char_ref '\''), - FromSet(c) => go!(self: push_value c), - NotFromSet(ref b) => go!(self: append_value b), - } - }, - //§ tag-attribute-value-double-quoted-state - XmlState::TagAttrValue(Unquoted) => loop { - let Some(popped_element) = - self.pop_except_from(input, small_char_set!('\n' '\t' ' ' '&' '>')) - else { - return ProcessResult::Done; - }; - - match popped_element { - FromSet('\t') | FromSet('\n') | FromSet(' ') => { - go!(self: to XmlState::TagAttrNameBefore) - }, - FromSet('&') => go!(self: consume_char_ref), - FromSet('>') => go!(self: emit_tag XmlState::Data), - FromSet(c) => go!(self: push_value c), - NotFromSet(ref b) => go!(self: append_value b), - } - }, - - //§ doctype-state - XmlState::Doctype => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => go!(self: to XmlState::BeforeDoctypeName), - _ => go!(self: error; reconsume XmlState::BeforeDoctypeName), - } - }, - //§ before-doctype-name-state - XmlState::BeforeDoctypeName => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '>' => go!(self: error; emit_doctype; to XmlState::Data), - c => go!(self: create_doctype; push_doctype_name (c.to_ascii_lowercase()); - to XmlState::DoctypeName), - } - }, - //§ doctype-name-state - XmlState::DoctypeName => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => go!(self: to XmlState::AfterDoctypeName), - '>' => go!(self: emit_doctype; to XmlState::Data), - c => go!(self: push_doctype_name (c.to_ascii_lowercase()); - to XmlState::DoctypeName), - } - }, - //§ after-doctype-name-state - XmlState::AfterDoctypeName => loop { - if eat!(self, input, "public") { - go!(self: to XmlState::AfterDoctypeKeyword(Public)); - } else if eat!(self, input, "system") { - go!(self: to XmlState::AfterDoctypeKeyword(System)); - } else { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '>' => go!(self: emit_doctype; to XmlState::Data), - _ => go!(self: error; to XmlState::BogusDoctype), - } - } - }, - //§ after-doctype-public-keyword-state - XmlState::AfterDoctypeKeyword(Public) => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => { - go!(self: to XmlState::BeforeDoctypeIdentifier(Public)) - }, - '"' => { - go!(self: error; clear_doctype_id Public; to XmlState::DoctypeIdentifierDoubleQuoted(Public)) - }, - '\'' => { - go!(self: error; clear_doctype_id Public; to XmlState::DoctypeIdentifierSingleQuoted(Public)) - }, - '>' => go!(self: error; emit_doctype; to XmlState::Data), - _ => go!(self: error; to XmlState::BogusDoctype), - } - }, - //§ after-doctype-system-keyword-state - XmlState::AfterDoctypeKeyword(System) => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => { - go!(self: to XmlState::BeforeDoctypeIdentifier(System)) - }, - '"' => { - go!(self: error; clear_doctype_id System; to XmlState::DoctypeIdentifierDoubleQuoted(System)) - }, - '\'' => { - go!(self: error; clear_doctype_id System; to XmlState::DoctypeIdentifierSingleQuoted(System)) - }, - '>' => go!(self: error; emit_doctype; to XmlState::Data), - _ => go!(self: error; to XmlState::BogusDoctype), - } - }, - //§ before_doctype_public_identifier_state before_doctype_system_identifier_state - XmlState::BeforeDoctypeIdentifier(kind) => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '"' => { - go!(self: clear_doctype_id kind; to XmlState::DoctypeIdentifierDoubleQuoted(kind)) - }, - '\'' => { - go!(self: clear_doctype_id kind; to XmlState::DoctypeIdentifierSingleQuoted(kind)) - }, - '>' => go!(self: error; emit_doctype; to XmlState::Data), - _ => go!(self: error; to XmlState::BogusDoctype), - } - }, - //§ doctype_public_identifier_double_quoted_state doctype_system_identifier_double_quoted_state - XmlState::DoctypeIdentifierDoubleQuoted(kind) => loop { - match get_char!(self, input) { - '"' => go!(self: to XmlState::AfterDoctypeIdentifier(kind)), - '>' => go!(self: error; emit_doctype; to XmlState::Data), - c => go!(self: push_doctype_id kind c), - } - }, - //§ doctype_public_identifier_single_quoted_state doctype_system_identifier_single_quoted_state - XmlState::DoctypeIdentifierSingleQuoted(kind) => loop { - match get_char!(self, input) { - '\'' => go!(self: to XmlState::AfterDoctypeIdentifier(kind)), - '>' => go!(self: error; emit_doctype; to XmlState::Data), - c => go!(self: push_doctype_id kind c), - } - }, - //§ doctype_public_identifier_single_quoted_state - XmlState::AfterDoctypeIdentifier(Public) => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => { - go!(self: to XmlState::BetweenDoctypePublicAndSystemIdentifiers) - }, - '\'' => { - go!(self: error; clear_doctype_id System; to XmlState::DoctypeIdentifierSingleQuoted(System)) - }, - '"' => { - go!(self: error; clear_doctype_id System; to XmlState::DoctypeIdentifierDoubleQuoted(System)) - }, - '>' => go!(self: emit_doctype; to XmlState::Data), - _ => go!(self: error; to XmlState::BogusDoctype), - } - }, - //§ doctype_system_identifier_single_quoted_state - XmlState::AfterDoctypeIdentifier(System) => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '>' => go!(self: emit_doctype; to XmlState::Data), - _ => go!(self: error; to XmlState::BogusDoctype), - } - }, - //§ between_doctype_public_and_system_identifier_state - XmlState::BetweenDoctypePublicAndSystemIdentifiers => loop { - match get_char!(self, input) { - '\t' | '\n' | '\x0C' | ' ' => (), - '>' => go!(self: emit_doctype; to XmlState::Data), - '\'' => go!(self: to XmlState::DoctypeIdentifierSingleQuoted(System)), - '"' => go!(self: to XmlState::DoctypeIdentifierDoubleQuoted(System)), - _ => go!(self: error; to XmlState::BogusDoctype), - } - }, - //§ bogus_doctype_state - XmlState::BogusDoctype => loop { - if get_char!(self, input) == '>' { - go!(self: emit_doctype; to XmlState::Data); - } - }, - } - } - - /// Indicate that we have reached the end of the input. - pub fn end(&self) { - // Handle EOF in the char ref sub-tokenizer, if there is one. - // Do this first because it might un-consume stuff. - let input = BufferQueue::default(); - match self.char_ref_tokenizer.take() { - None => (), - Some(mut tok) => { - tok.end_of_file(self, &input); - self.process_char_ref(tok.get_result()); - }, - } - - // Process all remaining buffered input. - // If we're waiting for lookahead, we're not gonna get it. - self.at_eof.set(true); - let _ = self.run(&input); - - loop { - if !matches!(self.eof_step(), ProcessResult::Continue) { - break; - } - } - - self.sink.end(); - - if self.opts.profile { - self.dump_profile(); - } - } - - #[cfg(for_c)] - fn dump_profile(&self) { - unreachable!(); - } - - #[cfg(not(for_c))] - fn dump_profile(&self) { - let mut results: Vec<(XmlState, u64)> = self - .state_profile - .borrow() - .iter() - .map(|(s, t)| (*s, *t)) - .collect(); - results.sort_by_key(|&(_, x)| Reverse(x)); - - let total: u64 = results.iter().map(|&(_, t)| t).sum(); - debug!("\nTokenizer profile, in nanoseconds"); - debug!( - "\n{:12} total in token sink", - self.time_in_sink.get() - ); - debug!("\n{total:12} total in tokenizer"); - - for (k, v) in results.into_iter() { - let pct = 100.0 * (v as f64) / (total as f64); - debug!("{v:12} {pct:4.1}% {k:?}"); - } - } - - fn eof_step(&self) -> ProcessResult { - debug!("processing EOF in state {:?}", self.state.get()); - match self.state.get() { - XmlState::Data => go!(self: eof), - XmlState::CommentStart | XmlState::CommentLessThan | XmlState::CommentLessThanBang => { - go!(self: reconsume XmlState::Comment) - }, - XmlState::CommentLessThanBangDash => go!(self: reconsume XmlState::CommentEndDash), - XmlState::CommentLessThanBangDashDash => go!(self: reconsume XmlState::CommentEnd), - XmlState::CommentStartDash - | XmlState::Comment - | XmlState::CommentEndDash - | XmlState::CommentEnd - | XmlState::CommentEndBang => go!(self: error_eof; emit_comment; eof), - XmlState::TagState => go!(self: error_eof; emit '<'; to XmlState::Data), - XmlState::EndTagState => go!(self: error_eof; emit '<'; emit '/'; to XmlState::Data), - XmlState::TagEmpty => go!(self: error_eof; to XmlState::TagAttrNameBefore), - XmlState::Cdata => go!(self: error_eof; emit_incomplete_cdata 0; to XmlState::Data), - XmlState::CdataBracket => { - go!(self: error_eof; emit_incomplete_cdata 1; to XmlState::Data) - }, - XmlState::CdataEnd => go!(self: error_eof; emit_incomplete_cdata 2; to XmlState::Data), - XmlState::Pi => go!(self: error_eof; to XmlState::BogusComment), - XmlState::PiTargetAfter => go!(self: reconsume XmlState::PiData), - // The '?' we consumed to get here never became a '?>', so keep it. - XmlState::PiAfter => go!(self: push_pi_data '?'; reconsume XmlState::PiData), - XmlState::MarkupDecl => go!(self: error_eof; to XmlState::BogusComment), - XmlState::TagName - | XmlState::TagAttrNameBefore - | XmlState::EndTagName - | XmlState::TagAttrNameAfter - | XmlState::EndTagNameAfter - | XmlState::TagAttrValueBefore - | XmlState::TagAttrValue(_) => go!(self: error_eof; emit_tag XmlState::Data), - XmlState::PiData | XmlState::PiTarget => go!(self: error_eof; emit_pi Data), - XmlState::TagAttrName => go!(self: error_eof; emit_start_tag Data), - XmlState::BeforeDoctypeName - | XmlState::Doctype - | XmlState::DoctypeName - | XmlState::AfterDoctypeName - | XmlState::AfterDoctypeKeyword(_) - | XmlState::BeforeDoctypeIdentifier(_) - | XmlState::AfterDoctypeIdentifier(_) - | XmlState::DoctypeIdentifierSingleQuoted(_) - | XmlState::DoctypeIdentifierDoubleQuoted(_) - | XmlState::BetweenDoctypePublicAndSystemIdentifiers => { - go!(self: error_eof; emit_doctype; to XmlState::Data) - }, - XmlState::BogusDoctype => go!(self: emit_doctype; to XmlState::Data), - XmlState::BogusComment => go!(self: emit_comment; to XmlState::Data), - } - } - - fn process_char_ref(&self, char_ref: CharRef) { - let CharRef { - mut chars, - mut num_chars, - } = char_ref; - - if num_chars == 0 { - chars[0] = '&'; - num_chars = 1; - } - - for i in 0..num_chars { - let c = chars[i as usize]; - match self.state.get() { - XmlState::Data | XmlState::Cdata => go!(self: emit c), - - XmlState::TagAttrValue(_) => go!(self: push_value c), - - _ => panic!( - "state {:?} should not be reachable in process_char_ref", - self.state.get() - ), - } - } - } - - fn step_char_ref_tokenizer(&self, input: &BufferQueue) -> ProcessResult { - let mut tok = self.char_ref_tokenizer.take().unwrap(); - let outcome = tok.step(self, input); - - let progress = match outcome { - char_ref::Done => { - self.process_char_ref(tok.get_result()); - return ProcessResult::Continue; - }, - - char_ref::Stuck => ProcessResult::Done, - char_ref::Progress => ProcessResult::Continue, - }; - - *self.char_ref_tokenizer.borrow_mut() = Some(tok); - progress - } - - fn finish_attribute(&self) { - if self.current_attr_name.borrow().is_empty() { - return; - } - - let qname = process_qname(replace( - &mut self.current_attr_name.borrow_mut(), - StrTendril::new(), - )); - - // Check for a duplicate attribute. Two attributes are the same only if - // both their prefix and their local name match, so xml:lang and lang - // are distinct names and may sit on the same element. - // FIXME: the spec says we should error as soon as the name is finished. - // FIXME: linear time search, do we care? - let dup = self - .current_tag_attrs - .borrow() - .iter() - .any(|a| a.name.prefix == qname.prefix && a.name.local == qname.local); - - if dup { - self.emit_error(Borrowed("Duplicate attribute")); - self.current_attr_value.borrow_mut().clear(); - } else { - let attr = Attribute { - name: qname.clone(), - value: replace(&mut self.current_attr_value.borrow_mut(), StrTendril::new()), - }; - - // Keep source order. The tree builder performs a declaration-only - // pass before binding any qualified names, so declarations do not - // need to be moved ahead of ordinary attributes here. - self.current_tag_attrs.borrow_mut().push(attr); - } - } - - fn create_attribute(&self, c: char) { - self.finish_attribute(); - - self.current_attr_name.borrow_mut().push_char(c); - } -} - -#[cfg(test)] -mod test { - - use super::{process_qname, ProcessResult, Token, TokenSink, XmlTokenizer}; - use crate::tendril::{SliceExt, StrTendril}; - use crate::{LocalName, Prefix}; - use markup5ever::buffer_queue::BufferQueue; - use std::cell::RefCell; - - struct PiCollector { - pis: RefCell>, - } - - impl TokenSink for PiCollector { - type Handle = (); - - fn process_token(&self, token: Token) -> ProcessResult<()> { - if let Token::ProcessingInstruction(pi) = token { - self.pis - .borrow_mut() - .push((pi.target.to_string(), pi.data.to_string())); - } - ProcessResult::Continue - } - } - - struct ErrorCollector { - errors: RefCell>, - } - - impl TokenSink for ErrorCollector { - type Handle = (); - - fn process_token(&self, token: Token) -> ProcessResult<()> { - if let Token::ParseError(error) = token { - self.errors.borrow_mut().push(error.to_string()); - } - ProcessResult::Continue - } - } - - fn tokenize_pis(input: &str) -> Vec<(String, String)> { - let sink = PiCollector { - pis: RefCell::new(Vec::new()), - }; - let queue = BufferQueue::default(); - queue.push_back(StrTendril::from(input)); - let tokenizer = XmlTokenizer::new(sink, Default::default()); - let _ = tokenizer.feed(&queue); - tokenizer.end(); - tokenizer.sink.pis.into_inner() - } - - fn tokenize_errors(input: &str) -> Vec { - let sink = ErrorCollector { - errors: RefCell::new(Vec::new()), - }; - let queue = BufferQueue::default(); - queue.push_back(StrTendril::from(input)); - let tokenizer = XmlTokenizer::new(sink, Default::default()); - let _ = tokenizer.feed(&queue); - tokenizer.end(); - tokenizer.sink.errors.into_inner() - } - - #[test] - fn pi_data_keeps_question_marks() { - assert_eq!( - tokenize_pis(r#""#), - vec![( - "xml-stylesheet".to_owned(), - r#"href="style.xsl?v=2""#.to_owned() - )] - ); - - assert_eq!( - tokenize_pis(""), - vec![("target".to_owned(), "a?b?c".to_owned())] - ); - - // A run of question marks only ends the instruction when one of them - // is followed by '>'. - assert_eq!( - tokenize_pis(""), - vec![("target".to_owned(), "a??".to_owned())] - ); - } - - #[test] - fn pi_only_ends_on_question_mark_gt() { - // The '>' here is data, because the character before it is not a '?'. - assert_eq!( - tokenize_pis("c?>"), - vec![("target".to_owned(), "a?b>c".to_owned())] - ); - } - - #[test] - fn pi_without_data_is_empty() { - assert_eq!( - tokenize_pis(""), - vec![("target".to_owned(), String::new())] - ); - - assert_eq!( - tokenize_pis(""), - vec![("target".to_owned(), String::new())] - ); - } - - #[test] - fn unterminated_pi_keeps_trailing_question_mark() { - assert_eq!( - tokenize_pis(""#).is_empty()); - assert!(tokenize_errors(r#""#).is_empty()); - } - - #[test] - fn different_prefixes_are_left_to_the_tree_builder() { - // Whether these two are duplicates depends on what a and b are bound - // to, and the tokenizer has no bindings, so it says nothing either way. - // XmlTreeBuilder::bind_attr_qname resolves the prefixes and compares - // expanded names, which is where a real duplicate gets caught. - assert!(tokenize_errors(r#""#).is_empty()); - } - - #[test] - fn real_duplicates_are_still_reported() { - assert_eq!( - tokenize_errors(r#""#), - vec!["Duplicate attribute".to_owned()] - ); - - assert_eq!( - tokenize_errors(r#""#), - vec!["Duplicate attribute".to_owned()] - ); - } - - #[test] - fn simple_namespace() { - let qname = process_qname("prefix:local".to_tendril()); - assert_eq!(qname.prefix, Some(Prefix::from("prefix"))); - assert_eq!(qname.local, LocalName::from("local")); - - let qname = process_qname("a:b".to_tendril()); - assert_eq!(qname.prefix, Some(Prefix::from("a"))); - assert_eq!(qname.local, LocalName::from("b")); - } - - #[test] - fn wrong_namespaces() { - let qname = process_qname(":local".to_tendril()); - assert_eq!(qname.prefix, None); - assert_eq!(qname.local, LocalName::from(":local")); - - let qname = process_qname("::local".to_tendril()); - assert_eq!(qname.prefix, None); - assert_eq!(qname.local, LocalName::from("::local")); - - let qname = process_qname("a::local".to_tendril()); - assert_eq!(qname.prefix, None); - assert_eq!(qname.local, LocalName::from("a::local")); - - let qname = process_qname("fake::".to_tendril()); - assert_eq!(qname.prefix, None); - assert_eq!(qname.local, LocalName::from("fake::")); - - let qname = process_qname(":::".to_tendril()); - assert_eq!(qname.prefix, None); - assert_eq!(qname.local, LocalName::from(":::")); - - let qname = process_qname(":a:b:".to_tendril()); - assert_eq!(qname.prefix, None); - assert_eq!(qname.local, LocalName::from(":a:b:")); - } -} diff --git a/vendor/xml5ever-0.40.0/src/tokenizer/qname.rs b/vendor/xml5ever-0.40.0/src/tokenizer/qname.rs deleted file mode 100644 index 886dafc2d0..0000000000 --- a/vendor/xml5ever-0.40.0/src/tokenizer/qname.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -enum QualNameState { - BeforeName, - InName, - AfterColon, -} - -pub struct QualNameTokenizer<'a> { - state: QualNameState, - slice: &'a [u8], - valid_index: Option, - curr_ind: usize, -} - -impl QualNameTokenizer<'_> { - pub fn new(tag: &[u8]) -> QualNameTokenizer<'_> { - QualNameTokenizer { - state: QualNameState::BeforeName, - slice: tag, - valid_index: None, - curr_ind: 0, - } - } - - pub fn run(&mut self) -> Option { - if !self.slice.is_empty() { - loop { - if !self.step() { - break; - } - } - } - self.valid_index - } - - fn incr(&mut self) -> bool { - if self.curr_ind + 1 < self.slice.len() { - self.curr_ind += 1; - return true; - } - false - } - - fn step(&mut self) -> bool { - match self.state { - QualNameState::BeforeName => self.do_before_name(), - QualNameState::InName => self.do_in_name(), - QualNameState::AfterColon => self.do_after_colon(), - } - } - - fn do_before_name(&mut self) -> bool { - if self.slice[self.curr_ind] == b':' { - false - } else { - self.state = QualNameState::InName; - self.incr() - } - } - - fn do_in_name(&mut self) -> bool { - if self.slice[self.curr_ind] == b':' && self.curr_ind + 1 < self.slice.len() { - self.valid_index = Some(self.curr_ind as u32); - self.state = QualNameState::AfterColon; - } - self.incr() - } - - fn do_after_colon(&mut self) -> bool { - if self.slice[self.curr_ind] == b':' { - self.valid_index = None; - return false; - } - self.incr() - } -} diff --git a/vendor/xml5ever-0.40.0/src/tokenizer/states.rs b/vendor/xml5ever-0.40.0/src/tokenizer/states.rs deleted file mode 100644 index 05dc9903fe..0000000000 --- a/vendor/xml5ever-0.40.0/src/tokenizer/states.rs +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Tokenizer states. - -/// Specifies either the public or system identifier from a [Document Type Declaration] (DTD). -/// -/// [Document Type Declaration]: https://en.wikipedia.org/wiki/Document_type_declaration -#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)] -pub enum DoctypeKind { - /// The public identifier. - Public, - /// The system identifier. - System, -} - -/// Specifies the different states a XML tokenizer will assume during parsing. -#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)] -pub enum XmlState { - /// The initial state of the parser. - /// - /// It is equivalent to the [`Data`](https://html.spec.whatwg.org/#data-state) state of the html parser, - /// except null codepoints do not cause errors. - Data, - /// Indicates that the parser has found a `<` character and will try to parse a tag. - TagState, - /// Indicates that the parser has consumed the `/` of a closing tag, like ``. - EndTagState, - /// Indicates that the parser is currently parsing the name of a closing tag, like the `foo` of ``. - EndTagName, - /// Indicates that the parser has finished parsing the name of a closing tag and expects a `>` to follow. - EndTagNameAfter, - /// Indicates that the parser has started parsing a [processing instruction] (PI). - /// - /// This state is reached after the initial `?` character has been consumed. - /// - /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction - Pi, - /// Indicates that the parser is currently parsing the target of a [processing instruction]. - /// - /// For example, the target of `` is `xml-stylesheet`. - /// - /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction - PiTarget, - /// Indicates that the parser has finished parsing the target of a [processing instruction]. - /// - /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction - PiTargetAfter, - /// Indicates that the parser is currently parsing the data of a [processing instruction]. - /// - /// The "data" refers to everything between the target and the closing `?` character. - /// - /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction - PiData, - /// Indicates that the parser has parsed the closing `?` of a [processing instruction]. - /// - /// [processing instruction]: https://en.wikipedia.org/wiki/Processing_Instruction - PiAfter, - /// Indicates that the parser has parsed the initial `!` of a markup declaration. - /// - /// Examples of such declarations include `` or ``. - MarkupDecl, - /// Indicates that the parser has parsed the start of a comment (``). - CommentEnd, - /// Indicates that the parser has parsed a `-` character within a comment which may or may not - /// be the beginning of the comment end (`-->`). - CommentEndDash, - /// Indicates that the parser has parsed `--!` within a comment which may or may not be part of the - /// end of the comment. Comments in XML can be closed with `--!>`. - CommentEndBang, - /// Indicates that the parser has parsed the beginning of a CDATA section (``). - CdataBracket, - /// Indicates that the parser has parsed two `]` characters within a CDATA section, which may be part of - /// the end of the section (`]]>`). - CdataEnd, - /// Indicates that the parser is currently parsing the name of a tag, such as `foo` in ``. - TagName, - /// Indicates that the parser has parsed the `/` of a self-closing tag, such as ``. - TagEmpty, - /// Indicates that the parser has finished parsing the name of a tag and is now expecting either attributes or - /// a `>`. - TagAttrNameBefore, - /// Indicates that the parser is currently parsing the name of an attribute within a tag, such as - /// `bar` in ``. - TagAttrName, - /// Indicates that the parser has finished parsing the name of an attribute. - TagAttrNameAfter, - /// Indicates that the parser is about to parse the value of an attribute. - TagAttrValueBefore, - /// Indicates that the parser is currently parsing the value of an attribute, such as `baz` in - /// ``. - /// - /// Includes information about how the value is quoted, because the quotes before and after the attribute - /// value need to match. - TagAttrValue(AttrValueKind), - /// Indicates that the parser has parsed the beginning of a document type definition (``. - DoctypeName, - /// Indicates that the parser has finished parsing the name of the document type definition and now optionally - /// expects either a public or a system identifier. - AfterDoctypeName, - /// Indicates that the parser has parsed a keyword for either a public or system identifier (`PUBLIC` or `SYSTEM`). - AfterDoctypeKeyword(DoctypeKind), - /// Indicates that the parser is about to parse the value of a public or system identifier within - /// a document type definition, such as `foo` in - /// ``. - BeforeDoctypeIdentifier(DoctypeKind), - /// Indicates that the parser is currently parsing the value of a public or system identifier - /// that is surrounded by double quotes , such as `foo` in - /// ``. - DoctypeIdentifierDoubleQuoted(DoctypeKind), - /// Indicates that the parser is currently parsing the value of a public or system identifier - /// that is surrounded by single quotes , such as `foo` in - /// ``. - DoctypeIdentifierSingleQuoted(DoctypeKind), - /// Indicates that the parser has finished parsing either a public or system identifier within a - /// document type definition. - AfterDoctypeIdentifier(DoctypeKind), - /// Indicates that the parser has finished parsing a public identifier and now expects - /// a system identifier. - BetweenDoctypePublicAndSystemIdentifiers, - /// Indicates that the parser is currently parsing an ill-formed document type defintion, such as - /// ``. - BogusDoctype, - /// Indicates that the parser is currently parsing an ill-formed comment, such as - /// ``. - BogusComment, -} - -/// Specifies how an attribute value is quoted, if at all. -#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)] -pub enum AttrValueKind { - /// A attribute value that is not surrounded by quotes, like `bar` in `foo=bar`. - Unquoted, - /// A attribute value that is not surrounded by quotes, like `bar` in `foo='bar'`. - SingleQuoted, - /// A attribute value that is not surrounded by quotes, like `bar` in `foo="bar"`. - DoubleQuoted, -} diff --git a/vendor/xml5ever-0.40.0/src/tree_builder/mod.rs b/vendor/xml5ever-0.40.0/src/tree_builder/mod.rs deleted file mode 100644 index 9dca11bd40..0000000000 --- a/vendor/xml5ever-0.40.0/src/tree_builder/mod.rs +++ /dev/null @@ -1,800 +0,0 @@ -// Copyright 2014-2017 The html5ever Project Developers. See the -// COPYRIGHT file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -mod types; - -use log::{debug, warn}; -use markup5ever::{local_name, namespace_prefix, ns}; -use std::borrow::Cow; -use std::borrow::Cow::Borrowed; -use std::cell::{Cell, Ref, RefCell}; -use std::collections::btree_map::Iter; -use std::collections::{BTreeMap, HashSet, VecDeque}; -use std::fmt::{Debug, Error, Formatter}; -use std::mem; - -pub use self::interface::{ElemName, NodeOrText, Tracer, TreeSink}; -use self::types::*; -use crate::interface::{self, create_element, AppendNode, Attribute, QualName}; -use crate::interface::{AppendText, ExpandedName}; -use crate::tokenizer::{self, EndTag, ProcessResult, StartTag, Tag, TokenSink}; -use crate::tokenizer::{Doctype, EmptyTag, Pi, ShortTag}; -use crate::{LocalName, Namespace, Prefix}; - -use crate::tendril::{StrTendril, Tendril}; - -/// XML-specific tree sink operations that are not part of markup5ever's -/// HTML-oriented [`TreeSink`] interface. -pub trait XmlTreeSink: TreeSink { - /// Create a CDATA section node. - fn create_cdata(&self, text: StrTendril) -> Self::Handle; -} - -static XML_URI: &str = "http://www.w3.org/XML/1998/namespace"; -static XMLNS_URI: &str = "http://www.w3.org/2000/xmlns/"; - -type InsResult = Result<(), Cow<'static, str>>; - -#[derive(Debug)] -struct NamespaceMapStack(Vec); - -impl NamespaceMapStack { - fn new() -> NamespaceMapStack { - NamespaceMapStack(vec![NamespaceMap::default()]) - } - - fn push(&mut self, map: NamespaceMap) { - self.0.push(map); - } - - fn pop(&mut self) { - self.0.pop(); - } -} - -pub(crate) struct NamespaceMap { - // Map that maps prefixes to URI. - // - // Key denotes namespace prefix, and value denotes - // URI it maps to. - // - // If value of value is None, that means the namespace - // denoted by key has been undeclared. - scope: BTreeMap, Option>, -} - -impl Debug for NamespaceMap { - fn fmt(&self, f: &mut Formatter) -> Result<(), Error> { - write!(f, "\nNamespaceMap[")?; - for (key, value) in &self.scope { - writeln!(f, " {key:?} : {value:?}")?; - } - write!(f, "]") - } -} - -impl NamespaceMap { - // Returns an empty namespace. - pub(crate) fn empty() -> NamespaceMap { - NamespaceMap { - scope: BTreeMap::new(), - } - } - - fn default() -> NamespaceMap { - NamespaceMap { - scope: { - let mut map = BTreeMap::new(); - map.insert(None, None); - map.insert(Some(namespace_prefix!("xml")), Some(ns!(xml))); - map.insert(Some(namespace_prefix!("xmlns")), Some(ns!(xmlns))); - map - }, - } - } - - pub(crate) fn get(&self, prefix: &Option) -> Option<&Option> { - self.scope.get(prefix) - } - - pub(crate) fn get_scope_iter(&self) -> Iter<'_, Option, Option> { - self.scope.iter() - } - - pub(crate) fn insert(&mut self, name: &QualName) { - let prefix = name.prefix.as_ref().cloned(); - let namespace = Some(Namespace::from(&*name.ns)); - self.scope.insert(prefix, namespace); - } - - fn insert_ns(&mut self, attr: &Attribute) -> InsResult { - if &*attr.value == XMLNS_URI { - return Err(Borrowed("Can't declare XMLNS URI")); - }; - - let opt_uri = if attr.value.is_empty() { - None - } else { - Some(Namespace::from(&*attr.value)) - }; - - let result = match (&attr.name.prefix, &*attr.name.local) { - (&Some(namespace_prefix!("xmlns")), "xml") => { - if &*attr.value != XML_URI { - Err(Borrowed("XML namespace can't be redeclared")) - } else { - Ok(()) - } - }, - - (&Some(namespace_prefix!("xmlns")), "xmlns") => { - Err(Borrowed("XMLNS namespaces can't be changed")) - }, - - (&Some(namespace_prefix!("xmlns")), _) | (&None, "xmlns") => { - // We can have two cases of properly defined xmlns - // First with default namespace e.g. - // - // - let ns_prefix = if &*attr.name.local == "xmlns" { - None - - // Second is with named namespace e.g. - // - // - } else { - Some(Prefix::from(&*attr.name.local)) - }; - - if opt_uri.is_some() && self.scope.contains_key(&ns_prefix) { - Err(Borrowed("Namespace already defined")) - } else { - self.scope.insert(ns_prefix, opt_uri); - Ok(()) - } - }, - - (_, _) => Err(Borrowed("Invalid namespace declaration.")), - }; - result - } -} - -/// Tree builder options, with an impl for Default. -#[derive(Copy, Clone, Default)] -pub struct XmlTreeBuilderOpts {} - -/// The XML tree builder. -pub struct XmlTreeBuilder { - /// Configuration options for XmlTreeBuilder - _opts: XmlTreeBuilderOpts, - - /// Consumer of tree modifications. - pub sink: Sink, - - /// The document node, which is created by the sink. - doc_handle: Handle, - - /// Stack of open elements, most recently added at end. - open_elems: RefCell>, - - /// Current element pointer. - curr_elem: RefCell>, - - /// Stack of namespace identifiers and namespaces. - namespace_stack: RefCell, - - /// Current namespace identifier - current_namespace: RefCell, - - /// Current tree builder phase. - phase: Cell, -} -impl XmlTreeBuilder -where - Handle: Clone, - Sink: XmlTreeSink, -{ - /// Create a new tree builder which sends tree modifications to a particular `TreeSink`. - /// - /// The tree builder is also a `TokenSink`. - pub fn new(sink: Sink, opts: XmlTreeBuilderOpts) -> XmlTreeBuilder { - let doc_handle = sink.get_document(); - XmlTreeBuilder { - _opts: opts, - sink, - doc_handle, - open_elems: RefCell::new(vec![]), - curr_elem: RefCell::new(None), - namespace_stack: RefCell::new(NamespaceMapStack::new()), - current_namespace: RefCell::new(NamespaceMap::empty()), - phase: Cell::new(XmlPhase::Start), - } - } - - /// Call the `Tracer`'s `trace_handle` method on every `Handle` in the tree builder's - /// internal state. This is intended to support garbage-collected DOMs. - pub fn trace_handles(&self, tracer: &dyn Tracer) { - tracer.trace_handle(&self.doc_handle); - for e in self.open_elems.borrow().iter() { - tracer.trace_handle(e); - } - if let Some(h) = self.curr_elem.borrow().as_ref() { - tracer.trace_handle(h); - } - } - - // Debug helper - #[cfg(not(for_c))] - #[allow(dead_code)] - fn dump_state(&self, label: String) { - debug!("dump_state on {label}"); - debug!(" open_elems:"); - for node in self.open_elems.borrow().iter() { - debug!(" {:?}", self.sink.elem_name(node)); - } - debug!(""); - } - - #[cfg(for_c)] - fn debug_step(&self, _mode: XmlPhase, _token: &Token) {} - - #[cfg(not(for_c))] - fn debug_step(&self, mode: XmlPhase, token: &Token) { - debug!( - "processing {:?} in insertion mode {:?}", - format!("{:?}", token), - mode - ); - } - - fn declare_ns(&self, attr: &mut Attribute) { - if let Err(msg) = self.current_namespace.borrow_mut().insert_ns(attr) { - self.sink.parse_error(msg); - } - // Namespace declarations are real DOM attributes. Keep their XMLNS - // namespace identity even when the declaration itself is invalid. - attr.name.ns = ns!(xmlns); - } - - fn find_uri(&self, prefix: &Option) -> Result, Cow<'static, str>> { - let mut uri = Err(Borrowed("No appropriate namespace found")); - - let current_namespace = self.current_namespace.borrow(); - for ns in self - .namespace_stack - .borrow() - .0 - .iter() - .chain(Some(&*current_namespace)) - .rev() - { - if let Some(el) = ns.get(prefix) { - uri = Ok(el.clone()); - break; - } - } - uri - } - - fn bind_qname(&self, name: &mut QualName) { - match self.find_uri(&name.prefix) { - Ok(uri) => { - let ns_uri = match uri { - Some(e) => e, - None => ns!(), - }; - name.ns = ns_uri; - }, - Err(msg) => { - self.sink.parse_error(msg); - }, - } - } - - // This method takes in name qualified name and binds it to the - // existing namespace context. - // - // Returns false if the attribute is a duplicate, returns true otherwise. - fn bind_attr_qname( - &self, - present_attrs: &mut HashSet<(Namespace, LocalName)>, - name: &mut QualName, - ) -> bool { - // Attributes don't have default namespace - let mut not_duplicate = true; - - if name.prefix.is_some() { - self.bind_qname(name); - not_duplicate = Self::check_duplicate_attr(present_attrs, name); - } - not_duplicate - } - - fn check_duplicate_attr( - present_attrs: &mut HashSet<(Namespace, LocalName)>, - name: &QualName, - ) -> bool { - let pair = (name.ns.clone(), name.local.clone()); - - if present_attrs.contains(&pair) { - return false; - } - present_attrs.insert(pair); - true - } - - fn process_namespaces(&self, tag: &mut Tag) { - // List of already present namespace local name attribute pairs. - let mut present_attrs: HashSet<(Namespace, LocalName)> = Default::default(); - - let mut new_attr = vec![]; - // First we extract all namespace declarations - for attr in tag.attrs.iter_mut().filter(|attr| { - attr.name.prefix == Some(namespace_prefix!("xmlns")) - || attr.name.local == local_name!("xmlns") - }) { - self.declare_ns(attr); - } - - // Bind ordinary attributes and retain namespace declarations in their - // original order. Tree sinks need the declarations as Attr nodes even - // though the namespace resolver has already consumed their values. - for attr in tag.attrs.iter_mut() { - let is_namespace_declaration = attr.name.ns == ns!(xmlns); - let keep = if is_namespace_declaration { - Self::check_duplicate_attr(&mut present_attrs, &attr.name) - } else { - self.bind_attr_qname(&mut present_attrs, &mut attr.name) - }; - if keep { - new_attr.push(attr.clone()); - } - } - tag.attrs = new_attr; - - // Then we bind the tags namespace. - self.bind_qname(&mut tag.name); - - // Finally, we dump current namespace if its unneeded. - let x = mem::replace( - &mut *self.current_namespace.borrow_mut(), - NamespaceMap::empty(), - ); - - // Only start tag doesn't dump current namespace. However,