mirror of
https://github.com/lexmount/moli.git
synced 2026-10-03 16:00:45 +00:00
Athul Nambiar
777ae9af36
`--obey-robots` parsed into `FetchConfig` and stopped there. Its only consumer was the network trace, which reported the flag's own value back out; no robots.txt was ever fetched or honored. The webfetch skill meanwhile told agents to enable it for crawl workloads, so a crawl could report that it obeyed robots.txt while ignoring it. Add `moli-robots`, a dependency-light RFC 9309 parser and matcher: - group parsing, including merged repeat groups and `User-agent` runs; - user-agent selection by specificity with `*` as the fallback group; - `Allow`/`Disallow` with `*` and `$`, longest match winning and `Allow` taking ties; - percent-encoding normalization so a pattern and a request target compare in one space; - the RFC 9309 section 2.3.1 response rules, where 4xx publishes no rules and 5xx or a transport failure disallows the origin. Wire it into `moli fetch`. The check runs before the browser starts, so a refused URL costs one robots.txt request and the target is never contacted. Scope is the top-level navigation. Subresource and protocol-server navigations run through NavigationEngine, whose redirect, service-worker, and CDP paths need a design decision of their own; `moli serve --obey-robots` now warns that it is not enforced instead of appearing to work. Fixes #79
118 lines
3.2 KiB
TOML
118 lines
3.2 KiB
TOML
[workspace]
|
|
resolver = "3"
|
|
members = [
|
|
"moli",
|
|
"moli-action-window",
|
|
"moli-bounded-buffer",
|
|
"moli-bounded-json",
|
|
"moli-broadcast-channel",
|
|
"moli-browser-profile",
|
|
"moli-canvas",
|
|
"moli-charset-parser",
|
|
"moli-cookie-cache",
|
|
"moli-protocol-cdp",
|
|
"moli-cookie-store",
|
|
"moli-css-parse",
|
|
"moli-crypto",
|
|
"moli-curl",
|
|
"moli-dns-resolver",
|
|
"moli-protocol",
|
|
"moli-encoding",
|
|
"moli-core",
|
|
"moli-import-map",
|
|
"moli-image",
|
|
"moli-indexeddb",
|
|
"moli-local-executor",
|
|
"moli-layout",
|
|
"moli-message-port",
|
|
"moli-multipart",
|
|
"moli-xpath",
|
|
"moli-module-script-tree",
|
|
"moli-module-syntax",
|
|
"moli-owner-queue",
|
|
"moli-opfs",
|
|
"moli-dom",
|
|
"moli-parser",
|
|
"moli-paint",
|
|
"moli-page-types",
|
|
"moli-process-signal",
|
|
"moli-renderer-v8",
|
|
"moli-robots",
|
|
"moli-protocol-server",
|
|
"moli-script",
|
|
"moli-selector",
|
|
"moli-site",
|
|
"moli-shared-worker",
|
|
"moli-storage-service",
|
|
"moli-storage-key",
|
|
"moli-streams",
|
|
"moli-structured-clone",
|
|
"moli-stylesheet-blocking",
|
|
"moli-svg",
|
|
"moli-test-support",
|
|
"moli-url",
|
|
"moli-url-policy",
|
|
"moli-v8-init",
|
|
"moli-v8-platform",
|
|
"moli-v8-test-util",
|
|
"moli-v8-util",
|
|
"moli-web-mime",
|
|
"moli-web-bot-auth",
|
|
"moli-web-errors",
|
|
"moli-protocol-webdriver-bidi",
|
|
"moli-protocol-webdriver-classic",
|
|
"moli-webcrypto",
|
|
"moli-webapi-declare",
|
|
"moli-webapi-declare-derive",
|
|
"moli-webidl",
|
|
"moli-webidl-callback",
|
|
"moli-webidl-derive",
|
|
"moli-cookie-jar",
|
|
"moli-websocket",
|
|
"moli-fetch",
|
|
"moli-file-api",
|
|
"moli-geometry",
|
|
"moli-header-field",
|
|
"moli-html-input-temporal",
|
|
"moli-time",
|
|
"moli-trace",
|
|
"moli-http-cache",
|
|
"moli-wpt-compat"
|
|
]
|
|
exclude = ["vendor/xml5ever-0.39.0"]
|
|
|
|
[workspace.package]
|
|
license = "MIT OR Apache-2.0"
|
|
|
|
[patch.crates-io]
|
|
xml5ever = { path = "vendor/xml5ever-0.39.0" }
|
|
curl = { git = "https://github.com/ldm0/curl-rust", branch = "lightmount" }
|
|
curl-sys = { git = "https://github.com/ldm0/curl-rust", branch = "lightmount" }
|
|
cookie = { git = "https://github.com/ldm0/cookie-rs", branch = "priority" }
|
|
v8 = { path = "vendor/v8-146.9.0" }
|
|
selectors = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
|
|
stylo = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
|
|
stylo_atoms = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
|
|
stylo_dom = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
|
|
stylo_static_prefs = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
|
|
stylo_traits = { git = "https://github.com/ldm0/stylo", rev = "74cd014af14d84db1fc8faed85086895012fc687" }
|
|
taffy = { git = "https://github.com/ldm0/taffy", rev = "28923cb1d9164245e10566ef8156fa5cfa9633bb" }
|
|
|
|
[workspace.lints.clippy]
|
|
disallowed_methods = "deny"
|
|
disallowed_types = "deny"
|
|
enum_variant_names = "allow"
|
|
result_unit_err = "allow"
|
|
too_many_arguments = "allow"
|
|
type_complexity = "allow"
|
|
|
|
[profile.dev]
|
|
panic = "abort"
|
|
|
|
[profile.release]
|
|
panic = "abort"
|
|
strip = false
|
|
|
|
[profile.test]
|
|
debug = 1
|