From ce7fa9cd172ef395ee895e57ad85d4fbedd185ed Mon Sep 17 00:00:00 2001 From: ldm0 Date: Sun, 16 Aug 2026 03:24:13 +0800 Subject: [PATCH] feat: implement Web Bot Auth signing --- Cargo.lock | 12 + Cargo.toml | 1 + moli-crypto/src/ed25519.rs | 98 ++++++ moli-crypto/src/lib.rs | 2 + moli-fetch/Cargo.toml | 1 + moli-fetch/src/blocking/cache/key.rs | 1 + moli-fetch/src/blocking/mod.rs | 28 +- moli-fetch/src/config.rs | 12 + moli-fetch/src/lib.rs | 1 + moli-fetch/src/runtime.rs | 187 ++++++----- moli-fetch/src/tests/mod.rs | 211 ++++++++++++- moli-fetch/src/tests/support.rs | 19 ++ moli-web-bot-auth/Cargo.toml | 18 ++ moli-web-bot-auth/src/key.rs | 133 ++++++++ moli-web-bot-auth/src/lib.rs | 12 + moli-web-bot-auth/src/profile.rs | 15 + moli-web-bot-auth/src/signer.rs | 291 ++++++++++++++++++ moli-web-bot-auth/src/test_support.rs | 24 ++ moli-web-bot-auth/src/wire.rs | 279 +++++++++++++++++ moli/src/cli.rs | 33 ++ moli/src/config.rs | 37 ++- moli/tests/cli.rs | 171 +++++++++- .../moli-webfetch/references/fetch-recipes.md | 6 + 23 files changed, 1461 insertions(+), 131 deletions(-) create mode 100644 moli-crypto/src/ed25519.rs create mode 100644 moli-web-bot-auth/Cargo.toml create mode 100644 moli-web-bot-auth/src/key.rs create mode 100644 moli-web-bot-auth/src/lib.rs create mode 100644 moli-web-bot-auth/src/profile.rs create mode 100644 moli-web-bot-auth/src/signer.rs create mode 100644 moli-web-bot-auth/src/test_support.rs create mode 100644 moli-web-bot-auth/src/wire.rs diff --git a/Cargo.lock b/Cargo.lock index 376d8fd34..972d6cb58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2221,6 +2221,7 @@ dependencies = [ "moli-trace", "moli-url", "moli-url-policy", + "moli-web-bot-auth", "moli-web-mime", "openssl-sys", "parking_lot", @@ -2825,6 +2826,17 @@ dependencies = [ "v8", ] +[[package]] +name = "moli-web-bot-auth" +version = "0.1.0" +dependencies = [ + "anyhow", + "aws-lc-rs", + "base64 0.22.1", + "moli-crypto", + "url", +] + [[package]] name = "moli-web-errors" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f77ec5928..9e3855530 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ members = [ "moli-v8-test-util", "moli-v8-util", "moli-web-mime", + "moli-web-bot-auth", "moli-web-errors", "moli-protocol-webdriver-bidi", "moli-protocol-webdriver-classic", diff --git a/moli-crypto/src/ed25519.rs b/moli-crypto/src/ed25519.rs new file mode 100644 index 000000000..0736c0de9 --- /dev/null +++ b/moli-crypto/src/ed25519.rs @@ -0,0 +1,98 @@ +use std::fmt; + +use aws_lc_rs::signature::{Ed25519KeyPair, KeyPair}; + +const ED25519_PUBLIC_KEY_LENGTH: usize = 32; +const ED25519_SIGNATURE_LENGTH: usize = 64; + +pub struct Ed25519SigningKey { + key_pair: Ed25519KeyPair, + public_key: [u8; ED25519_PUBLIC_KEY_LENGTH], +} + +impl Ed25519SigningKey { + pub fn from_pkcs8(der: &[u8]) -> Result { + let key_pair = Ed25519KeyPair::from_pkcs8(der).map_err(|_| Ed25519Error)?; + let public_key = key_pair + .public_key() + .as_ref() + .try_into() + .map_err(|_| Ed25519Error)?; + Ok(Self { + key_pair, + public_key, + }) + } + + pub fn public_key(&self) -> &[u8; ED25519_PUBLIC_KEY_LENGTH] { + &self.public_key + } + + pub fn sign(&self, message: &[u8]) -> Result<[u8; ED25519_SIGNATURE_LENGTH], Ed25519Error> { + self.key_pair + .try_sign(message) + .map_err(|_| Ed25519Error)? + .as_ref() + .try_into() + .map_err(|_| Ed25519Error) + } +} + +impl fmt::Debug for Ed25519SigningKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Ed25519SigningKey") + .field("public_key", &self.public_key) + .finish_non_exhaustive() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Ed25519Error; + +impl fmt::Display for Ed25519Error { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("invalid Ed25519 key or signing operation failed") + } +} + +impl std::error::Error for Ed25519Error {} + +#[cfg(test)] +mod tests { + use super::*; + + const RFC_9421_PRIVATE_KEY_DER: &[u8] = &[ + 0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04, + 0x20, 0x9f, 0x83, 0x62, 0xf8, 0x7a, 0x48, 0x4a, 0x95, 0x4e, 0x6e, 0x74, 0x0c, 0x5b, 0x4c, + 0x0e, 0x84, 0x22, 0x91, 0x39, 0xa2, 0x0a, 0xa8, 0xab, 0x56, 0xff, 0x66, 0x58, 0x6f, 0x6a, + 0x7d, 0x29, 0xc5, + ]; + + #[test] + fn imports_pkcs8_and_signs_deterministically() { + let key = Ed25519SigningKey::from_pkcs8(RFC_9421_PRIVATE_KEY_DER).unwrap(); + assert_eq!( + key.public_key(), + &[ + 0x26, 0xb4, 0x0b, 0x8f, 0x93, 0xff, 0xf3, 0xd8, 0x97, 0x11, 0x2f, 0x7e, 0xbc, 0x58, + 0x2b, 0x23, 0x2d, 0xbd, 0x72, 0x51, 0x7d, 0x08, 0x2f, 0xe8, 0x3c, 0xfb, 0x30, 0xdd, + 0xce, 0x43, 0xd1, 0xbb, + ] + ); + + let first = key.sign(b"web bot auth").unwrap(); + let second = key.sign(b"web bot auth").unwrap(); + assert_eq!(first.len(), ED25519_SIGNATURE_LENGTH); + assert_eq!(first, second); + assert_ne!(first, key.sign(b"different message").unwrap()); + } + + #[test] + fn rejects_non_ed25519_pkcs8() { + assert_eq!( + Ed25519SigningKey::from_pkcs8(b"not a key").unwrap_err(), + Ed25519Error + ); + } +} diff --git a/moli-crypto/src/lib.rs b/moli-crypto/src/lib.rs index 891b8fae3..c52fdb80c 100644 --- a/moli-crypto/src/lib.rs +++ b/moli-crypto/src/lib.rs @@ -5,7 +5,9 @@ //! owner (`moli-webcrypto` or the renderer subsystem using the primitive). mod digest; +mod ed25519; mod random; pub use digest::{DigestAlgorithm, Sha256Context, sha1_digest, sha256_digest, sha256_hex}; +pub use ed25519::{Ed25519Error, Ed25519SigningKey}; pub use random::fill_secure_random; diff --git a/moli-fetch/Cargo.toml b/moli-fetch/Cargo.toml index b4f57e7f7..87542af5c 100644 --- a/moli-fetch/Cargo.toml +++ b/moli-fetch/Cargo.toml @@ -23,6 +23,7 @@ moli-http-cache = { path = "../moli-http-cache" } moli-trace = { path = "../moli-trace" } moli-url = { path = "../moli-url" } moli-url-policy = { path = "../moli-url-policy" } +moli-web-bot-auth = { path = "../moli-web-bot-auth" } moli-web-mime = { path = "../moli-web-mime" } parking_lot = "0.12" strum = { version = "0.28.0", features = ["derive"] } diff --git a/moli-fetch/src/blocking/cache/key.rs b/moli-fetch/src/blocking/cache/key.rs index a4801c2a5..00c847732 100644 --- a/moli-fetch/src/blocking/cache/key.rs +++ b/moli-fetch/src/blocking/cache/key.rs @@ -19,6 +19,7 @@ pub(super) fn cache_store_and_key_for_request( || !request.method.eq_ignore_ascii_case("GET") || request.body.is_some() || request.auth().is_some() + || config.web_bot_auth().is_some() || !config.default_request_headers().is_empty() || !subresource_validation_allows_http_cache(request) || cookie_header.is_some() diff --git a/moli-fetch/src/blocking/mod.rs b/moli-fetch/src/blocking/mod.rs index ffd176d00..42a5748d7 100644 --- a/moli-fetch/src/blocking/mod.rs +++ b/moli-fetch/src/blocking/mod.rs @@ -244,21 +244,12 @@ pub(crate) fn outgoing_request_headers_for_url( outgoing } -pub(crate) fn network_request_extra_info_for_url( +pub(crate) fn network_request_extra_info_from_headers( config: &FetchConfig, - request: &Request, - request_url: &Url, - redirect_chain: &[RedirectInfo], - cookie_header: Option<&str>, + outgoing_headers: &[(String, String)], cookie_report: Option<&StoredCookieQueryReport>, ) -> NetworkRequestExtraInfo { - let mut headers = outgoing_request_headers_for_url( - config, - request, - request_url, - redirect_chain, - cookie_header, - ); + let mut headers = outgoing_headers.to_vec(); append_header_if_missing(&mut headers, "User-Agent", config.user_agent().to_owned()); NetworkRequestExtraInfo { headers, @@ -599,7 +590,7 @@ pub(crate) fn configure_easy( cookie_header: Option<&str>, http_version: RequestHttpVersion, validation_headers: Option>, -) -> Result<()> { +) -> Result> { ensure_http_network_transport_url(request_url)?; enforce_request_target_policy(config, request_url)?; configure_curl_http_protocol_allowlist(easy)?; @@ -712,17 +703,22 @@ pub(crate) fn configure_easy( } let mut headers = List::new(); - let outgoing_headers = outgoing_request_headers_for_url( + let mut outgoing_headers = outgoing_request_headers_for_url( config, request, request_url, redirect_chain, cookie_header, ); + if let Some(web_bot_auth) = config.web_bot_auth() { + web_bot_auth + .append_request_headers(&mut outgoing_headers, &request.method, request_url) + .with_context(|| anyhow!("failed to sign web bot auth request for {request_url}"))?; + } let mut has_headers = false; let mut has_content_type_header = false; - for (name, value) in outgoing_headers { + for (name, value) in &outgoing_headers { has_content_type_header |= name.eq_ignore_ascii_case("content-type"); let header_line = if value.is_empty() { format!("{name}:") @@ -797,7 +793,7 @@ pub(crate) fn configure_easy( } } - Ok(()) + Ok(outgoing_headers) } pub(crate) fn configure_openssl_tls_context( diff --git a/moli-fetch/src/config.rs b/moli-fetch/src/config.rs index 179140ef4..523568466 100644 --- a/moli-fetch/src/config.rs +++ b/moli-fetch/src/config.rs @@ -3,6 +3,8 @@ use std::num::NonZeroU32; use cidr::AnyIpCidr; use moli_browser_profile::{BrowserIdentityProfile, DEFAULT_ACCEPT_LANGUAGE}; +use crate::WebBotAuthSigner; + #[derive(Debug, Clone, PartialEq)] pub struct FetchConfig { browser_identity: BrowserIdentityProfile, @@ -34,6 +36,7 @@ pub struct FetchConfig { block_private_networks: bool, block_cidrs: Vec, tls_verify_host: bool, + web_bot_auth: Option, } impl FetchConfig { @@ -144,6 +147,14 @@ impl FetchConfig { self.tls_verify_host = tls_verify_host; } + pub fn web_bot_auth(&self) -> Option<&WebBotAuthSigner> { + self.web_bot_auth.as_ref() + } + + pub fn set_web_bot_auth(&mut self, web_bot_auth: Option) { + self.web_bot_auth = web_bot_auth; + } + pub fn set_http_proxy(&mut self, http_proxy: Option) { self.http_proxy = http_proxy; } @@ -274,6 +285,7 @@ impl Default for FetchConfig { block_private_networks: false, block_cidrs: Vec::new(), tls_verify_host: true, + web_bot_auth: None, } } } diff --git a/moli-fetch/src/lib.rs b/moli-fetch/src/lib.rs index 34247e14b..95b055bfe 100644 --- a/moli-fetch/src/lib.rs +++ b/moli-fetch/src/lib.rs @@ -45,6 +45,7 @@ pub use headers::{ is_forbidden_response_header_name, is_no_cors_safelisted_request_header, }; pub use moli_cookie_jar::SharedBrowserCookieStore as SharedCookieStore; +pub use moli_web_bot_auth::{WebBotAuthProfile, WebBotAuthSigner}; pub use network_fetch_result::{ NetworkExchangeObservation, NetworkFetchFailure, NetworkFetchFailureRequestContext, NetworkFetchResult, NetworkObservationJournal, NetworkRequestObservation, diff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rs index de50acd9d..771312bf3 100644 --- a/moli-fetch/src/runtime.rs +++ b/moli-fetch/src/runtime.rs @@ -45,7 +45,7 @@ use crate::{ cookie_access_report_for_request, cookie_header_from_report, finish_streaming_cached_response, load_cached_streaming_response_lookup, log_request_completion, merge_cached_not_modified_streaming_response_lookup, - network_request_extra_info_for_url, next_followed_redirect_url_from_parts, + network_request_extra_info_from_headers, next_followed_redirect_url_from_parts, remove_cached_response, response_headers_forbid_cache_storage, store_response_cookies, transfer_metrics_from_easy, validation_headers_for_cached_streaming_response_lookup, }, @@ -863,22 +863,6 @@ impl RuntimeOwner { &job.request, &job.current_url, ); - let request_extra_info = job.request.is_top_level_navigation_request().then(|| { - network_request_extra_info_for_url( - &self.config, - &prepared_request.request, - &job.current_url, - &job.redirect_chain, - cookie_header.as_deref(), - request_cookie_report.as_ref(), - ) - }); - attach_next_request_extra_info( - &mut job.redirect_chain, - request_cookie_report.clone(), - request_extra_info.as_ref(), - ); - let mut easy = Easy2::new(FetchTransferHandler::new_buffered(ResponseCollector::new( Some(job.cancel_handle.clone()), ))); @@ -894,7 +878,7 @@ impl RuntimeOwner { ) { return Err((job.response_tx, error)); } - if let Err(error) = configure_easy( + let outgoing_headers = match configure_easy( &mut easy, &self.config, &prepared_request.request, @@ -909,8 +893,21 @@ impl RuntimeOwner { ) .with_context(|| anyhow!("failed to configure curl request for {}", job.current_url)) { - return Err((job.response_tx, error)); - } + Ok(headers) => headers, + Err(error) => return Err((job.response_tx, error)), + }; + let request_extra_info = job.request.is_top_level_navigation_request().then(|| { + network_request_extra_info_from_headers( + &self.config, + &outgoing_headers, + request_cookie_report.as_ref(), + ) + }); + attach_next_request_extra_info( + &mut job.redirect_chain, + request_cookie_report.clone(), + request_extra_info.as_ref(), + ); let label = job.current_url.to_string(); let dns_resolution = curl_dns_resolution(&self.config, &job.current_url); @@ -995,16 +992,6 @@ impl RuntimeOwner { &job.request, &job.current_url, ); - let request_extra_info = job.request.is_top_level_navigation_request().then(|| { - network_request_extra_info_for_url( - &self.config, - &prepared_request.request, - &job.current_url, - &job.redirect_chain, - cookie_header.as_deref(), - request_cookie_report.as_ref(), - ) - }); match load_cached_streaming_response_lookup( &self.config, &prepared_request.request, @@ -1033,12 +1020,6 @@ impl RuntimeOwner { Err(error) => return Err((Box::new(job), None, error)), } - attach_next_request_extra_info( - &mut job.redirect_chain, - request_cookie_report.clone(), - request_extra_info.as_ref(), - ); - let mut easy = job.easy.take().unwrap_or_else(|| { Easy2::new(FetchTransferHandler::new_streaming( StreamingResponseCollector::new( @@ -1061,6 +1042,41 @@ impl RuntimeOwner { cookie_header.clone(), )); + if let Err(error) = configure_network_observation( + &mut easy, + &job.request, + request_cookie_report.as_ref(), + self.config.http_proxy().is_some() && job.current_url.scheme() == "https", + ) { + return Err((Box::new(job), Some(easy), error)); + } + let outgoing_headers = match configure_easy( + &mut easy, + &self.config, + &prepared_request.request, + &job.current_url, + &job.redirect_chain, + cookie_header.as_deref(), + job.http_version, + None, + ) + .with_context(|| anyhow!("failed to configure curl request for {}", job.current_url)) + { + Ok(headers) => headers, + Err(error) => return Err((Box::new(job), Some(easy), error)), + }; + let request_extra_info = job.request.is_top_level_navigation_request().then(|| { + network_request_extra_info_from_headers( + &self.config, + &outgoing_headers, + request_cookie_report.as_ref(), + ) + }); + attach_next_request_extra_info( + &mut job.redirect_chain, + request_cookie_report.clone(), + request_extra_info.as_ref(), + ); let collector = easy .get_mut() .streaming_mut() @@ -1076,28 +1092,6 @@ impl RuntimeOwner { cache_plan, ); collector.set_client_hint_response_policy(prepared_request.response_policy); - if let Err(error) = configure_network_observation( - &mut easy, - &job.request, - request_cookie_report.as_ref(), - self.config.http_proxy().is_some() && job.current_url.scheme() == "https", - ) { - return Err((Box::new(job), Some(easy), error)); - } - if let Err(error) = configure_easy( - &mut easy, - &self.config, - &prepared_request.request, - &job.current_url, - &job.redirect_chain, - cookie_header.as_deref(), - job.http_version, - None, - ) - .with_context(|| anyhow!("failed to configure curl request for {}", job.current_url)) - { - return Err((Box::new(job), Some(easy), error)); - } let label = job.current_url.to_string(); let dns_resolution = curl_dns_resolution(&self.config, &job.current_url); @@ -1189,16 +1183,6 @@ impl RuntimeOwner { &job.request, &job.current_url, ); - let request_extra_info = job.request.is_top_level_navigation_request().then(|| { - network_request_extra_info_for_url( - &self.config, - &prepared_request.request, - &job.current_url, - &job.redirect_chain, - cookie_header.as_deref(), - request_cookie_report.as_ref(), - ) - }); let mut stale_cached_lookup = None; match load_cached_streaming_response_lookup( &self.config, @@ -1231,12 +1215,6 @@ impl RuntimeOwner { Err(error) => return Err((Box::new(job), None, error)), } - attach_next_request_extra_info( - &mut job.redirect_chain, - request_cookie_report.clone(), - request_extra_info.as_ref(), - ); - let mut easy = job.easy.take().unwrap_or_else(|| { Easy2::new(FetchTransferHandler::new_raw_streaming( RawStreamingResponseCollector::new( @@ -1258,6 +1236,43 @@ impl RuntimeOwner { job.current_url.clone(), cookie_header.clone(), )); + if let Err(error) = configure_network_observation( + &mut easy, + &job.request, + request_cookie_report.as_ref(), + self.config.http_proxy().is_some() && job.current_url.scheme() == "https", + ) { + return Err((Box::new(job), Some(easy), error)); + } + let outgoing_headers = match configure_easy( + &mut easy, + &self.config, + &prepared_request.request, + &job.current_url, + &job.redirect_chain, + cookie_header.as_deref(), + job.http_version, + stale_cached_lookup + .as_ref() + .map(validation_headers_for_cached_streaming_response_lookup), + ) + .with_context(|| anyhow!("failed to configure curl request for {}", job.current_url)) + { + Ok(headers) => headers, + Err(error) => return Err((Box::new(job), Some(easy), error)), + }; + let request_extra_info = job.request.is_top_level_navigation_request().then(|| { + network_request_extra_info_from_headers( + &self.config, + &outgoing_headers, + request_cookie_report.as_ref(), + ) + }); + attach_next_request_extra_info( + &mut job.redirect_chain, + request_cookie_report.clone(), + request_extra_info.as_ref(), + ); let collector = easy .get_mut() .raw_streaming_mut() @@ -1274,30 +1289,6 @@ impl RuntimeOwner { stale_cached_lookup.is_some(), ); collector.set_client_hint_response_policy(prepared_request.response_policy); - if let Err(error) = configure_network_observation( - &mut easy, - &job.request, - request_cookie_report.as_ref(), - self.config.http_proxy().is_some() && job.current_url.scheme() == "https", - ) { - return Err((Box::new(job), Some(easy), error)); - } - if let Err(error) = configure_easy( - &mut easy, - &self.config, - &prepared_request.request, - &job.current_url, - &job.redirect_chain, - cookie_header.as_deref(), - job.http_version, - stale_cached_lookup - .as_ref() - .map(validation_headers_for_cached_streaming_response_lookup), - ) - .with_context(|| anyhow!("failed to configure curl request for {}", job.current_url)) - { - return Err((Box::new(job), Some(easy), error)); - } let label = job.current_url.to_string(); let dns_resolution = curl_dns_resolution(&self.config, &job.current_url); diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rs index 7e8512765..e6d7b41e1 100644 --- a/moli-fetch/src/tests/mod.rs +++ b/moli-fetch/src/tests/mod.rs @@ -8,6 +8,7 @@ use moli_browser_profile::DEFAULT_ACCEPT_LANGUAGE; use moli_cookie_jar::{NetworkCookieRequestContext, new_shared_browser_cookie_store}; use moli_http_cache::{HttpCacheEntryMetadata, HttpCacheStore}; use std::{ + collections::BTreeSet, fs, io::Read, num::NonZeroU32, @@ -28,7 +29,8 @@ use crate::{ RequestAuthScheme, RequestAuthTarget, RequestCacheMode, RequestCredentialsMode, RequestMode, RequestRedirectMode, RequestResourceType, Response, ResponseBody, ResponseHead, ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, StreamingResponseCollector, - SubresourceRequestMetadata, http_cache_stats, runtime::FetchRuntimeOwner, + SubresourceRequestMetadata, WebBotAuthProfile, WebBotAuthSigner, http_cache_stats, + runtime::FetchRuntimeOwner, }; use self::support::{ @@ -39,6 +41,19 @@ use self::support::{ const ENV_PROXY_CHILD_TEST: &str = "MOLI_FETCH_ENV_PROXY_CHILD"; const ENV_PROXY_URL: &str = "MOLI_FETCH_ENV_PROXY_URL"; const TEST_HIGH_ENTROPY_CLIENT_HINTS: &str = "Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Platform-Version, Sec-CH-UA-Model, Sec-CH-UA-WoW64"; +const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\ +MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\ +-----END PRIVATE KEY-----\n"; + +fn test_web_bot_auth_signer() -> WebBotAuthSigner { + WebBotAuthSigner::from_pem( + RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), + "bot.example", + Some("poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U"), + WebBotAuthProfile::Cloudflare, + ) + .unwrap() +} fn sample_response_head() -> ResponseHead { ResponseHead { @@ -1191,6 +1206,172 @@ async fn fetch_raw_stream_treats_https11_switching_protocols_as_final_response() Ok(()) } +#[test] +fn web_bot_auth_resigns_restarts_redirects_and_subresources() -> Result<()> { + let server = ScriptedHttps11Server::spawn(vec![ + ScriptedResponse::status(403, "Challenge") + .with_header("Accept-CH", "Sec-CH-UA-Arch") + .with_header("Critical-CH", "Sec-CH-UA-Arch"), + ScriptedResponse::status(302, "Found").with_header("Location", "/final"), + ScriptedResponse::ok("final"), + ScriptedResponse::ok("asset"), + ]); + let mut config = FetchConfig::default(); + config.set_tls_verify_host(false); + config.set_web_bot_auth(Some(test_web_bot_auth_signer())); + let client = FetchClient::new(&config, new_shared_browser_cookie_store()); + + let response = fetch_response_for_test(&client, Request::get(&server.url_path("/start"))?)?; + assert_eq!(response.body_text(), "final"); + assert_eq!(response.redirect_chain.len(), 2); + assert_eq!(response.redirect_chain[0].status, 307); + assert_eq!(response.redirect_chain[1].status, 302); + + let final_url = Url::parse(&server.url_path("/final"))?; + let subresource = Request::new( + "POST", + &server.url_path("/asset"), + Some("probe".to_owned()), + Vec::new(), + )? + .with_browser_request_metadata(BrowserRequestMetadata::Fetch) + .with_initiator_url(&final_url); + assert_eq!( + fetch_response_for_test(&client, subresource)?.body_text(), + "asset" + ); + + assert_eq!( + server.requests(), + [ + "/start".to_owned(), + "/start".to_owned(), + "/final".to_owned(), + "/asset".to_owned(), + ] + ); + let request_heads = server.request_heads(); + assert_eq!(request_heads.len(), 4); + for request_head in &request_heads { + assert_eq!( + request_head_header_value(request_head, "Signature-Agent"), + Some("\"https://bot.example\"") + ); + let signature_input = request_head_header_value(request_head, "Signature-Input") + .expect("signed request should include Signature-Input"); + assert!( + signature_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")") + ); + assert!(request_head_header_value(request_head, "Signature").is_some()); + } + assert!(request_heads[3].starts_with("POST /asset HTTP/1.1\r\n")); + + let nonces = request_heads + .iter() + .map(|request_head| { + nonce_from_signature_input( + request_head_header_value(request_head, "Signature-Input").unwrap(), + ) + .to_owned() + }) + .collect::>(); + assert_eq!(nonces.len(), request_heads.len()); + + let restart = &response.redirect_chain[0]; + assert_request_extra_signature_matches_wire( + &restart + .response_extra_info + .as_ref() + .expect("restart response extra info") + .request_extra_info, + &request_heads[0], + ); + assert_request_extra_signature_matches_wire( + restart + .request_extra_info + .as_ref() + .expect("restarted request extra info"), + &request_heads[1], + ); + let redirect = &response.redirect_chain[1]; + assert_request_extra_signature_matches_wire( + &redirect + .response_extra_info + .as_ref() + .expect("redirect response extra info") + .request_extra_info, + &request_heads[1], + ); + assert_request_extra_signature_matches_wire( + redirect + .request_extra_info + .as_ref() + .expect("redirect target request extra info"), + &request_heads[2], + ); + assert_request_extra_signature_matches_wire( + response + .network_request_extra_info() + .expect("final request extra info"), + &request_heads[2], + ); + + server.shutdown(); + Ok(()) +} + +#[test] +fn web_bot_auth_is_not_sent_over_plain_http() -> Result<()> { + let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("plain")]); + let mut config = FetchConfig::default(); + config.set_web_bot_auth(Some(test_web_bot_auth_signer())); + + let response = fetch_with_config_for_test(&config, Request::get(&server.url())?)?; + assert_eq!(response.body_text(), "plain"); + let requests = server.requests(); + assert_eq!(requests.len(), 1); + for name in ["Signature-Agent", "Signature-Input", "Signature"] { + assert_eq!(request_head_header_value(&requests[0], name), None); + } + + server.shutdown(); + Ok(()) +} + +#[test] +fn web_bot_auth_bypasses_shared_http_cache() -> Result<()> { + let cache_dir = unique_test_cache_dir(); + let server = ScriptedHttps11Server::spawn(vec![ + ScriptedResponse::ok("first").with_header("Cache-Control", "max-age=60"), + ScriptedResponse::ok("second").with_header("Cache-Control", "max-age=60"), + ]); + let mut config = FetchConfig::default(); + config.set_tls_verify_host(false); + config.set_http_cache_dir(Some(cache_dir.display().to_string())); + config.set_web_bot_auth(Some(test_web_bot_auth_signer())); + + let first = fetch_with_config_for_test(&config, Request::get(&server.url())?)?; + let second = fetch_with_config_for_test(&config, Request::get(&server.url())?)?; + assert_eq!(first.body_text(), "first"); + assert_eq!(second.body_text(), "second"); + assert!(!first.from_cache); + assert!(!second.from_cache); + assert_eq!(server.hits(), 2); + let request_heads = server.request_heads(); + assert_ne!( + request_head_header_value(&request_heads[0], "Signature-Input"), + request_head_header_value(&request_heads[1], "Signature-Input") + ); + assert!( + !cache_dir.exists() || fs::read_dir(&cache_dir)?.next().is_none(), + "authenticated responses must not enter the shared disk cache" + ); + + server.shutdown(); + let _ = fs::remove_dir_all(cache_dir); + Ok(()) +} + #[tokio::test] async fn fetch_raw_stream_finishes_switching_protocols_body_without_connection_close() -> Result<()> { @@ -3616,6 +3797,34 @@ fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a s .map(|(_, value)| value.as_str()) } +fn request_head_header_value<'a>(request_head: &'a str, name: &str) -> Option<&'a str> { + request_head.lines().skip(1).find_map(|line| { + let (header_name, value) = line.split_once(':')?; + header_name.eq_ignore_ascii_case(name).then(|| value.trim()) + }) +} + +fn nonce_from_signature_input(signature_input: &str) -> &str { + signature_input + .split(";nonce=\"") + .nth(1) + .and_then(|value| value.split('"').next()) + .expect("Signature-Input should contain a nonce parameter") +} + +fn assert_request_extra_signature_matches_wire( + extra_info: &crate::NetworkRequestExtraInfo, + request_head: &str, +) { + for name in ["Signature-Agent", "Signature-Input", "Signature"] { + assert_eq!( + header_value(&extra_info.headers, name), + request_head_header_value(request_head, name), + "network extra info and wire request differ for {name}" + ); + } +} + #[test] fn fetch_client_cache_respects_pragma_no_cache_response() { let cache_dir = unique_test_cache_dir(); diff --git a/moli-fetch/src/tests/support.rs b/moli-fetch/src/tests/support.rs index d7b86d8bc..5d3b8f06b 100644 --- a/moli-fetch/src/tests/support.rs +++ b/moli-fetch/src/tests/support.rs @@ -43,6 +43,7 @@ pub(super) struct ScriptedHttps11Server { addr: std::net::SocketAddr, hits: Arc, requests: Arc>>, + request_heads: Arc>>, shutdown_tx: std_mpsc::Sender<()>, join_handle: Option>, } @@ -56,16 +57,19 @@ impl ScriptedHttps11Server { let addr = listener.local_addr().unwrap(); let hits = Arc::new(AtomicUsize::new(0)); let requests = Arc::new(Mutex::new(Vec::new())); + let request_heads = Arc::new(Mutex::new(Vec::new())); let (shutdown_tx, shutdown_rx) = std_mpsc::channel(); let responses = Arc::new(Mutex::new(VecDeque::from(responses))); let hits_for_thread = Arc::clone(&hits); let requests_for_thread = Arc::clone(&requests); + let request_heads_for_thread = Arc::clone(&request_heads); let join_handle = thread::spawn(move || { run_https11_server( listener, shutdown_rx, hits_for_thread, requests_for_thread, + request_heads_for_thread, responses, ); }); @@ -74,6 +78,7 @@ impl ScriptedHttps11Server { addr, hits, requests, + request_heads, shutdown_tx, join_handle: Some(join_handle), } @@ -83,6 +88,11 @@ impl ScriptedHttps11Server { format!("https://{}/cache", self.addr) } + pub(super) fn url_path(&self, path: &str) -> String { + let path = path.strip_prefix('/').unwrap_or(path); + format!("https://{}/{}", self.addr, path) + } + pub(super) fn hits(&self) -> usize { self.hits.load(Ordering::SeqCst) } @@ -91,6 +101,10 @@ impl ScriptedHttps11Server { self.requests.lock().clone() } + pub(super) fn request_heads(&self) -> Vec { + self.request_heads.lock().clone() + } + pub(super) fn shutdown(mut self) { let _ = self.shutdown_tx.send(()); if let Some(join_handle) = self.join_handle.take() { @@ -710,6 +724,7 @@ fn run_https11_server( shutdown_rx: std_mpsc::Receiver<()>, hits: Arc, requests: Arc>>, + request_heads: Arc>>, responses: Arc>>, ) { let rt = tokio::runtime::Builder::new_current_thread() @@ -731,6 +746,7 @@ fn run_https11_server( let tls_acceptor = tls_acceptor.clone(); let hits = Arc::clone(&hits); let requests = Arc::clone(&requests); + let request_heads = Arc::clone(&request_heads); let responses = Arc::clone(&responses); tokio::spawn(async move { let Ok(mut stream) = tls_acceptor.accept(stream).await else { @@ -751,6 +767,9 @@ fn run_https11_server( if let Some(path) = request_path_from_head(&request) { requests.lock().push(path); } + request_heads + .lock() + .push(String::from_utf8_lossy(&request).into_owned()); let _ = hits.fetch_add(1, Ordering::SeqCst) + 1; let response_spec = { let mut responses = responses.lock(); diff --git a/moli-web-bot-auth/Cargo.toml b/moli-web-bot-auth/Cargo.toml new file mode 100644 index 000000000..5ce2292f6 --- /dev/null +++ b/moli-web-bot-auth/Cargo.toml @@ -0,0 +1,18 @@ +[package] +license.workspace = true +name = "moli-web-bot-auth" +version = "0.1.0" +edition = "2024" +description = "Web Bot Auth HTTP message signatures for Moli" + +[dependencies] +anyhow = "1.0.100" +base64 = "0.22" +moli-crypto = { path = "../moli-crypto" } +url = "2.5.7" + +[dev-dependencies] +aws-lc-rs = { version = "1.16.2", default-features = false, features = ["aws-lc-sys"] } + +[lints] +workspace = true diff --git a/moli-web-bot-auth/src/key.rs b/moli-web-bot-auth/src/key.rs new file mode 100644 index 000000000..d9f7ea81b --- /dev/null +++ b/moli-web-bot-auth/src/key.rs @@ -0,0 +1,133 @@ +use std::str; + +use anyhow::{Context, Result, anyhow, bail}; +use base64::{Engine as _, engine::general_purpose}; +use moli_crypto::{Ed25519SigningKey, sha256_digest}; + +pub(crate) struct WebBotAuthKey { + signing_key: Ed25519SigningKey, + public_key: [u8; 32], + keyid: String, +} + +impl WebBotAuthKey { + pub(crate) fn from_pem(private_key_pem: &[u8], expected_keyid: Option<&str>) -> Result { + let private_key_der = decode_pkcs8_private_key_pem(private_key_pem)?; + let signing_key = Ed25519SigningKey::from_pkcs8(&private_key_der).map_err(|_| { + anyhow!("web bot auth key must be an unencrypted PKCS#8 Ed25519 private key") + })?; + let public_key = *signing_key.public_key(); + let keyid = jwk_thumbprint(&public_key); + if let Some(expected_keyid) = expected_keyid + && expected_keyid != keyid + { + bail!( + "--web-bot-auth-keyid does not match the private key; expected `{expected_keyid}`, computed `{keyid}`" + ); + } + + Ok(Self { + signing_key, + public_key, + keyid, + }) + } + + pub(crate) fn keyid(&self) -> &str { + &self.keyid + } + + pub(crate) fn public_key(&self) -> &[u8; 32] { + &self.public_key + } + + pub(crate) fn sign(&self, message: &[u8]) -> Result<[u8; 64]> { + self.signing_key + .sign(message) + .context("failed to create web bot auth Ed25519 signature") + } +} + +fn decode_pkcs8_private_key_pem(private_key_pem: &[u8]) -> Result> { + const BEGIN: &str = "-----BEGIN PRIVATE KEY-----"; + const END: &str = "-----END PRIVATE KEY-----"; + + let pem = str::from_utf8(private_key_pem) + .context("web bot auth private key PEM is not valid UTF-8")?; + if pem.contains("-----BEGIN ENCRYPTED PRIVATE KEY-----") { + bail!("encrypted web bot auth private keys are not supported"); + } + let pem = pem.trim(); + let body = pem + .strip_prefix(BEGIN) + .and_then(|pem| pem.strip_suffix(END)) + .ok_or_else(|| { + anyhow!("web bot auth key must use BEGIN PRIVATE KEY PKCS#8 PEM encoding") + })?; + if body.contains("-----") { + bail!("web bot auth key PEM must contain exactly one private key"); + } + let encoded = body + .chars() + .filter(|character| !character.is_ascii_whitespace()) + .collect::(); + if encoded.is_empty() { + bail!("web bot auth private key PEM body is empty"); + } + general_purpose::STANDARD + .decode(encoded) + .context("web bot auth private key PEM contains invalid base64") +} + +fn jwk_thumbprint(public_key: &[u8; 32]) -> String { + let x = general_purpose::URL_SAFE_NO_PAD.encode(public_key); + let canonical_jwk = format!(r#"{{"crv":"Ed25519","kty":"OKP","x":"{x}"}}"#); + general_purpose::URL_SAFE_NO_PAD.encode(sha256_digest(canonical_jwk)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{EXPECTED_KEYID, RFC_9421_ED25519_PRIVATE_KEY}; + + #[test] + fn derives_rfc_jwk_thumbprint_and_public_key() { + let key = WebBotAuthKey::from_pem( + RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), + Some(EXPECTED_KEYID), + ) + .unwrap(); + + assert_eq!(key.keyid(), EXPECTED_KEYID); + assert_eq!( + general_purpose::URL_SAFE_NO_PAD.encode(key.public_key()), + "JrQLj5P_89iXES9-vFgrIy29clF9CC_oPPsw3c5D0bs" + ); + } + + #[test] + fn rejects_mismatched_keyid_and_invalid_pem() { + let wrong_keyid = + WebBotAuthKey::from_pem(RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), Some("wrong")) + .err() + .expect("mismatched keyid should fail") + .to_string(); + assert!(wrong_keyid.contains("does not match")); + assert!(wrong_keyid.contains(EXPECTED_KEYID)); + + let invalid_pem = WebBotAuthKey::from_pem(b"not a private key", None) + .err() + .expect("invalid PEM should fail") + .to_string(); + assert!(invalid_pem.contains("BEGIN PRIVATE KEY")); + + let encrypted = WebBotAuthKey::from_pem( + b"-----BEGIN ENCRYPTED PRIVATE KEY-----\nAA==\n-----END ENCRYPTED PRIVATE KEY-----", + None, + ) + .err() + .expect("encrypted PEM should fail") + .to_string(); + assert!(encrypted.contains("encrypted")); + } +} diff --git a/moli-web-bot-auth/src/lib.rs b/moli-web-bot-auth/src/lib.rs new file mode 100644 index 000000000..2c13d9d0c --- /dev/null +++ b/moli-web-bot-auth/src/lib.rs @@ -0,0 +1,12 @@ +//! Web Bot Auth request signing built on RFC 9421 HTTP message signatures. + +mod key; +mod profile; +mod signer; +mod wire; + +#[cfg(test)] +mod test_support; + +pub use profile::WebBotAuthProfile; +pub use signer::WebBotAuthSigner; diff --git a/moli-web-bot-auth/src/profile.rs b/moli-web-bot-auth/src/profile.rs new file mode 100644 index 000000000..122ed672c --- /dev/null +++ b/moli-web-bot-auth/src/profile.rs @@ -0,0 +1,15 @@ +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum WebBotAuthProfile { + #[default] + Cloudflare, + IetfDraft01, +} + +impl WebBotAuthProfile { + pub const fn as_str(self) -> &'static str { + match self { + Self::Cloudflare => "cloudflare", + Self::IetfDraft01 => "ietf-01", + } + } +} diff --git a/moli-web-bot-auth/src/signer.rs b/moli-web-bot-auth/src/signer.rs new file mode 100644 index 000000000..3e811aa85 --- /dev/null +++ b/moli-web-bot-auth/src/signer.rs @@ -0,0 +1,291 @@ +use std::{ + fmt, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::{SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result, anyhow, bail}; +use base64::{Engine as _, engine::general_purpose}; +use moli_crypto::{DigestAlgorithm, fill_secure_random}; +use url::Url; + +use crate::{ + key::WebBotAuthKey, + profile::WebBotAuthProfile, + wire::{SignatureOptions, sign_request}, +}; + +const SIGNATURE_LABEL: &str = "sig1"; +const SIGNATURE_TTL_SECS: u64 = 60; +const NONCE_BYTES: usize = 64; + +#[derive(Clone)] +pub struct WebBotAuthSigner { + inner: Arc, +} + +struct WebBotAuthSignerInner { + key: WebBotAuthKey, + signature_agent_origin: String, + profile: WebBotAuthProfile, + nonce_seed: [u8; 32], + nonce_counter: AtomicU64, +} + +impl WebBotAuthSigner { + pub fn from_pem( + private_key_pem: &[u8], + domain: &str, + expected_keyid: Option<&str>, + profile: WebBotAuthProfile, + ) -> Result { + let key = WebBotAuthKey::from_pem(private_key_pem, expected_keyid)?; + let signature_agent_origin = normalize_signature_agent_origin(domain)?; + let mut nonce_seed = [0_u8; 32]; + fill_secure_random(&mut nonce_seed) + .map_err(|_| anyhow!("failed to initialize web bot auth nonce generation"))?; + + Ok(Self { + inner: Arc::new(WebBotAuthSignerInner { + key, + signature_agent_origin, + profile, + nonce_seed, + nonce_counter: AtomicU64::new(0), + }), + }) + } + + pub fn keyid(&self) -> &str { + self.inner.key.keyid() + } + + pub fn public_key(&self) -> &[u8; 32] { + self.inner.key.public_key() + } + + pub fn signature_agent_origin(&self) -> &str { + &self.inner.signature_agent_origin + } + + pub fn profile(&self) -> WebBotAuthProfile { + self.inner.profile + } + + pub fn append_request_headers( + &self, + headers: &mut Vec<(String, String)>, + method: &str, + request_url: &Url, + ) -> Result<()> { + if request_url.scheme() != "https" { + return Ok(()); + } + + headers.retain(|(name, _)| !is_web_bot_auth_header(name)); + let created = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let signed = sign_request( + &self.inner.key, + &self.inner.signature_agent_origin, + self.inner.profile, + method, + request_url, + &SignatureOptions { + signature_label: SIGNATURE_LABEL, + signature_agent_label: SIGNATURE_LABEL, + created, + expires: created.saturating_add(SIGNATURE_TTL_SECS), + nonce: self.next_nonce(), + cover_request_target: true, + }, + )?; + headers.extend(signed.headers.into_pairs()); + Ok(()) + } + + fn next_nonce(&self) -> String { + let counter = self.inner.nonce_counter.fetch_add(1, Ordering::Relaxed); + let mut material = Vec::with_capacity(32 + 8 + 24); + material.extend_from_slice(b"moli-web-bot-auth-nonce\0"); + material.extend_from_slice(&self.inner.nonce_seed); + material.extend_from_slice(&counter.to_be_bytes()); + let nonce = DigestAlgorithm::Sha512.digest_bytes(&material); + debug_assert_eq!(nonce.len(), NONCE_BYTES); + general_purpose::STANDARD.encode(nonce) + } +} + +impl fmt::Debug for WebBotAuthSigner { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("WebBotAuthSigner") + .field("keyid", &self.inner.key.keyid()) + .field("signature_agent_origin", &self.inner.signature_agent_origin) + .field("profile", &self.inner.profile) + .finish_non_exhaustive() + } +} + +impl PartialEq for WebBotAuthSigner { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) + || (self.inner.key.public_key() == other.inner.key.public_key() + && self.inner.signature_agent_origin == other.inner.signature_agent_origin + && self.inner.profile == other.inner.profile) + } +} + +impl Eq for WebBotAuthSigner {} + +fn normalize_signature_agent_origin(domain: &str) -> Result { + if domain.is_empty() || domain.trim() != domain { + bail!("--web-bot-auth-domain must be a non-empty domain without surrounding whitespace"); + } + if domain.contains(['/', '?', '#', '@']) { + bail!( + "--web-bot-auth-domain must contain only a host and optional port, without a scheme, credentials, path, query, or fragment" + ); + } + + let url = Url::parse(&format!("https://{domain}")) + .context("failed to parse --web-bot-auth-domain as an HTTPS origin")?; + if url.host().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != "/" + || url.query().is_some() + || url.fragment().is_some() + { + bail!("--web-bot-auth-domain must identify a single HTTPS origin"); + } + Ok(url.origin().ascii_serialization()) +} + +fn is_web_bot_auth_header(name: &str) -> bool { + name.eq_ignore_ascii_case("signature-agent") + || name.eq_ignore_ascii_case("signature-input") + || name.eq_ignore_ascii_case("signature") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{EXPECTED_KEYID, RFC_9421_ED25519_PRIVATE_KEY, signer}; + + #[test] + fn normalizes_signature_agent_origin() { + let signer = signer(WebBotAuthProfile::Cloudflare); + + assert_eq!(signer.keyid(), EXPECTED_KEYID); + assert_eq!( + signer.signature_agent_origin(), + "https://signature-agent.test" + ); + } + + #[test] + fn production_requests_use_unique_nonces() { + let signer = signer(WebBotAuthProfile::Cloudflare); + let request_url = Url::parse("https://example.com/path").unwrap(); + let mut first_headers = Vec::new(); + signer + .append_request_headers(&mut first_headers, "post", &request_url) + .unwrap(); + let mut second_headers = Vec::new(); + signer + .append_request_headers(&mut second_headers, "post", &request_url) + .unwrap(); + + let first_input = header_value(&first_headers, "signature-input"); + assert!(first_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")")); + assert!(first_input.contains(";expires=")); + assert!(first_input.contains(";tag=\"web-bot-auth\"")); + assert_ne!( + nonce_from_signature_input(first_input), + nonce_from_signature_input(header_value(&second_headers, "signature-input")) + ); + } + + #[test] + fn omits_signatures_on_insecure_requests_and_replaces_spoofed_https_headers() { + let signer = signer(WebBotAuthProfile::Cloudflare); + let spoofed = vec![ + ("Signature-Agent".to_owned(), "spoofed".to_owned()), + ("Signature-Input".to_owned(), "spoofed".to_owned()), + ("Signature".to_owned(), "spoofed".to_owned()), + ]; + let mut http_headers = spoofed.clone(); + signer + .append_request_headers( + &mut http_headers, + "GET", + &Url::parse("http://example.com/").unwrap(), + ) + .unwrap(); + assert_eq!(http_headers, spoofed); + + let mut https_headers = spoofed; + signer + .append_request_headers( + &mut https_headers, + "GET", + &Url::parse("https://example.com/").unwrap(), + ) + .unwrap(); + assert_eq!(https_headers.len(), 3); + assert!(https_headers.iter().all(|(_, value)| value != "spoofed")); + } + + #[test] + fn validates_domain() { + for domain in [ + "", + " example.com", + "https://example.com", + "user@example.com", + "example.com/path", + "example.com?query", + ] { + let error = WebBotAuthSigner::from_pem( + RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), + domain, + None, + WebBotAuthProfile::Cloudflare, + ) + .unwrap_err(); + assert!(error.to_string().contains("--web-bot-auth-domain")); + } + } + + #[test] + fn debug_output_never_contains_private_key_material() { + let signer = signer(WebBotAuthProfile::Cloudflare); + let rendered = format!("{signer:?}"); + + assert!(rendered.contains(EXPECTED_KEYID)); + assert!(!rendered.contains("IJ+DYvh6")); + assert!(!rendered.contains("PRIVATE KEY")); + } + + fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> &'a str { + headers + .iter() + .find(|(candidate, _)| candidate.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + .unwrap() + } + + fn nonce_from_signature_input(input: &str) -> &str { + input + .split(";nonce=\"") + .nth(1) + .and_then(|value| value.split('"').next()) + .unwrap() + } +} diff --git a/moli-web-bot-auth/src/test_support.rs b/moli-web-bot-auth/src/test_support.rs new file mode 100644 index 000000000..d156f1081 --- /dev/null +++ b/moli-web-bot-auth/src/test_support.rs @@ -0,0 +1,24 @@ +use crate::{WebBotAuthProfile, WebBotAuthSigner, key::WebBotAuthKey}; + +pub(crate) const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\ +MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\ +-----END PRIVATE KEY-----\n"; +pub(crate) const EXPECTED_KEYID: &str = "poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U"; + +pub(crate) fn key() -> WebBotAuthKey { + WebBotAuthKey::from_pem( + RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), + Some(EXPECTED_KEYID), + ) + .unwrap() +} + +pub(crate) fn signer(profile: WebBotAuthProfile) -> WebBotAuthSigner { + WebBotAuthSigner::from_pem( + RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), + "signature-agent.test", + Some(EXPECTED_KEYID), + profile, + ) + .unwrap() +} diff --git a/moli-web-bot-auth/src/wire.rs b/moli-web-bot-auth/src/wire.rs new file mode 100644 index 000000000..7b8e0cc98 --- /dev/null +++ b/moli-web-bot-auth/src/wire.rs @@ -0,0 +1,279 @@ +use anyhow::{Context, Result}; +use base64::{Engine as _, engine::general_purpose}; +use url::{Host, Url}; + +use crate::{key::WebBotAuthKey, profile::WebBotAuthProfile}; + +pub(crate) struct SignatureOptions<'a> { + pub(crate) signature_label: &'a str, + pub(crate) signature_agent_label: &'a str, + pub(crate) created: u64, + pub(crate) expires: u64, + pub(crate) nonce: String, + pub(crate) cover_request_target: bool, +} + +pub(crate) struct SignedRequest { + pub(crate) headers: WebBotAuthHeaders, + #[cfg(test)] + pub(crate) signature_base: String, +} + +pub(crate) struct WebBotAuthHeaders { + pub(crate) signature_agent: String, + pub(crate) signature_input: String, + pub(crate) signature: String, +} + +impl WebBotAuthHeaders { + pub(crate) fn into_pairs(self) -> [(String, String); 3] { + [ + ("Signature-Agent".to_owned(), self.signature_agent), + ("Signature-Input".to_owned(), self.signature_input), + ("Signature".to_owned(), self.signature), + ] + } +} + +pub(crate) fn sign_request( + key: &WebBotAuthKey, + signature_agent_origin: &str, + profile: WebBotAuthProfile, + method: &str, + request_url: &Url, + options: &SignatureOptions<'_>, +) -> Result { + let authority = request_authority(request_url)?; + let method = method.to_ascii_uppercase(); + let path = request_url.path(); + let signature_agent = signature_agent_header( + profile, + signature_agent_origin, + options.signature_agent_label, + ); + let signature_agent_component = + signature_agent_component_identifier(profile, options.signature_agent_label); + + let mut covered_components = vec!["\"@authority\"".to_owned()]; + if options.cover_request_target { + covered_components.push("\"@method\"".to_owned()); + covered_components.push("\"@path\"".to_owned()); + } + covered_components.push(signature_agent_component.clone()); + + let signature_params = format!( + "({});created={};keyid=\"{}\";alg=\"ed25519\";expires={};nonce=\"{}\";tag=\"web-bot-auth\"", + covered_components.join(" "), + options.created, + key.keyid(), + options.expires, + options.nonce, + ); + + let mut signature_base_lines = vec![format!("\"@authority\": {authority}")]; + if options.cover_request_target { + signature_base_lines.push(format!("\"@method\": {method}")); + signature_base_lines.push(format!("\"@path\": {path}")); + } + signature_base_lines.push(format!( + "{signature_agent_component}: \"{signature_agent_origin}\"" + )); + signature_base_lines.push(format!("\"@signature-params\": {signature_params}")); + let signature_base = signature_base_lines.join("\n"); + let signature = key.sign(signature_base.as_bytes())?; + + Ok(SignedRequest { + headers: WebBotAuthHeaders { + signature_agent, + signature_input: format!("{}={signature_params}", options.signature_label), + signature: format!( + "{}=:{}:", + options.signature_label, + general_purpose::STANDARD.encode(signature) + ), + }, + #[cfg(test)] + signature_base, + }) +} + +fn signature_agent_header( + profile: WebBotAuthProfile, + signature_agent_origin: &str, + label: &str, +) -> String { + match profile { + WebBotAuthProfile::Cloudflare => format!("\"{signature_agent_origin}\""), + WebBotAuthProfile::IetfDraft01 => { + format!("{label}=\"{signature_agent_origin}\"") + } + } +} + +fn signature_agent_component_identifier(profile: WebBotAuthProfile, label: &str) -> String { + match profile { + WebBotAuthProfile::Cloudflare => "\"signature-agent\"".to_owned(), + WebBotAuthProfile::IetfDraft01 => { + format!("\"signature-agent\";key=\"{label}\"") + } + } +} + +fn request_authority(request_url: &Url) -> Result { + let host = match request_url + .host() + .context("Web Bot Auth request URL must have an authority")? + { + Host::Domain(domain) => domain.to_owned(), + Host::Ipv4(address) => address.to_string(), + Host::Ipv6(address) => format!("[{address}]"), + }; + Ok(match request_url.port() { + Some(port) => format!("{host}:{port}"), + None => host, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{EXPECTED_KEYID, key}; + use aws_lc_rs::signature::{ED25519, UnparsedPublicKey}; + + #[test] + fn matches_ietf_draft_01_ed25519_test_vector() { + let key = key(); + let request_url = Url::parse("https://example.com/foo?param=Value&Pet=dog").unwrap(); + let signed = sign_request( + &key, + "https://signature-agent.test", + WebBotAuthProfile::IetfDraft01, + "POST", + &request_url, + &SignatureOptions { + signature_label: "sig2", + signature_agent_label: "agent2", + created: 1_735_689_600, + expires: 4_889_289_600, + nonce: "n9p433xm+NJ3ph3upfBIGmsuwHw387YV7Q/F+6BSpGCVjYCqQw6rznNA8PVVLySrAWsv0hQtFioQb6E1YsauiA==".to_owned(), + cover_request_target: false, + }, + ) + .unwrap(); + + assert_eq!(key.keyid(), EXPECTED_KEYID); + assert_eq!( + signed.headers.signature_agent, + "agent2=\"https://signature-agent.test\"" + ); + assert_eq!( + signed.headers.signature_input, + "sig2=(\"@authority\" \"signature-agent\";key=\"agent2\");created=1735689600;keyid=\"poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U\";alg=\"ed25519\";expires=4889289600;nonce=\"n9p433xm+NJ3ph3upfBIGmsuwHw387YV7Q/F+6BSpGCVjYCqQw6rznNA8PVVLySrAWsv0hQtFioQb6E1YsauiA==\";tag=\"web-bot-auth\"" + ); + assert_eq!( + signed.headers.signature, + "sig2=:RdNFx5Bj6au3YgAMQL/RzmUlZE8QZLIaXGRpw985hWnwPfMxT228NMk6ehRS1PSl4e8PhbNZACSanGdhEwYCCg==:" + ); + } + + #[test] + fn matches_cloudflare_legacy_ed25519_test_vector() { + let key = key(); + let request_url = Url::parse("https://example.com/foo?param=Value&Pet=dog").unwrap(); + let signed = sign_request( + &key, + "https://signature-agent.test", + WebBotAuthProfile::Cloudflare, + "POST", + &request_url, + &SignatureOptions { + signature_label: "sig2", + signature_agent_label: "agent2", + created: 1_735_689_600, + expires: 1_735_693_200, + nonce: "e8N7S2MFd/qrd6T2R3tdfAuuANngKI7LFtKYI/vowzk4lAZYadIX6wW25MwG7DCT9RUKAJ0qVkU0mEeLElW1qg==".to_owned(), + cover_request_target: false, + }, + ) + .unwrap(); + + assert_eq!( + signed.headers.signature_agent, + "\"https://signature-agent.test\"" + ); + assert_eq!( + signed.headers.signature, + "sig2=:jdq0SqOwHdyHr9+r5jw3iYZH6aNGKijYp/EstF4RQTQdi5N5YYKrD+mCT1HA1nZDsi6nJKuHxUi/5Syp3rLWBA==:" + ); + } + + #[test] + fn production_components_cover_authority_method_and_path() { + let key = key(); + let request_url = Url::parse("https://example.com:8443/a%20path?query=yes").unwrap(); + let signed = sign_request( + &key, + "https://signature-agent.test", + WebBotAuthProfile::Cloudflare, + "post", + &request_url, + &SignatureOptions { + signature_label: "sig1", + signature_agent_label: "sig1", + created: 1_735_689_600, + expires: 1_735_689_660, + nonce: general_purpose::STANDARD.encode([7_u8; 64]), + cover_request_target: true, + }, + ) + .unwrap(); + + assert!( + signed + .signature_base + .contains("\"@authority\": example.com:8443") + ); + assert!(signed.signature_base.contains("\"@method\": POST")); + assert!(signed.signature_base.contains("\"@path\": /a%20path")); + let signature = signed + .headers + .signature + .strip_prefix("sig1=:") + .and_then(|value| value.strip_suffix(':')) + .and_then(|value| general_purpose::STANDARD.decode(value).ok()) + .unwrap(); + UnparsedPublicKey::new(&ED25519, key.public_key()) + .verify(signed.signature_base.as_bytes(), &signature) + .unwrap(); + } + + #[test] + fn signing_rejects_a_request_url_without_an_authority() { + let key = key(); + let request_url = Url::parse("data:text/plain,no-authority").unwrap(); + let result = sign_request( + &key, + "https://signature-agent.test", + WebBotAuthProfile::Cloudflare, + "GET", + &request_url, + &SignatureOptions { + signature_label: "sig1", + signature_agent_label: "sig1", + created: 1_735_689_600, + expires: 1_735_689_660, + nonce: general_purpose::STANDARD.encode([7_u8; 64]), + cover_request_target: true, + }, + ); + + let error = match result { + Ok(_) => panic!("signing a URL without an authority must fail"), + Err(error) => error, + }; + assert_eq!( + error.to_string(), + "Web Bot Auth request URL must have an authority" + ); + } +} diff --git a/moli/src/cli.rs b/moli/src/cli.rs index ede40b461..e35e9eaa4 100644 --- a/moli/src/cli.rs +++ b/moli/src/cli.rs @@ -32,6 +32,10 @@ const FETCH_INFER_FLAGS: &[&str] = &[ "--disable-subframes", "--wait-ms", "--profile-dir", + "--web-bot-auth-key-file", + "--web-bot-auth-keyid", + "--web-bot-auth-domain", + "--web-bot-auth-profile", ]; const SERVE_INFER_FLAGS: &[&str] = &["--host", "--port", "--timeout", "--layout"]; const EXPLICIT_COMMANDS: &[&str] = &["fetch", "serve", "help", "version"]; @@ -367,6 +371,35 @@ pub struct CommonArgs { #[arg(long)] pub user_agent_suffix: Option, + + /// Unencrypted PKCS#8 Ed25519 private key used for Web Bot Auth signatures. + #[arg(long, value_name = "PATH", requires = "web_bot_auth_domain")] + pub web_bot_auth_key_file: Option, + + /// Assert the RFC 7638 JWK thumbprint derived from the Web Bot Auth key. + #[arg(long, value_name = "THUMBPRINT", requires = "web_bot_auth_key_file")] + pub web_bot_auth_keyid: Option, + + /// Operator domain publishing /.well-known/http-message-signatures-directory. + #[arg(long, value_name = "DOMAIN", requires = "web_bot_auth_key_file")] + pub web_bot_auth_domain: Option, + + /// Signature-Agent wire format. Cloudflare compatibility is the default. + #[arg( + long, + value_enum, + default_value = "cloudflare", + requires = "web_bot_auth_key_file" + )] + pub web_bot_auth_profile: WebBotAuthProfileChoice, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum)] +pub enum WebBotAuthProfileChoice { + #[default] + Cloudflare, + #[value(name = "ietf-01")] + IetfDraft01, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] diff --git a/moli/src/config.rs b/moli/src/config.rs index 4cd98d9f9..28e979438 100644 --- a/moli/src/config.rs +++ b/moli/src/config.rs @@ -6,10 +6,13 @@ use moli_core::{ page::{SubresourceJsonPathEquals, SubresourceResponseWaitCriteria}, runtime::BrowserConfig, }; +use moli_fetch::{FetchConfig, WebBotAuthProfile, WebBotAuthSigner}; use std::path::PathBuf; use std::str::FromStr; -use crate::cli::{Cli, Commands, CommonArgs, DumpFormat, LogFormat, StripOptions}; +use crate::cli::{ + Cli, Commands, CommonArgs, DumpFormat, LogFormat, StripOptions, WebBotAuthProfileChoice, +}; use crate::network_trace::NetworkTraceConfigSummary; pub use moli_protocol_server::ServerConfig; @@ -183,6 +186,7 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()> .browser .fetch_mut() .set_tls_verify_host(!common.insecure_disable_tls_host_verification); + configure_web_bot_auth(config.browser.fetch_mut(), common)?; for source in &common.document_start_script { config.add_document_start_script(source.clone()); @@ -199,6 +203,37 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()> Ok(()) } +fn configure_web_bot_auth(fetch: &mut FetchConfig, common: &CommonArgs) -> Result<()> { + let (key_file, domain) = match ( + common.web_bot_auth_key_file.as_deref(), + common.web_bot_auth_domain.as_deref(), + ) { + (None, None) => { + if common.web_bot_auth_keyid.is_some() { + bail!("--web-bot-auth-keyid requires --web-bot-auth-key-file"); + } + return Ok(()); + } + (Some(_), None) => bail!("--web-bot-auth-key-file requires --web-bot-auth-domain"), + (None, Some(_)) => bail!("--web-bot-auth-domain requires --web-bot-auth-key-file"), + (Some(key_file), Some(domain)) => (key_file, domain), + }; + let private_key_pem = std::fs::read(key_file) + .with_context(|| format!("failed to read Web Bot Auth private key `{key_file}`"))?; + let profile = match common.web_bot_auth_profile { + WebBotAuthProfileChoice::Cloudflare => WebBotAuthProfile::Cloudflare, + WebBotAuthProfileChoice::IetfDraft01 => WebBotAuthProfile::IetfDraft01, + }; + let signer = WebBotAuthSigner::from_pem( + &private_key_pem, + domain, + common.web_bot_auth_keyid.as_deref(), + profile, + )?; + fetch.set_web_bot_auth(Some(signer)); + Ok(()) +} + fn validate_http_host_resolve_entries(entries: &[String]) -> Result<()> { for entry in entries { validate_http_host_resolve_entry(entry)?; diff --git a/moli/tests/cli.rs b/moli/tests/cli.rs index fb46bdd75..bde0fa79f 100644 --- a/moli/tests/cli.rs +++ b/moli/tests/cli.rs @@ -1,15 +1,27 @@ use clap::Parser; -use std::{num::NonZeroU32, process::Command}; +use std::{ + fs, + num::NonZeroU32, + path::PathBuf, + process::Command, + sync::atomic::{AtomicU64, Ordering}, +}; use moli::cli::{ Cli, Commands, CommonArgs, DumpFormat, FetchArgs, FetchWaitUntil, LogFormat, LogLevel, RequestHeaderArg, ResponseJsonPathArg, ServeArgs, StripModeChoice, StripOptions, - normalize_args_for_compat, + WebBotAuthProfileChoice, normalize_args_for_compat, }; use moli::config::AppConfig; use moli_browser_profile::BrowserProfilePaths; use moli_core::OptionalResourceFetchMask; -use moli_fetch::FetchConfig; +use moli_fetch::{FetchConfig, WebBotAuthProfile}; + +const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\ +MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\ +-----END PRIVATE KEY-----\n"; +const RFC_9421_ED25519_KEYID: &str = "poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U"; +static NEXT_TEMP_KEY_FILE: AtomicU64 = AtomicU64::new(0); #[test] fn parses_explicit_fetch_command_with_compatibility_flags() { @@ -125,6 +137,10 @@ fn parses_explicit_fetch_command_with_compatibility_flags() { log_filter_scopes: Some("http,event".to_owned()), user_agent: None, user_agent_suffix: Some("internal-tester".to_owned()), + web_bot_auth_key_file: None, + web_bot_auth_keyid: None, + web_bot_auth_domain: None, + web_bot_auth_profile: WebBotAuthProfileChoice::Cloudflare, }, url: "https://example.com".to_owned(), })) @@ -200,22 +216,79 @@ fn parses_binary_dump_modes_with_inferred_fetch_command() { } #[test] -fn rejects_removed_web_bot_auth_flags() { - for flag in [ +fn parses_web_bot_auth_flags() { + let cli = Cli::try_parse_from(normalize_args_for_compat([ + "moli", + "fetch", "--web-bot-auth-key-file", + "bot-key.pem", "--web-bot-auth-keyid", + RFC_9421_ED25519_KEYID, "--web-bot-auth-domain", - ] { - let error = Cli::try_parse_from(normalize_args_for_compat([ - "moli", - "fetch", - flag, - "unused", - "https://example.com", - ])) - .unwrap_err(); + "bot.example", + "--web-bot-auth-profile", + "ietf-01", + "https://example.com", + ])) + .unwrap(); - assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument); + let Commands::Fetch(args) = cli.command else { + panic!("expected fetch command"); + }; + assert_eq!( + args.common.web_bot_auth_key_file.as_deref(), + Some("bot-key.pem") + ); + assert_eq!( + args.common.web_bot_auth_keyid.as_deref(), + Some(RFC_9421_ED25519_KEYID) + ); + assert_eq!( + args.common.web_bot_auth_domain.as_deref(), + Some("bot.example") + ); + assert_eq!( + args.common.web_bot_auth_profile, + WebBotAuthProfileChoice::IetfDraft01 + ); +} + +#[test] +fn web_bot_auth_profile_default_does_not_enable_signing() { + let cli = Cli::try_parse_from(normalize_args_for_compat([ + "moli", + "fetch", + "https://example.com", + ])) + .unwrap(); + + let Commands::Fetch(args) = cli.command else { + panic!("expected fetch command"); + }; + assert_eq!( + args.common.web_bot_auth_profile, + WebBotAuthProfileChoice::Cloudflare + ); + assert!(args.common.web_bot_auth_key_file.is_none()); +} + +#[test] +fn web_bot_auth_flags_require_key_and_domain_together() { + for args in [ + vec!["--web-bot-auth-key-file", "bot-key.pem"], + vec!["--web-bot-auth-domain", "bot.example"], + vec!["--web-bot-auth-keyid", RFC_9421_ED25519_KEYID], + vec!["--web-bot-auth-profile", "ietf-01"], + ] { + let mut command = vec!["moli", "fetch"]; + command.extend(args); + command.push("https://example.com"); + let error = Cli::try_parse_from(normalize_args_for_compat(command)).unwrap_err(); + + assert_eq!( + error.kind(), + clap::error::ErrorKind::MissingRequiredArgument + ); } } @@ -550,6 +623,54 @@ fn app_config_uses_moli_user_agent_defaults() { assert_eq!(config.browser.fetch().user_agent(), "ExampleBrowser/1.0"); } +#[test] +fn app_config_loads_and_validates_web_bot_auth_key() { + let key_file = write_temp_web_bot_auth_key(); + let key_path_arg = key_file.path.to_string_lossy().into_owned(); + let cli = Cli::try_parse_from(normalize_args_for_compat([ + "moli", + "fetch", + "--web-bot-auth-key-file", + key_path_arg.as_str(), + "--web-bot-auth-keyid", + RFC_9421_ED25519_KEYID, + "--web-bot-auth-domain", + "bot.example:8443", + "--web-bot-auth-profile", + "ietf-01", + "https://example.com", + ])) + .unwrap(); + + let config = AppConfig::from_cli(&cli).unwrap(); + let signer = config.browser.fetch().web_bot_auth().unwrap(); + assert_eq!(signer.keyid(), RFC_9421_ED25519_KEYID); + assert_eq!(signer.signature_agent_origin(), "https://bot.example:8443"); + assert_eq!(signer.profile(), WebBotAuthProfile::IetfDraft01); +} + +#[test] +fn app_config_rejects_mismatched_web_bot_auth_keyid() { + let key_file = write_temp_web_bot_auth_key(); + let key_path_arg = key_file.path.to_string_lossy().into_owned(); + let cli = Cli::try_parse_from(normalize_args_for_compat([ + "moli", + "fetch", + "--web-bot-auth-key-file", + key_path_arg.as_str(), + "--web-bot-auth-keyid", + "wrong-thumbprint", + "--web-bot-auth-domain", + "bot.example", + "https://example.com", + ])) + .unwrap(); + + let error = AppConfig::from_cli(&cli).unwrap_err().to_string(); + assert!(error.contains("does not match the private key")); + assert!(error.contains(RFC_9421_ED25519_KEYID)); +} + #[test] fn app_config_preserves_repeatable_request_headers() { let cli = Cli::try_parse_from(normalize_args_for_compat([ @@ -574,6 +695,26 @@ fn app_config_preserves_repeatable_request_headers() { assert!(config.browser.fetch().default_request_headers().is_empty()); } +struct TempWebBotAuthKeyFile { + path: PathBuf, +} + +impl Drop for TempWebBotAuthKeyFile { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +fn write_temp_web_bot_auth_key() -> TempWebBotAuthKeyFile { + let sequence = NEXT_TEMP_KEY_FILE.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "moli-web-bot-auth-{}-{sequence}.pem", + std::process::id() + )); + fs::write(&path, RFC_9421_ED25519_PRIVATE_KEY).unwrap(); + TempWebBotAuthKeyFile { path } +} + #[test] fn parses_request_headers_with_embedded_colons_and_empty_values() { let cli = Cli::try_parse_from(normalize_args_for_compat([ diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.md index e122efa68..e2deba9a1 100644 --- a/skills/moli-webfetch/references/fetch-recipes.md +++ b/skills/moli-webfetch/references/fetch-recipes.md @@ -135,6 +135,12 @@ For a crawl rather than a single lookup: - Use `--http-proxy`, `--http-no-proxy`, or `--http-host-resolve HOST:PORT:ADDR` when required by the environment. - Use either `--user-agent` or `--user-agent-suffix`, not both. +- Use `--web-bot-auth-key-file ` together with + `--web-bot-auth-domain ` only when the user supplied an authorized + bot identity. The key remains local, and Moli signs HTTPS requests only. +- Add `--web-bot-auth-keyid ` to assert the derived JWK thumbprint. + Keep the default Cloudflare profile unless the receiver explicitly supports + `--web-bot-auth-profile ietf-01`. - Use `--document-start-script` or `--document-start-script-file` only when the task explicitly requires pre-navigation instrumentation. - Combine `--block-private-networks` with `--block-cidrs` for untrusted URL