starred/moli
1
0
Fork 0
mirror of https://github.com/lexmount/moli.git synced 2026-09-26 16:01:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement Web Bot Auth signing

Browse Source
This commit is contained in:
ldm0
2026-08-16 15:19:40 +08:00
committed by Donough Liu
parent b0b6aa8062
commit ce7fa9cd17
23 changed files with 1461 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+12
View File
@@ -2221,6 +2221,7 @@ dependencies = [
"moli-trace",
"moli-url",
"moli-url-policy",
"moli-web-bot-auth",
"moli-web-mime",
"openssl-sys",
"parking_lot",
@@ -2825,6 +2826,17 @@ dependencies = [
"v8",
]
[[package]]
name = "moli-web-bot-auth"
version = "0.1.0"
dependencies = [
"anyhow",
"aws-lc-rs",
"base64 0.22.1",
"moli-crypto",
"url",
]
[[package]]
name = "moli-web-errors"
version = "0.1.0"
Cargo.toml
+1
View File
@@ -55,6 +55,7 @@ members = [
"moli-v8-test-util",
"moli-v8-util",
"moli-web-mime",
"moli-web-bot-auth",
"moli-web-errors",
"moli-protocol-webdriver-bidi",
"moli-protocol-webdriver-classic",
moli-crypto/src/ed25519.rs
+98
View File
@@ -0,0 +1,98 @@
use std::fmt;
use aws_lc_rs::signature::{Ed25519KeyPair, KeyPair};
const ED25519_PUBLIC_KEY_LENGTH: usize = 32;
const ED25519_SIGNATURE_LENGTH: usize = 64;
pub struct Ed25519SigningKey {
key_pair: Ed25519KeyPair,
public_key: [u8; ED25519_PUBLIC_KEY_LENGTH],
}
impl Ed25519SigningKey {
pub fn from_pkcs8(der: &[u8]) -> Result<Self, Ed25519Error> {
let key_pair = Ed25519KeyPair::from_pkcs8(der).map_err(|_| Ed25519Error)?;
let public_key = key_pair
.public_key()
.as_ref()
.try_into()
.map_err(|_| Ed25519Error)?;
Ok(Self {
key_pair,
public_key,
})
}
pub fn public_key(&self) -> &[u8; ED25519_PUBLIC_KEY_LENGTH] {
&self.public_key
}
pub fn sign(&self, message: &[u8]) -> Result<[u8; ED25519_SIGNATURE_LENGTH], Ed25519Error> {
self.key_pair
.try_sign(message)
.map_err(|_| Ed25519Error)?
.as_ref()
.try_into()
.map_err(|_| Ed25519Error)
}
}
impl fmt::Debug for Ed25519SigningKey {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Ed25519SigningKey")
.field("public_key", &self.public_key)
.finish_non_exhaustive()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Ed25519Error;
impl fmt::Display for Ed25519Error {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("invalid Ed25519 key or signing operation failed")
}
}
impl std::error::Error for Ed25519Error {}
#[cfg(test)]
mod tests {
use super::*;
const RFC_9421_PRIVATE_KEY_DER: &[u8] = &[
0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04,
0x20, 0x9f, 0x83, 0x62, 0xf8, 0x7a, 0x48, 0x4a, 0x95, 0x4e, 0x6e, 0x74, 0x0c, 0x5b, 0x4c,
0x0e, 0x84, 0x22, 0x91, 0x39, 0xa2, 0x0a, 0xa8, 0xab, 0x56, 0xff, 0x66, 0x58, 0x6f, 0x6a,
0x7d, 0x29, 0xc5,
];
#[test]
fn imports_pkcs8_and_signs_deterministically() {
let key = Ed25519SigningKey::from_pkcs8(RFC_9421_PRIVATE_KEY_DER).unwrap();
assert_eq!(
key.public_key(),
&[
0x26, 0xb4, 0x0b, 0x8f, 0x93, 0xff, 0xf3, 0xd8, 0x97, 0x11, 0x2f, 0x7e, 0xbc, 0x58,
0x2b, 0x23, 0x2d, 0xbd, 0x72, 0x51, 0x7d, 0x08, 0x2f, 0xe8, 0x3c, 0xfb, 0x30, 0xdd,
0xce, 0x43, 0xd1, 0xbb,
]
);
let first = key.sign(b"web bot auth").unwrap();
let second = key.sign(b"web bot auth").unwrap();
assert_eq!(first.len(), ED25519_SIGNATURE_LENGTH);
assert_eq!(first, second);
assert_ne!(first, key.sign(b"different message").unwrap());
}
#[test]
fn rejects_non_ed25519_pkcs8() {
assert_eq!(
Ed25519SigningKey::from_pkcs8(b"not a key").unwrap_err(),
Ed25519Error
);
}
}
moli-crypto/src/lib.rs
+2
View File
@@ -5,7 +5,9 @@
//! owner (`moli-webcrypto` or the renderer subsystem using the primitive).
mod digest;
mod ed25519;
mod random;
pub use digest::{DigestAlgorithm, Sha256Context, sha1_digest, sha256_digest, sha256_hex};
pub use ed25519::{Ed25519Error, Ed25519SigningKey};
pub use random::fill_secure_random;
moli-fetch/Cargo.toml
+1
View File
@@ -23,6 +23,7 @@ moli-http-cache = { path = "../moli-http-cache" }
moli-trace = { path = "../moli-trace" }
moli-url = { path = "../moli-url" }
moli-url-policy = { path = "../moli-url-policy" }
moli-web-bot-auth = { path = "../moli-web-bot-auth" }
moli-web-mime = { path = "../moli-web-mime" }
parking_lot = "0.12"
strum = { version = "0.28.0", features = ["derive"] }
moli-fetch/src/blocking/cache/key.rs
Vendored
+1
View File
@@ -19,6 +19,7 @@ pub(super) fn cache_store_and_key_for_request(
|| !request.method.eq_ignore_ascii_case("GET")
|| request.body.is_some()
|| request.auth().is_some()
|| config.web_bot_auth().is_some()
|| !config.default_request_headers().is_empty()
|| !subresource_validation_allows_http_cache(request)
|| cookie_header.is_some()
moli-fetch/src/blocking/mod.rs
+12 -16
View File
@@ -244,21 +244,12 @@ pub(crate) fn outgoing_request_headers_for_url(
outgoing
}
pub(crate) fn network_request_extra_info_for_url(
pub(crate) fn network_request_extra_info_from_headers(
config: &FetchConfig,
request: &Request,
request_url: &Url,
redirect_chain: &[RedirectInfo],
cookie_header: Option<&str>,
outgoing_headers: &[(String, String)],
cookie_report: Option<&StoredCookieQueryReport>,
) -> NetworkRequestExtraInfo {
let mut headers = outgoing_request_headers_for_url(
config,
request,
request_url,
redirect_chain,
cookie_header,
);
let mut headers = outgoing_headers.to_vec();
append_header_if_missing(&mut headers, "User-Agent", config.user_agent().to_owned());
NetworkRequestExtraInfo {
headers,
@@ -599,7 +590,7 @@ pub(crate) fn configure_easy<H: Handler>(
cookie_header: Option<&str>,
http_version: RequestHttpVersion,
validation_headers: Option<Vec<(String, String)>>,
) -> Result<()> {
) -> Result<Vec<(String, String)>> {
ensure_http_network_transport_url(request_url)?;
enforce_request_target_policy(config, request_url)?;
configure_curl_http_protocol_allowlist(easy)?;
@@ -712,17 +703,22 @@ pub(crate) fn configure_easy<H: Handler>(
}
let mut headers = List::new();
let outgoing_headers = outgoing_request_headers_for_url(
let mut outgoing_headers = outgoing_request_headers_for_url(
config,
request,
request_url,
redirect_chain,
cookie_header,
);
if let Some(web_bot_auth) = config.web_bot_auth() {
web_bot_auth
.append_request_headers(&mut outgoing_headers, &request.method, request_url)
.with_context(|| anyhow!("failed to sign web bot auth request for {request_url}"))?;
}
let mut has_headers = false;
let mut has_content_type_header = false;
for (name, value) in outgoing_headers {
for (name, value) in &outgoing_headers {
has_content_type_header |= name.eq_ignore_ascii_case("content-type");
let header_line = if value.is_empty() {
format!("{name}:")
@@ -797,7 +793,7 @@ pub(crate) fn configure_easy<H: Handler>(
}
}
Ok(())
Ok(outgoing_headers)
}
pub(crate) fn configure_openssl_tls_context(
moli-fetch/src/config.rs
+12
View File
@@ -3,6 +3,8 @@ use std::num::NonZeroU32;
use cidr::AnyIpCidr;
use moli_browser_profile::{BrowserIdentityProfile, DEFAULT_ACCEPT_LANGUAGE};
use crate::WebBotAuthSigner;
#[derive(Debug, Clone, PartialEq)]
pub struct FetchConfig {
browser_identity: BrowserIdentityProfile,
@@ -34,6 +36,7 @@ pub struct FetchConfig {
block_private_networks: bool,
block_cidrs: Vec<AnyIpCidr>,
tls_verify_host: bool,
web_bot_auth: Option<WebBotAuthSigner>,
}
impl FetchConfig {
@@ -144,6 +147,14 @@ impl FetchConfig {
self.tls_verify_host = tls_verify_host;
}
pub fn web_bot_auth(&self) -> Option<&WebBotAuthSigner> {
self.web_bot_auth.as_ref()
}
pub fn set_web_bot_auth(&mut self, web_bot_auth: Option<WebBotAuthSigner>) {
self.web_bot_auth = web_bot_auth;
}
pub fn set_http_proxy(&mut self, http_proxy: Option<String>) {
self.http_proxy = http_proxy;
}
@@ -274,6 +285,7 @@ impl Default for FetchConfig {
block_private_networks: false,
block_cidrs: Vec::new(),
tls_verify_host: true,
web_bot_auth: None,
}
}
}
moli-fetch/src/lib.rs
+1
View File
@@ -45,6 +45,7 @@ pub use headers::{
is_forbidden_response_header_name, is_no_cors_safelisted_request_header,
};
pub use moli_cookie_jar::SharedBrowserCookieStore as SharedCookieStore;
pub use moli_web_bot_auth::{WebBotAuthProfile, WebBotAuthSigner};
pub use network_fetch_result::{
NetworkExchangeObservation, NetworkFetchFailure, NetworkFetchFailureRequestContext,
NetworkFetchResult, NetworkObservationJournal, NetworkRequestObservation,
moli-fetch/src/runtime.rs
+89 -98
View File
@@ -45,7 +45,7 @@ use crate::{
cookie_access_report_for_request, cookie_header_from_report,
finish_streaming_cached_response, load_cached_streaming_response_lookup,
log_request_completion, merge_cached_not_modified_streaming_response_lookup,
network_request_extra_info_for_url, next_followed_redirect_url_from_parts,
network_request_extra_info_from_headers, next_followed_redirect_url_from_parts,
remove_cached_response, response_headers_forbid_cache_storage, store_response_cookies,
transfer_metrics_from_easy, validation_headers_for_cached_streaming_response_lookup,
},
@@ -863,22 +863,6 @@ impl RuntimeOwner {
&job.request,
&job.current_url,
);
let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
network_request_extra_info_for_url(
&self.config,
&prepared_request.request,
&job.current_url,
&job.redirect_chain,
cookie_header.as_deref(),
request_cookie_report.as_ref(),
)
});
attach_next_request_extra_info(
&mut job.redirect_chain,
request_cookie_report.clone(),
request_extra_info.as_ref(),
);
let mut easy = Easy2::new(FetchTransferHandler::new_buffered(ResponseCollector::new(
Some(job.cancel_handle.clone()),
)));
@@ -894,7 +878,7 @@ impl RuntimeOwner {
) {
return Err((job.response_tx, error));
}
if let Err(error) = configure_easy(
let outgoing_headers = match configure_easy(
&mut easy,
&self.config,
&prepared_request.request,
@@ -909,8 +893,21 @@ impl RuntimeOwner {
)
.with_context(|| anyhow!("failed to configure curl request for {}", job.current_url))
{
return Err((job.response_tx, error));
}
Ok(headers) => headers,
Err(error) => return Err((job.response_tx, error)),
};
let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
network_request_extra_info_from_headers(
&self.config,
&outgoing_headers,
request_cookie_report.as_ref(),
)
});
attach_next_request_extra_info(
&mut job.redirect_chain,
request_cookie_report.clone(),
request_extra_info.as_ref(),
);
let label = job.current_url.to_string();
let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
@@ -995,16 +992,6 @@ impl RuntimeOwner {
&job.request,
&job.current_url,
);
let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
network_request_extra_info_for_url(
&self.config,
&prepared_request.request,
&job.current_url,
&job.redirect_chain,
cookie_header.as_deref(),
request_cookie_report.as_ref(),
)
});
match load_cached_streaming_response_lookup(
&self.config,
&prepared_request.request,
@@ -1033,12 +1020,6 @@ impl RuntimeOwner {
Err(error) => return Err((Box::new(job), None, error)),
}
attach_next_request_extra_info(
&mut job.redirect_chain,
request_cookie_report.clone(),
request_extra_info.as_ref(),
);
let mut easy = job.easy.take().unwrap_or_else(|| {
Easy2::new(FetchTransferHandler::new_streaming(
StreamingResponseCollector::new(
@@ -1061,6 +1042,41 @@ impl RuntimeOwner {
cookie_header.clone(),
));
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
request_cookie_report.as_ref(),
self.config.http_proxy().is_some() && job.current_url.scheme() == "https",
) {
return Err((Box::new(job), Some(easy), error));
}
let outgoing_headers = match configure_easy(
&mut easy,
&self.config,
&prepared_request.request,
&job.current_url,
&job.redirect_chain,
cookie_header.as_deref(),
job.http_version,
None,
)
.with_context(|| anyhow!("failed to configure curl request for {}", job.current_url))
{
Ok(headers) => headers,
Err(error) => return Err((Box::new(job), Some(easy), error)),
};
let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
network_request_extra_info_from_headers(
&self.config,
&outgoing_headers,
request_cookie_report.as_ref(),
)
});
attach_next_request_extra_info(
&mut job.redirect_chain,
request_cookie_report.clone(),
request_extra_info.as_ref(),
);
let collector = easy
.get_mut()
.streaming_mut()
@@ -1076,28 +1092,6 @@ impl RuntimeOwner {
cache_plan,
);
collector.set_client_hint_response_policy(prepared_request.response_policy);
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
request_cookie_report.as_ref(),
self.config.http_proxy().is_some() && job.current_url.scheme() == "https",
) {
return Err((Box::new(job), Some(easy), error));
}
if let Err(error) = configure_easy(
&mut easy,
&self.config,
&prepared_request.request,
&job.current_url,
&job.redirect_chain,
cookie_header.as_deref(),
job.http_version,
None,
)
.with_context(|| anyhow!("failed to configure curl request for {}", job.current_url))
{
return Err((Box::new(job), Some(easy), error));
}
let label = job.current_url.to_string();
let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
@@ -1189,16 +1183,6 @@ impl RuntimeOwner {
&job.request,
&job.current_url,
);
let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
network_request_extra_info_for_url(
&self.config,
&prepared_request.request,
&job.current_url,
&job.redirect_chain,
cookie_header.as_deref(),
request_cookie_report.as_ref(),
)
});
let mut stale_cached_lookup = None;
match load_cached_streaming_response_lookup(
&self.config,
@@ -1231,12 +1215,6 @@ impl RuntimeOwner {
Err(error) => return Err((Box::new(job), None, error)),
}
attach_next_request_extra_info(
&mut job.redirect_chain,
request_cookie_report.clone(),
request_extra_info.as_ref(),
);
let mut easy = job.easy.take().unwrap_or_else(|| {
Easy2::new(FetchTransferHandler::new_raw_streaming(
RawStreamingResponseCollector::new(
@@ -1258,6 +1236,43 @@ impl RuntimeOwner {
job.current_url.clone(),
cookie_header.clone(),
));
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
request_cookie_report.as_ref(),
self.config.http_proxy().is_some() && job.current_url.scheme() == "https",
) {
return Err((Box::new(job), Some(easy), error));
}
let outgoing_headers = match configure_easy(
&mut easy,
&self.config,
&prepared_request.request,
&job.current_url,
&job.redirect_chain,
cookie_header.as_deref(),
job.http_version,
stale_cached_lookup
.as_ref()
.map(validation_headers_for_cached_streaming_response_lookup),
)
.with_context(|| anyhow!("failed to configure curl request for {}", job.current_url))
{
Ok(headers) => headers,
Err(error) => return Err((Box::new(job), Some(easy), error)),
};
let request_extra_info = job.request.is_top_level_navigation_request().then(|| {
network_request_extra_info_from_headers(
&self.config,
&outgoing_headers,
request_cookie_report.as_ref(),
)
});
attach_next_request_extra_info(
&mut job.redirect_chain,
request_cookie_report.clone(),
request_extra_info.as_ref(),
);
let collector = easy
.get_mut()
.raw_streaming_mut()
@@ -1274,30 +1289,6 @@ impl RuntimeOwner {
stale_cached_lookup.is_some(),
);
collector.set_client_hint_response_policy(prepared_request.response_policy);
if let Err(error) = configure_network_observation(
&mut easy,
&job.request,
request_cookie_report.as_ref(),
self.config.http_proxy().is_some() && job.current_url.scheme() == "https",
) {
return Err((Box::new(job), Some(easy), error));
}
if let Err(error) = configure_easy(
&mut easy,
&self.config,
&prepared_request.request,
&job.current_url,
&job.redirect_chain,
cookie_header.as_deref(),
job.http_version,
stale_cached_lookup
.as_ref()
.map(validation_headers_for_cached_streaming_response_lookup),
)
.with_context(|| anyhow!("failed to configure curl request for {}", job.current_url))
{
return Err((Box::new(job), Some(easy), error));
}
let label = job.current_url.to_string();
let dns_resolution = curl_dns_resolution(&self.config, &job.current_url);
moli-fetch/src/tests/mod.rs
+210 -1
View File
@@ -8,6 +8,7 @@ use moli_browser_profile::DEFAULT_ACCEPT_LANGUAGE;
use moli_cookie_jar::{NetworkCookieRequestContext, new_shared_browser_cookie_store};
use moli_http_cache::{HttpCacheEntryMetadata, HttpCacheStore};
use std::{
collections::BTreeSet,
fs,
io::Read,
num::NonZeroU32,
@@ -28,7 +29,8 @@ use crate::{
RequestAuthScheme, RequestAuthTarget, RequestCacheMode, RequestCredentialsMode, RequestMode,
RequestRedirectMode, RequestResourceType, Response, ResponseBody, ResponseHead,
ScriptFetchRequestMetadata, ScriptFetchSchedulerPriority, StreamingResponseCollector,
SubresourceRequestMetadata, http_cache_stats, runtime::FetchRuntimeOwner,
SubresourceRequestMetadata, WebBotAuthProfile, WebBotAuthSigner, http_cache_stats,
runtime::FetchRuntimeOwner,
};
use self::support::{
@@ -39,6 +41,19 @@ use self::support::{
const ENV_PROXY_CHILD_TEST: &str = "MOLI_FETCH_ENV_PROXY_CHILD";
const ENV_PROXY_URL: &str = "MOLI_FETCH_ENV_PROXY_URL";
const TEST_HIGH_ENTROPY_CLIENT_HINTS: &str = "Sec-CH-UA-Full-Version, Sec-CH-UA-Full-Version-List, Sec-CH-UA-Arch, Sec-CH-UA-Bitness, Sec-CH-UA-Platform-Version, Sec-CH-UA-Model, Sec-CH-UA-WoW64";
const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\
MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\
-----END PRIVATE KEY-----\n";
fn test_web_bot_auth_signer() -> WebBotAuthSigner {
WebBotAuthSigner::from_pem(
RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
"bot.example",
Some("poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U"),
WebBotAuthProfile::Cloudflare,
)
.unwrap()
}
fn sample_response_head() -> ResponseHead {
ResponseHead {
@@ -1191,6 +1206,172 @@ async fn fetch_raw_stream_treats_https11_switching_protocols_as_final_response()
Ok(())
}
#[test]
fn web_bot_auth_resigns_restarts_redirects_and_subresources() -> Result<()> {
let server = ScriptedHttps11Server::spawn(vec![
ScriptedResponse::status(403, "Challenge")
.with_header("Accept-CH", "Sec-CH-UA-Arch")
.with_header("Critical-CH", "Sec-CH-UA-Arch"),
ScriptedResponse::status(302, "Found").with_header("Location", "/final"),
ScriptedResponse::ok("final"),
ScriptedResponse::ok("asset"),
]);
let mut config = FetchConfig::default();
config.set_tls_verify_host(false);
config.set_web_bot_auth(Some(test_web_bot_auth_signer()));
let client = FetchClient::new(&config, new_shared_browser_cookie_store());
let response = fetch_response_for_test(&client, Request::get(&server.url_path("/start"))?)?;
assert_eq!(response.body_text(), "final");
assert_eq!(response.redirect_chain.len(), 2);
assert_eq!(response.redirect_chain[0].status, 307);
assert_eq!(response.redirect_chain[1].status, 302);
let final_url = Url::parse(&server.url_path("/final"))?;
let subresource = Request::new(
"POST",
&server.url_path("/asset"),
Some("probe".to_owned()),
Vec::new(),
)?
.with_browser_request_metadata(BrowserRequestMetadata::Fetch)
.with_initiator_url(&final_url);
assert_eq!(
fetch_response_for_test(&client, subresource)?.body_text(),
"asset"
);
assert_eq!(
server.requests(),
[
"/start".to_owned(),
"/start".to_owned(),
"/final".to_owned(),
"/asset".to_owned(),
]
);
let request_heads = server.request_heads();
assert_eq!(request_heads.len(), 4);
for request_head in &request_heads {
assert_eq!(
request_head_header_value(request_head, "Signature-Agent"),
Some("\"https://bot.example\"")
);
let signature_input = request_head_header_value(request_head, "Signature-Input")
.expect("signed request should include Signature-Input");
assert!(
signature_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")")
);
assert!(request_head_header_value(request_head, "Signature").is_some());
}
assert!(request_heads[3].starts_with("POST /asset HTTP/1.1\r\n"));
let nonces = request_heads
.iter()
.map(|request_head| {
nonce_from_signature_input(
request_head_header_value(request_head, "Signature-Input").unwrap(),
)
.to_owned()
})
.collect::<BTreeSet<_>>();
assert_eq!(nonces.len(), request_heads.len());
let restart = &response.redirect_chain[0];
assert_request_extra_signature_matches_wire(
&restart
.response_extra_info
.as_ref()
.expect("restart response extra info")
.request_extra_info,
&request_heads[0],
);
assert_request_extra_signature_matches_wire(
restart
.request_extra_info
.as_ref()
.expect("restarted request extra info"),
&request_heads[1],
);
let redirect = &response.redirect_chain[1];
assert_request_extra_signature_matches_wire(
&redirect
.response_extra_info
.as_ref()
.expect("redirect response extra info")
.request_extra_info,
&request_heads[1],
);
assert_request_extra_signature_matches_wire(
redirect
.request_extra_info
.as_ref()
.expect("redirect target request extra info"),
&request_heads[2],
);
assert_request_extra_signature_matches_wire(
response
.network_request_extra_info()
.expect("final request extra info"),
&request_heads[2],
);
server.shutdown();
Ok(())
}
#[test]
fn web_bot_auth_is_not_sent_over_plain_http() -> Result<()> {
let server = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("plain")]);
let mut config = FetchConfig::default();
config.set_web_bot_auth(Some(test_web_bot_auth_signer()));
let response = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
assert_eq!(response.body_text(), "plain");
let requests = server.requests();
assert_eq!(requests.len(), 1);
for name in ["Signature-Agent", "Signature-Input", "Signature"] {
assert_eq!(request_head_header_value(&requests[0], name), None);
}
server.shutdown();
Ok(())
}
#[test]
fn web_bot_auth_bypasses_shared_http_cache() -> Result<()> {
let cache_dir = unique_test_cache_dir();
let server = ScriptedHttps11Server::spawn(vec![
ScriptedResponse::ok("first").with_header("Cache-Control", "max-age=60"),
ScriptedResponse::ok("second").with_header("Cache-Control", "max-age=60"),
]);
let mut config = FetchConfig::default();
config.set_tls_verify_host(false);
config.set_http_cache_dir(Some(cache_dir.display().to_string()));
config.set_web_bot_auth(Some(test_web_bot_auth_signer()));
let first = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
let second = fetch_with_config_for_test(&config, Request::get(&server.url())?)?;
assert_eq!(first.body_text(), "first");
assert_eq!(second.body_text(), "second");
assert!(!first.from_cache);
assert!(!second.from_cache);
assert_eq!(server.hits(), 2);
let request_heads = server.request_heads();
assert_ne!(
request_head_header_value(&request_heads[0], "Signature-Input"),
request_head_header_value(&request_heads[1], "Signature-Input")
);
assert!(
!cache_dir.exists() || fs::read_dir(&cache_dir)?.next().is_none(),
"authenticated responses must not enter the shared disk cache"
);
server.shutdown();
let _ = fs::remove_dir_all(cache_dir);
Ok(())
}
#[tokio::test]
async fn fetch_raw_stream_finishes_switching_protocols_body_without_connection_close() -> Result<()>
{
@@ -3616,6 +3797,34 @@ fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a s
.map(|(_, value)| value.as_str())
}
fn request_head_header_value<'a>(request_head: &'a str, name: &str) -> Option<&'a str> {
request_head.lines().skip(1).find_map(|line| {
let (header_name, value) = line.split_once(':')?;
header_name.eq_ignore_ascii_case(name).then(|| value.trim())
})
}
fn nonce_from_signature_input(signature_input: &str) -> &str {
signature_input
.split(";nonce=\"")
.nth(1)
.and_then(|value| value.split('"').next())
.expect("Signature-Input should contain a nonce parameter")
}
fn assert_request_extra_signature_matches_wire(
extra_info: &crate::NetworkRequestExtraInfo,
request_head: &str,
) {
for name in ["Signature-Agent", "Signature-Input", "Signature"] {
assert_eq!(
header_value(&extra_info.headers, name),
request_head_header_value(request_head, name),
"network extra info and wire request differ for {name}"
);
}
}
#[test]
fn fetch_client_cache_respects_pragma_no_cache_response() {
let cache_dir = unique_test_cache_dir();
moli-fetch/src/tests/support.rs
+19
View File
@@ -43,6 +43,7 @@ pub(super) struct ScriptedHttps11Server {
addr: std::net::SocketAddr,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
request_heads: Arc<Mutex<Vec<String>>>,
shutdown_tx: std_mpsc::Sender<()>,
join_handle: Option<thread::JoinHandle<()>>,
}
@@ -56,16 +57,19 @@ impl ScriptedHttps11Server {
let addr = listener.local_addr().unwrap();
let hits = Arc::new(AtomicUsize::new(0));
let requests = Arc::new(Mutex::new(Vec::new()));
let request_heads = Arc::new(Mutex::new(Vec::new()));
let (shutdown_tx, shutdown_rx) = std_mpsc::channel();
let responses = Arc::new(Mutex::new(VecDeque::from(responses)));
let hits_for_thread = Arc::clone(&hits);
let requests_for_thread = Arc::clone(&requests);
let request_heads_for_thread = Arc::clone(&request_heads);
let join_handle = thread::spawn(move || {
run_https11_server(
listener,
shutdown_rx,
hits_for_thread,
requests_for_thread,
request_heads_for_thread,
responses,
);
});
@@ -74,6 +78,7 @@ impl ScriptedHttps11Server {
addr,
hits,
requests,
request_heads,
shutdown_tx,
join_handle: Some(join_handle),
}
@@ -83,6 +88,11 @@ impl ScriptedHttps11Server {
format!("https://{}/cache", self.addr)
}
pub(super) fn url_path(&self, path: &str) -> String {
let path = path.strip_prefix('/').unwrap_or(path);
format!("https://{}/{}", self.addr, path)
}
pub(super) fn hits(&self) -> usize {
self.hits.load(Ordering::SeqCst)
}
@@ -91,6 +101,10 @@ impl ScriptedHttps11Server {
self.requests.lock().clone()
}
pub(super) fn request_heads(&self) -> Vec<String> {
self.request_heads.lock().clone()
}
pub(super) fn shutdown(mut self) {
let _ = self.shutdown_tx.send(());
if let Some(join_handle) = self.join_handle.take() {
@@ -710,6 +724,7 @@ fn run_https11_server(
shutdown_rx: std_mpsc::Receiver<()>,
hits: Arc<AtomicUsize>,
requests: Arc<Mutex<Vec<String>>>,
request_heads: Arc<Mutex<Vec<String>>>,
responses: Arc<Mutex<VecDeque<ScriptedResponse>>>,
) {
let rt = tokio::runtime::Builder::new_current_thread()
@@ -731,6 +746,7 @@ fn run_https11_server(
let tls_acceptor = tls_acceptor.clone();
let hits = Arc::clone(&hits);
let requests = Arc::clone(&requests);
let request_heads = Arc::clone(&request_heads);
let responses = Arc::clone(&responses);
tokio::spawn(async move {
let Ok(mut stream) = tls_acceptor.accept(stream).await else {
@@ -751,6 +767,9 @@ fn run_https11_server(
if let Some(path) = request_path_from_head(&request) {
requests.lock().push(path);
}
request_heads
.lock()
.push(String::from_utf8_lossy(&request).into_owned());
let _ = hits.fetch_add(1, Ordering::SeqCst) + 1;
let response_spec = {
let mut responses = responses.lock();
moli-web-bot-auth/Cargo.toml
+18
View File
@@ -0,0 +1,18 @@
[package]
license.workspace = true
name = "moli-web-bot-auth"
version = "0.1.0"
edition = "2024"
description = "Web Bot Auth HTTP message signatures for Moli"
[dependencies]
anyhow = "1.0.100"
base64 = "0.22"
moli-crypto = { path = "../moli-crypto" }
url = "2.5.7"
[dev-dependencies]
aws-lc-rs = { version = "1.16.2", default-features = false, features = ["aws-lc-sys"] }
[lints]
workspace = true
moli-web-bot-auth/src/key.rs
+133
View File
@@ -0,0 +1,133 @@
use std::str;
use anyhow::{Context, Result, anyhow, bail};
use base64::{Engine as _, engine::general_purpose};
use moli_crypto::{Ed25519SigningKey, sha256_digest};
pub(crate) struct WebBotAuthKey {
signing_key: Ed25519SigningKey,
public_key: [u8; 32],
keyid: String,
}
impl WebBotAuthKey {
pub(crate) fn from_pem(private_key_pem: &[u8], expected_keyid: Option<&str>) -> Result<Self> {
let private_key_der = decode_pkcs8_private_key_pem(private_key_pem)?;
let signing_key = Ed25519SigningKey::from_pkcs8(&private_key_der).map_err(|_| {
anyhow!("web bot auth key must be an unencrypted PKCS#8 Ed25519 private key")
})?;
let public_key = *signing_key.public_key();
let keyid = jwk_thumbprint(&public_key);
if let Some(expected_keyid) = expected_keyid
&& expected_keyid != keyid
{
bail!(
"--web-bot-auth-keyid does not match the private key; expected `{expected_keyid}`, computed `{keyid}`"
);
}
Ok(Self {
signing_key,
public_key,
keyid,
})
}
pub(crate) fn keyid(&self) -> &str {
&self.keyid
}
pub(crate) fn public_key(&self) -> &[u8; 32] {
&self.public_key
}
pub(crate) fn sign(&self, message: &[u8]) -> Result<[u8; 64]> {
self.signing_key
.sign(message)
.context("failed to create web bot auth Ed25519 signature")
}
}
fn decode_pkcs8_private_key_pem(private_key_pem: &[u8]) -> Result<Vec<u8>> {
const BEGIN: &str = "-----BEGIN PRIVATE KEY-----";
const END: &str = "-----END PRIVATE KEY-----";
let pem = str::from_utf8(private_key_pem)
.context("web bot auth private key PEM is not valid UTF-8")?;
if pem.contains("-----BEGIN ENCRYPTED PRIVATE KEY-----") {
bail!("encrypted web bot auth private keys are not supported");
}
let pem = pem.trim();
let body = pem
.strip_prefix(BEGIN)
.and_then(|pem| pem.strip_suffix(END))
.ok_or_else(|| {
anyhow!("web bot auth key must use BEGIN PRIVATE KEY PKCS#8 PEM encoding")
})?;
if body.contains("-----") {
bail!("web bot auth key PEM must contain exactly one private key");
}
let encoded = body
.chars()
.filter(|character| !character.is_ascii_whitespace())
.collect::<String>();
if encoded.is_empty() {
bail!("web bot auth private key PEM body is empty");
}
general_purpose::STANDARD
.decode(encoded)
.context("web bot auth private key PEM contains invalid base64")
}
fn jwk_thumbprint(public_key: &[u8; 32]) -> String {
let x = general_purpose::URL_SAFE_NO_PAD.encode(public_key);
let canonical_jwk = format!(r#"{{"crv":"Ed25519","kty":"OKP","x":"{x}"}}"#);
general_purpose::URL_SAFE_NO_PAD.encode(sha256_digest(canonical_jwk))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::{EXPECTED_KEYID, RFC_9421_ED25519_PRIVATE_KEY};
#[test]
fn derives_rfc_jwk_thumbprint_and_public_key() {
let key = WebBotAuthKey::from_pem(
RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
Some(EXPECTED_KEYID),
)
.unwrap();
assert_eq!(key.keyid(), EXPECTED_KEYID);
assert_eq!(
general_purpose::URL_SAFE_NO_PAD.encode(key.public_key()),
"JrQLj5P_89iXES9-vFgrIy29clF9CC_oPPsw3c5D0bs"
);
}
#[test]
fn rejects_mismatched_keyid_and_invalid_pem() {
let wrong_keyid =
WebBotAuthKey::from_pem(RFC_9421_ED25519_PRIVATE_KEY.as_bytes(), Some("wrong"))
.err()
.expect("mismatched keyid should fail")
.to_string();
assert!(wrong_keyid.contains("does not match"));
assert!(wrong_keyid.contains(EXPECTED_KEYID));
let invalid_pem = WebBotAuthKey::from_pem(b"not a private key", None)
.err()
.expect("invalid PEM should fail")
.to_string();
assert!(invalid_pem.contains("BEGIN PRIVATE KEY"));
let encrypted = WebBotAuthKey::from_pem(
b"-----BEGIN ENCRYPTED PRIVATE KEY-----\nAA==\n-----END ENCRYPTED PRIVATE KEY-----",
None,
)
.err()
.expect("encrypted PEM should fail")
.to_string();
assert!(encrypted.contains("encrypted"));
}
}
moli-web-bot-auth/src/lib.rs
+12
View File
@@ -0,0 +1,12 @@
//! Web Bot Auth request signing built on RFC 9421 HTTP message signatures.
mod key;
mod profile;
mod signer;
mod wire;
#[cfg(test)]
mod test_support;
pub use profile::WebBotAuthProfile;
pub use signer::WebBotAuthSigner;
moli-web-bot-auth/src/profile.rs
+15
View File
@@ -0,0 +1,15 @@
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum WebBotAuthProfile {
#[default]
Cloudflare,
IetfDraft01,
}
impl WebBotAuthProfile {
pub const fn as_str(self) -> &'static str {
match self {
Self::Cloudflare => "cloudflare",
Self::IetfDraft01 => "ietf-01",
}
}
}
moli-web-bot-auth/src/signer.rs
+291
View File
@@ -0,0 +1,291 @@
use std::{
fmt,
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
time::{SystemTime, UNIX_EPOCH},
};
use anyhow::{Context, Result, anyhow, bail};
use base64::{Engine as _, engine::general_purpose};
use moli_crypto::{DigestAlgorithm, fill_secure_random};
use url::Url;
use crate::{
key::WebBotAuthKey,
profile::WebBotAuthProfile,
wire::{SignatureOptions, sign_request},
};
const SIGNATURE_LABEL: &str = "sig1";
const SIGNATURE_TTL_SECS: u64 = 60;
const NONCE_BYTES: usize = 64;
#[derive(Clone)]
pub struct WebBotAuthSigner {
inner: Arc<WebBotAuthSignerInner>,
}
struct WebBotAuthSignerInner {
key: WebBotAuthKey,
signature_agent_origin: String,
profile: WebBotAuthProfile,
nonce_seed: [u8; 32],
nonce_counter: AtomicU64,
}
impl WebBotAuthSigner {
pub fn from_pem(
private_key_pem: &[u8],
domain: &str,
expected_keyid: Option<&str>,
profile: WebBotAuthProfile,
) -> Result<Self> {
let key = WebBotAuthKey::from_pem(private_key_pem, expected_keyid)?;
let signature_agent_origin = normalize_signature_agent_origin(domain)?;
let mut nonce_seed = [0_u8; 32];
fill_secure_random(&mut nonce_seed)
.map_err(|_| anyhow!("failed to initialize web bot auth nonce generation"))?;
Ok(Self {
inner: Arc::new(WebBotAuthSignerInner {
key,
signature_agent_origin,
profile,
nonce_seed,
nonce_counter: AtomicU64::new(0),
}),
})
}
pub fn keyid(&self) -> &str {
self.inner.key.keyid()
}
pub fn public_key(&self) -> &[u8; 32] {
self.inner.key.public_key()
}
pub fn signature_agent_origin(&self) -> &str {
&self.inner.signature_agent_origin
}
pub fn profile(&self) -> WebBotAuthProfile {
self.inner.profile
}
pub fn append_request_headers(
&self,
headers: &mut Vec<(String, String)>,
method: &str,
request_url: &Url,
) -> Result<()> {
if request_url.scheme() != "https" {
return Ok(());
}
headers.retain(|(name, _)| !is_web_bot_auth_header(name));
let created = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let signed = sign_request(
&self.inner.key,
&self.inner.signature_agent_origin,
self.inner.profile,
method,
request_url,
&SignatureOptions {
signature_label: SIGNATURE_LABEL,
signature_agent_label: SIGNATURE_LABEL,
created,
expires: created.saturating_add(SIGNATURE_TTL_SECS),
nonce: self.next_nonce(),
cover_request_target: true,
},
)?;
headers.extend(signed.headers.into_pairs());
Ok(())
}
fn next_nonce(&self) -> String {
let counter = self.inner.nonce_counter.fetch_add(1, Ordering::Relaxed);
let mut material = Vec::with_capacity(32 + 8 + 24);
material.extend_from_slice(b"moli-web-bot-auth-nonce\0");
material.extend_from_slice(&self.inner.nonce_seed);
material.extend_from_slice(&counter.to_be_bytes());
let nonce = DigestAlgorithm::Sha512.digest_bytes(&material);
debug_assert_eq!(nonce.len(), NONCE_BYTES);
general_purpose::STANDARD.encode(nonce)
}
}
impl fmt::Debug for WebBotAuthSigner {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("WebBotAuthSigner")
.field("keyid", &self.inner.key.keyid())
.field("signature_agent_origin", &self.inner.signature_agent_origin)
.field("profile", &self.inner.profile)
.finish_non_exhaustive()
}
}
impl PartialEq for WebBotAuthSigner {
fn eq(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.inner, &other.inner)
|| (self.inner.key.public_key() == other.inner.key.public_key()
&& self.inner.signature_agent_origin == other.inner.signature_agent_origin
&& self.inner.profile == other.inner.profile)
}
}
impl Eq for WebBotAuthSigner {}
fn normalize_signature_agent_origin(domain: &str) -> Result<String> {
if domain.is_empty() || domain.trim() != domain {
bail!("--web-bot-auth-domain must be a non-empty domain without surrounding whitespace");
}
if domain.contains(['/', '?', '#', '@']) {
bail!(
"--web-bot-auth-domain must contain only a host and optional port, without a scheme, credentials, path, query, or fragment"
);
}
let url = Url::parse(&format!("https://{domain}"))
.context("failed to parse --web-bot-auth-domain as an HTTPS origin")?;
if url.host().is_none()
|| !url.username().is_empty()
|| url.password().is_some()
|| url.path() != "/"
|| url.query().is_some()
|| url.fragment().is_some()
{
bail!("--web-bot-auth-domain must identify a single HTTPS origin");
}
Ok(url.origin().ascii_serialization())
}
fn is_web_bot_auth_header(name: &str) -> bool {
name.eq_ignore_ascii_case("signature-agent")
|| name.eq_ignore_ascii_case("signature-input")
|| name.eq_ignore_ascii_case("signature")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::{EXPECTED_KEYID, RFC_9421_ED25519_PRIVATE_KEY, signer};
#[test]
fn normalizes_signature_agent_origin() {
let signer = signer(WebBotAuthProfile::Cloudflare);
assert_eq!(signer.keyid(), EXPECTED_KEYID);
assert_eq!(
signer.signature_agent_origin(),
"https://signature-agent.test"
);
}
#[test]
fn production_requests_use_unique_nonces() {
let signer = signer(WebBotAuthProfile::Cloudflare);
let request_url = Url::parse("https://example.com/path").unwrap();
let mut first_headers = Vec::new();
signer
.append_request_headers(&mut first_headers, "post", &request_url)
.unwrap();
let mut second_headers = Vec::new();
signer
.append_request_headers(&mut second_headers, "post", &request_url)
.unwrap();
let first_input = header_value(&first_headers, "signature-input");
assert!(first_input.contains("(\"@authority\" \"@method\" \"@path\" \"signature-agent\")"));
assert!(first_input.contains(";expires="));
assert!(first_input.contains(";tag=\"web-bot-auth\""));
assert_ne!(
nonce_from_signature_input(first_input),
nonce_from_signature_input(header_value(&second_headers, "signature-input"))
);
}
#[test]
fn omits_signatures_on_insecure_requests_and_replaces_spoofed_https_headers() {
let signer = signer(WebBotAuthProfile::Cloudflare);
let spoofed = vec![
("Signature-Agent".to_owned(), "spoofed".to_owned()),
("Signature-Input".to_owned(), "spoofed".to_owned()),
("Signature".to_owned(), "spoofed".to_owned()),
];
let mut http_headers = spoofed.clone();
signer
.append_request_headers(
&mut http_headers,
"GET",
&Url::parse("http://example.com/").unwrap(),
)
.unwrap();
assert_eq!(http_headers, spoofed);
let mut https_headers = spoofed;
signer
.append_request_headers(
&mut https_headers,
"GET",
&Url::parse("https://example.com/").unwrap(),
)
.unwrap();
assert_eq!(https_headers.len(), 3);
assert!(https_headers.iter().all(|(_, value)| value != "spoofed"));
}
#[test]
fn validates_domain() {
for domain in [
"",
" example.com",
"https://example.com",
"user@example.com",
"example.com/path",
"example.com?query",
] {
let error = WebBotAuthSigner::from_pem(
RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
domain,
None,
WebBotAuthProfile::Cloudflare,
)
.unwrap_err();
assert!(error.to_string().contains("--web-bot-auth-domain"));
}
}
#[test]
fn debug_output_never_contains_private_key_material() {
let signer = signer(WebBotAuthProfile::Cloudflare);
let rendered = format!("{signer:?}");
assert!(rendered.contains(EXPECTED_KEYID));
assert!(!rendered.contains("IJ+DYvh6"));
assert!(!rendered.contains("PRIVATE KEY"));
}
fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
headers
.iter()
.find(|(candidate, _)| candidate.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
.unwrap()
}
fn nonce_from_signature_input(input: &str) -> &str {
input
.split(";nonce=\"")
.nth(1)
.and_then(|value| value.split('"').next())
.unwrap()
}
}
moli-web-bot-auth/src/test_support.rs
+24
View File
@@ -0,0 +1,24 @@
use crate::{WebBotAuthProfile, WebBotAuthSigner, key::WebBotAuthKey};
pub(crate) const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\
MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\
-----END PRIVATE KEY-----\n";
pub(crate) const EXPECTED_KEYID: &str = "poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U";
pub(crate) fn key() -> WebBotAuthKey {
WebBotAuthKey::from_pem(
RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
Some(EXPECTED_KEYID),
)
.unwrap()
}
pub(crate) fn signer(profile: WebBotAuthProfile) -> WebBotAuthSigner {
WebBotAuthSigner::from_pem(
RFC_9421_ED25519_PRIVATE_KEY.as_bytes(),
"signature-agent.test",
Some(EXPECTED_KEYID),
profile,
)
.unwrap()
}
moli-web-bot-auth/src/wire.rs
+279
View File
@@ -0,0 +1,279 @@
use anyhow::{Context, Result};
use base64::{Engine as _, engine::general_purpose};
use url::{Host, Url};
use crate::{key::WebBotAuthKey, profile::WebBotAuthProfile};
pub(crate) struct SignatureOptions<'a> {
pub(crate) signature_label: &'a str,
pub(crate) signature_agent_label: &'a str,
pub(crate) created: u64,
pub(crate) expires: u64,
pub(crate) nonce: String,
pub(crate) cover_request_target: bool,
}
pub(crate) struct SignedRequest {
pub(crate) headers: WebBotAuthHeaders,
#[cfg(test)]
pub(crate) signature_base: String,
}
pub(crate) struct WebBotAuthHeaders {
pub(crate) signature_agent: String,
pub(crate) signature_input: String,
pub(crate) signature: String,
}
impl WebBotAuthHeaders {
pub(crate) fn into_pairs(self) -> [(String, String); 3] {
[
("Signature-Agent".to_owned(), self.signature_agent),
("Signature-Input".to_owned(), self.signature_input),
("Signature".to_owned(), self.signature),
]
}
}
pub(crate) fn sign_request(
key: &WebBotAuthKey,
signature_agent_origin: &str,
profile: WebBotAuthProfile,
method: &str,
request_url: &Url,
options: &SignatureOptions<'_>,
) -> Result<SignedRequest> {
let authority = request_authority(request_url)?;
let method = method.to_ascii_uppercase();
let path = request_url.path();
let signature_agent = signature_agent_header(
profile,
signature_agent_origin,
options.signature_agent_label,
);
let signature_agent_component =
signature_agent_component_identifier(profile, options.signature_agent_label);
let mut covered_components = vec!["\"@authority\"".to_owned()];
if options.cover_request_target {
covered_components.push("\"@method\"".to_owned());
covered_components.push("\"@path\"".to_owned());
}
covered_components.push(signature_agent_component.clone());
let signature_params = format!(
"({});created={};keyid=\"{}\";alg=\"ed25519\";expires={};nonce=\"{}\";tag=\"web-bot-auth\"",
covered_components.join(" "),
options.created,
key.keyid(),
options.expires,
options.nonce,
);
let mut signature_base_lines = vec![format!("\"@authority\": {authority}")];
if options.cover_request_target {
signature_base_lines.push(format!("\"@method\": {method}"));
signature_base_lines.push(format!("\"@path\": {path}"));
}
signature_base_lines.push(format!(
"{signature_agent_component}: \"{signature_agent_origin}\""
));
signature_base_lines.push(format!("\"@signature-params\": {signature_params}"));
let signature_base = signature_base_lines.join("\n");
let signature = key.sign(signature_base.as_bytes())?;
Ok(SignedRequest {
headers: WebBotAuthHeaders {
signature_agent,
signature_input: format!("{}={signature_params}", options.signature_label),
signature: format!(
"{}=:{}:",
options.signature_label,
general_purpose::STANDARD.encode(signature)
),
},
#[cfg(test)]
signature_base,
})
}
fn signature_agent_header(
profile: WebBotAuthProfile,
signature_agent_origin: &str,
label: &str,
) -> String {
match profile {
WebBotAuthProfile::Cloudflare => format!("\"{signature_agent_origin}\""),
WebBotAuthProfile::IetfDraft01 => {
format!("{label}=\"{signature_agent_origin}\"")
}
}
}
fn signature_agent_component_identifier(profile: WebBotAuthProfile, label: &str) -> String {
match profile {
WebBotAuthProfile::Cloudflare => "\"signature-agent\"".to_owned(),
WebBotAuthProfile::IetfDraft01 => {
format!("\"signature-agent\";key=\"{label}\"")
}
}
}
fn request_authority(request_url: &Url) -> Result<String> {
let host = match request_url
.host()
.context("Web Bot Auth request URL must have an authority")?
{
Host::Domain(domain) => domain.to_owned(),
Host::Ipv4(address) => address.to_string(),
Host::Ipv6(address) => format!("[{address}]"),
};
Ok(match request_url.port() {
Some(port) => format!("{host}:{port}"),
None => host,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::{EXPECTED_KEYID, key};
use aws_lc_rs::signature::{ED25519, UnparsedPublicKey};
#[test]
fn matches_ietf_draft_01_ed25519_test_vector() {
let key = key();
let request_url = Url::parse("https://example.com/foo?param=Value&Pet=dog").unwrap();
let signed = sign_request(
&key,
"https://signature-agent.test",
WebBotAuthProfile::IetfDraft01,
"POST",
&request_url,
&SignatureOptions {
signature_label: "sig2",
signature_agent_label: "agent2",
created: 1_735_689_600,
expires: 4_889_289_600,
nonce: "n9p433xm+NJ3ph3upfBIGmsuwHw387YV7Q/F+6BSpGCVjYCqQw6rznNA8PVVLySrAWsv0hQtFioQb6E1YsauiA==".to_owned(),
cover_request_target: false,
},
)
.unwrap();
assert_eq!(key.keyid(), EXPECTED_KEYID);
assert_eq!(
signed.headers.signature_agent,
"agent2=\"https://signature-agent.test\""
);
assert_eq!(
signed.headers.signature_input,
"sig2=(\"@authority\" \"signature-agent\";key=\"agent2\");created=1735689600;keyid=\"poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U\";alg=\"ed25519\";expires=4889289600;nonce=\"n9p433xm+NJ3ph3upfBIGmsuwHw387YV7Q/F+6BSpGCVjYCqQw6rznNA8PVVLySrAWsv0hQtFioQb6E1YsauiA==\";tag=\"web-bot-auth\""
);
assert_eq!(
signed.headers.signature,
"sig2=:RdNFx5Bj6au3YgAMQL/RzmUlZE8QZLIaXGRpw985hWnwPfMxT228NMk6ehRS1PSl4e8PhbNZACSanGdhEwYCCg==:"
);
}
#[test]
fn matches_cloudflare_legacy_ed25519_test_vector() {
let key = key();
let request_url = Url::parse("https://example.com/foo?param=Value&Pet=dog").unwrap();
let signed = sign_request(
&key,
"https://signature-agent.test",
WebBotAuthProfile::Cloudflare,
"POST",
&request_url,
&SignatureOptions {
signature_label: "sig2",
signature_agent_label: "agent2",
created: 1_735_689_600,
expires: 1_735_693_200,
nonce: "e8N7S2MFd/qrd6T2R3tdfAuuANngKI7LFtKYI/vowzk4lAZYadIX6wW25MwG7DCT9RUKAJ0qVkU0mEeLElW1qg==".to_owned(),
cover_request_target: false,
},
)
.unwrap();
assert_eq!(
signed.headers.signature_agent,
"\"https://signature-agent.test\""
);
assert_eq!(
signed.headers.signature,
"sig2=:jdq0SqOwHdyHr9+r5jw3iYZH6aNGKijYp/EstF4RQTQdi5N5YYKrD+mCT1HA1nZDsi6nJKuHxUi/5Syp3rLWBA==:"
);
}
#[test]
fn production_components_cover_authority_method_and_path() {
let key = key();
let request_url = Url::parse("https://example.com:8443/a%20path?query=yes").unwrap();
let signed = sign_request(
&key,
"https://signature-agent.test",
WebBotAuthProfile::Cloudflare,
"post",
&request_url,
&SignatureOptions {
signature_label: "sig1",
signature_agent_label: "sig1",
created: 1_735_689_600,
expires: 1_735_689_660,
nonce: general_purpose::STANDARD.encode([7_u8; 64]),
cover_request_target: true,
},
)
.unwrap();
assert!(
signed
.signature_base
.contains("\"@authority\": example.com:8443")
);
assert!(signed.signature_base.contains("\"@method\": POST"));
assert!(signed.signature_base.contains("\"@path\": /a%20path"));
let signature = signed
.headers
.signature
.strip_prefix("sig1=:")
.and_then(|value| value.strip_suffix(':'))
.and_then(|value| general_purpose::STANDARD.decode(value).ok())
.unwrap();
UnparsedPublicKey::new(&ED25519, key.public_key())
.verify(signed.signature_base.as_bytes(), &signature)
.unwrap();
}
#[test]
fn signing_rejects_a_request_url_without_an_authority() {
let key = key();
let request_url = Url::parse("data:text/plain,no-authority").unwrap();
let result = sign_request(
&key,
"https://signature-agent.test",
WebBotAuthProfile::Cloudflare,
"GET",
&request_url,
&SignatureOptions {
signature_label: "sig1",
signature_agent_label: "sig1",
created: 1_735_689_600,
expires: 1_735_689_660,
nonce: general_purpose::STANDARD.encode([7_u8; 64]),
cover_request_target: true,
},
);
let error = match result {
Ok(_) => panic!("signing a URL without an authority must fail"),
Err(error) => error,
};
assert_eq!(
error.to_string(),
"Web Bot Auth request URL must have an authority"
);
}
}
moli/src/cli.rs
+33
View File
@@ -32,6 +32,10 @@ const FETCH_INFER_FLAGS: &[&str] = &[
"--disable-subframes",
"--wait-ms",
"--profile-dir",
"--web-bot-auth-key-file",
"--web-bot-auth-keyid",
"--web-bot-auth-domain",
"--web-bot-auth-profile",
];
const SERVE_INFER_FLAGS: &[&str] = &["--host", "--port", "--timeout", "--layout"];
const EXPLICIT_COMMANDS: &[&str] = &["fetch", "serve", "help", "version"];
@@ -367,6 +371,35 @@ pub struct CommonArgs {
#[arg(long)]
pub user_agent_suffix: Option<String>,
/// Unencrypted PKCS#8 Ed25519 private key used for Web Bot Auth signatures.
#[arg(long, value_name = "PATH", requires = "web_bot_auth_domain")]
pub web_bot_auth_key_file: Option<String>,
/// Assert the RFC 7638 JWK thumbprint derived from the Web Bot Auth key.
#[arg(long, value_name = "THUMBPRINT", requires = "web_bot_auth_key_file")]
pub web_bot_auth_keyid: Option<String>,
/// Operator domain publishing /.well-known/http-message-signatures-directory.
#[arg(long, value_name = "DOMAIN", requires = "web_bot_auth_key_file")]
pub web_bot_auth_domain: Option<String>,
/// Signature-Agent wire format. Cloudflare compatibility is the default.
#[arg(
long,
value_enum,
default_value = "cloudflare",
requires = "web_bot_auth_key_file"
)]
pub web_bot_auth_profile: WebBotAuthProfileChoice,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, ValueEnum)]
pub enum WebBotAuthProfileChoice {
#[default]
Cloudflare,
#[value(name = "ietf-01")]
IetfDraft01,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
moli/src/config.rs
+36 -1
View File
@@ -6,10 +6,13 @@ use moli_core::{
page::{SubresourceJsonPathEquals, SubresourceResponseWaitCriteria},
runtime::BrowserConfig,
};
use moli_fetch::{FetchConfig, WebBotAuthProfile, WebBotAuthSigner};
use std::path::PathBuf;
use std::str::FromStr;
use crate::cli::{Cli, Commands, CommonArgs, DumpFormat, LogFormat, StripOptions};
use crate::cli::{
Cli, Commands, CommonArgs, DumpFormat, LogFormat, StripOptions, WebBotAuthProfileChoice,
};
use crate::network_trace::NetworkTraceConfigSummary;
pub use moli_protocol_server::ServerConfig;
@@ -183,6 +186,7 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()>
.browser
.fetch_mut()
.set_tls_verify_host(!common.insecure_disable_tls_host_verification);
configure_web_bot_auth(config.browser.fetch_mut(), common)?;
for source in &common.document_start_script {
config.add_document_start_script(source.clone());
@@ -199,6 +203,37 @@ fn apply_common_args(config: &mut AppConfig, common: &CommonArgs) -> Result<()>
Ok(())
}
fn configure_web_bot_auth(fetch: &mut FetchConfig, common: &CommonArgs) -> Result<()> {
let (key_file, domain) = match (
common.web_bot_auth_key_file.as_deref(),
common.web_bot_auth_domain.as_deref(),
) {
(None, None) => {
if common.web_bot_auth_keyid.is_some() {
bail!("--web-bot-auth-keyid requires --web-bot-auth-key-file");
}
return Ok(());
}
(Some(_), None) => bail!("--web-bot-auth-key-file requires --web-bot-auth-domain"),
(None, Some(_)) => bail!("--web-bot-auth-domain requires --web-bot-auth-key-file"),
(Some(key_file), Some(domain)) => (key_file, domain),
};
let private_key_pem = std::fs::read(key_file)
.with_context(|| format!("failed to read Web Bot Auth private key `{key_file}`"))?;
let profile = match common.web_bot_auth_profile {
WebBotAuthProfileChoice::Cloudflare => WebBotAuthProfile::Cloudflare,
WebBotAuthProfileChoice::IetfDraft01 => WebBotAuthProfile::IetfDraft01,
};
let signer = WebBotAuthSigner::from_pem(
&private_key_pem,
domain,
common.web_bot_auth_keyid.as_deref(),
profile,
)?;
fetch.set_web_bot_auth(Some(signer));
Ok(())
}
fn validate_http_host_resolve_entries(entries: &[String]) -> Result<()> {
for entry in entries {
validate_http_host_resolve_entry(entry)?;
moli/tests/cli.rs
+156 -15
View File
@@ -1,15 +1,27 @@
use clap::Parser;
use std::{num::NonZeroU32, process::Command};
use std::{
fs,
num::NonZeroU32,
path::PathBuf,
process::Command,
sync::atomic::{AtomicU64, Ordering},
};
use moli::cli::{
Cli, Commands, CommonArgs, DumpFormat, FetchArgs, FetchWaitUntil, LogFormat, LogLevel,
RequestHeaderArg, ResponseJsonPathArg, ServeArgs, StripModeChoice, StripOptions,
normalize_args_for_compat,
WebBotAuthProfileChoice, normalize_args_for_compat,
};
use moli::config::AppConfig;
use moli_browser_profile::BrowserProfilePaths;
use moli_core::OptionalResourceFetchMask;
use moli_fetch::FetchConfig;
use moli_fetch::{FetchConfig, WebBotAuthProfile};
const RFC_9421_ED25519_PRIVATE_KEY: &str = "-----BEGIN PRIVATE KEY-----\n\
MC4CAQAwBQYDK2VwBCIEIJ+DYvh6SEqVTm50DFtMDoQikTmiCqirVv9mWG9qfSnF\n\
-----END PRIVATE KEY-----\n";
const RFC_9421_ED25519_KEYID: &str = "poqkLGiymh_W0uP6PZFw-dvez3QJT5SolqXBCW38r0U";
static NEXT_TEMP_KEY_FILE: AtomicU64 = AtomicU64::new(0);
#[test]
fn parses_explicit_fetch_command_with_compatibility_flags() {
@@ -125,6 +137,10 @@ fn parses_explicit_fetch_command_with_compatibility_flags() {
log_filter_scopes: Some("http,event".to_owned()),
user_agent: None,
user_agent_suffix: Some("internal-tester".to_owned()),
web_bot_auth_key_file: None,
web_bot_auth_keyid: None,
web_bot_auth_domain: None,
web_bot_auth_profile: WebBotAuthProfileChoice::Cloudflare,
},
url: "https://example.com".to_owned(),
}))
@@ -200,22 +216,79 @@ fn parses_binary_dump_modes_with_inferred_fetch_command() {
}
#[test]
fn rejects_removed_web_bot_auth_flags() {
for flag in [
fn parses_web_bot_auth_flags() {
let cli = Cli::try_parse_from(normalize_args_for_compat([
"moli",
"fetch",
"--web-bot-auth-key-file",
"bot-key.pem",
"--web-bot-auth-keyid",
RFC_9421_ED25519_KEYID,
"--web-bot-auth-domain",
] {
let error = Cli::try_parse_from(normalize_args_for_compat([
"moli",
"fetch",
flag,
"unused",
"https://example.com",
]))
.unwrap_err();
"bot.example",
"--web-bot-auth-profile",
"ietf-01",
"https://example.com",
]))
.unwrap();
assert_eq!(error.kind(), clap::error::ErrorKind::UnknownArgument);
let Commands::Fetch(args) = cli.command else {
panic!("expected fetch command");
};
assert_eq!(
args.common.web_bot_auth_key_file.as_deref(),
Some("bot-key.pem")
);
assert_eq!(
args.common.web_bot_auth_keyid.as_deref(),
Some(RFC_9421_ED25519_KEYID)
);
assert_eq!(
args.common.web_bot_auth_domain.as_deref(),
Some("bot.example")
);
assert_eq!(
args.common.web_bot_auth_profile,
WebBotAuthProfileChoice::IetfDraft01
);
}
#[test]
fn web_bot_auth_profile_default_does_not_enable_signing() {
let cli = Cli::try_parse_from(normalize_args_for_compat([
"moli",
"fetch",
"https://example.com",
]))
.unwrap();
let Commands::Fetch(args) = cli.command else {
panic!("expected fetch command");
};
assert_eq!(
args.common.web_bot_auth_profile,
WebBotAuthProfileChoice::Cloudflare
);
assert!(args.common.web_bot_auth_key_file.is_none());
}
#[test]
fn web_bot_auth_flags_require_key_and_domain_together() {
for args in [
vec!["--web-bot-auth-key-file", "bot-key.pem"],
vec!["--web-bot-auth-domain", "bot.example"],
vec!["--web-bot-auth-keyid", RFC_9421_ED25519_KEYID],
vec!["--web-bot-auth-profile", "ietf-01"],
] {
let mut command = vec!["moli", "fetch"];
command.extend(args);
command.push("https://example.com");
let error = Cli::try_parse_from(normalize_args_for_compat(command)).unwrap_err();
assert_eq!(
error.kind(),
clap::error::ErrorKind::MissingRequiredArgument
);
}
}
@@ -550,6 +623,54 @@ fn app_config_uses_moli_user_agent_defaults() {
assert_eq!(config.browser.fetch().user_agent(), "ExampleBrowser/1.0");
}
#[test]
fn app_config_loads_and_validates_web_bot_auth_key() {
let key_file = write_temp_web_bot_auth_key();
let key_path_arg = key_file.path.to_string_lossy().into_owned();
let cli = Cli::try_parse_from(normalize_args_for_compat([
"moli",
"fetch",
"--web-bot-auth-key-file",
key_path_arg.as_str(),
"--web-bot-auth-keyid",
RFC_9421_ED25519_KEYID,
"--web-bot-auth-domain",
"bot.example:8443",
"--web-bot-auth-profile",
"ietf-01",
"https://example.com",
]))
.unwrap();
let config = AppConfig::from_cli(&cli).unwrap();
let signer = config.browser.fetch().web_bot_auth().unwrap();
assert_eq!(signer.keyid(), RFC_9421_ED25519_KEYID);
assert_eq!(signer.signature_agent_origin(), "https://bot.example:8443");
assert_eq!(signer.profile(), WebBotAuthProfile::IetfDraft01);
}
#[test]
fn app_config_rejects_mismatched_web_bot_auth_keyid() {
let key_file = write_temp_web_bot_auth_key();
let key_path_arg = key_file.path.to_string_lossy().into_owned();
let cli = Cli::try_parse_from(normalize_args_for_compat([
"moli",
"fetch",
"--web-bot-auth-key-file",
key_path_arg.as_str(),
"--web-bot-auth-keyid",
"wrong-thumbprint",
"--web-bot-auth-domain",
"bot.example",
"https://example.com",
]))
.unwrap();
let error = AppConfig::from_cli(&cli).unwrap_err().to_string();
assert!(error.contains("does not match the private key"));
assert!(error.contains(RFC_9421_ED25519_KEYID));
}
#[test]
fn app_config_preserves_repeatable_request_headers() {
let cli = Cli::try_parse_from(normalize_args_for_compat([
@@ -574,6 +695,26 @@ fn app_config_preserves_repeatable_request_headers() {
assert!(config.browser.fetch().default_request_headers().is_empty());
}
struct TempWebBotAuthKeyFile {
path: PathBuf,
}
impl Drop for TempWebBotAuthKeyFile {
fn drop(&mut self) {
let _ = fs::remove_file(&self.path);
}
}
fn write_temp_web_bot_auth_key() -> TempWebBotAuthKeyFile {
let sequence = NEXT_TEMP_KEY_FILE.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"moli-web-bot-auth-{}-{sequence}.pem",
std::process::id()
));
fs::write(&path, RFC_9421_ED25519_PRIVATE_KEY).unwrap();
TempWebBotAuthKeyFile { path }
}
#[test]
fn parses_request_headers_with_embedded_colons_and_empty_values() {
let cli = Cli::try_parse_from(normalize_args_for_compat([
skills/moli-webfetch/references/fetch-recipes.md
+6
View File
@@ -135,6 +135,12 @@ For a crawl rather than a single lookup:
- Use `--http-proxy`, `--http-no-proxy`, or
`--http-host-resolve HOST:PORT:ADDR` when required by the environment.
- Use either `--user-agent` or `--user-agent-suffix`, not both.
- Use `--web-bot-auth-key-file <PKCS8-PEM>` together with
`--web-bot-auth-domain <DOMAIN>` only when the user supplied an authorized
bot identity. The key remains local, and Moli signs HTTPS requests only.
- Add `--web-bot-auth-keyid <THUMBPRINT>` to assert the derived JWK thumbprint.
Keep the default Cloudflare profile unless the receiver explicitly supports
`--web-bot-auth-profile ietf-01`.
- Use `--document-start-script` or `--document-start-script-file` only when the
task explicitly requires pre-navigation instrumentation.
- Combine `--block-private-networks` with `--block-cidrs` for untrusted URL