diff --git a/moli-cookie-cache/src/lib.rs b/moli-cookie-cache/src/lib.rs index d27024609e..23a114ee3c 100644 --- a/moli-cookie-cache/src/lib.rs +++ b/moli-cookie-cache/src/lib.rs @@ -9,7 +9,7 @@ mod cache; mod netscape; pub use cache::{load_cookie_cache, save_cookie_cache}; -pub use netscape::load_cookie_file; +pub use netscape::{CookieFileSaveReport, load_cookie_file, save_cookie_file}; #[cfg(test)] mod tests; diff --git a/moli-cookie-cache/src/netscape.rs b/moli-cookie-cache/src/netscape.rs index a618156bec..b8ffc1bc51 100644 --- a/moli-cookie-cache/src/netscape.rs +++ b/moli-cookie-cache/src/netscape.rs @@ -1,9 +1,21 @@ -use std::{fs::File, io::BufReader, path::Path}; +use std::{fmt::Write as _, fs::File, io::BufReader, path::Path}; use anyhow::{Context, Result}; use moli_cookie_jar::{StoredCookie, StoredCookieSameSite, StoredCookieSourceScheme}; use time::OffsetDateTime; +use crate::atomic_file::write_file_atomically; + +const NETSCAPE_COOKIE_FILE_HEADER: &str = "# Netscape HTTP Cookie File\n\ +# This file was generated by Moli. Do not edit.\n\n"; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CookieFileSaveReport { + pub written: usize, + pub skipped_expired: usize, + pub skipped_partitioned: usize, +} + pub fn load_cookie_file(path: impl AsRef) -> Result> { let path = path.as_ref(); let file = File::open(path) @@ -12,6 +24,78 @@ pub fn load_cookie_file(path: impl AsRef) -> Result> { .with_context(|| format!("failed to parse cookie file `{}`", path.display())) } +/// Saves unexpired, unpartitioned cookies in the interoperable Netscape format. +/// +/// Netscape cookie files cannot represent partition keys, so partitioned +/// cookies are deliberately omitted instead of broadening their scope on a +/// later import. The returned report describes written and omitted records. +pub fn save_cookie_file( + path: impl AsRef, + cookies: impl IntoIterator, +) -> Result { + let path = path.as_ref(); + let mut output = NETSCAPE_COOKIE_FILE_HEADER.to_owned(); + let mut report = CookieFileSaveReport::default(); + + for cookie in cookies { + if cookie.is_expired() { + report.skipped_expired += 1; + continue; + } + if cookie.partition_key.is_some() { + report.skipped_partitioned += 1; + continue; + } + append_netscape_cookie(&mut output, &cookie)?; + report.written += 1; + } + + write_file_atomically(path, output.as_bytes(), "cookie jar")?; + Ok(report) +} + +fn append_netscape_cookie(output: &mut String, cookie: &StoredCookie) -> Result<()> { + let domain = cookie.domain.trim_start_matches('.'); + anyhow::ensure!(!domain.is_empty(), "cookie domain is empty"); + anyhow::ensure!(!cookie.name.is_empty(), "cookie name is empty"); + anyhow::ensure!( + cookie.path.starts_with('/'), + "cookie path must start with `/`" + ); + for (field, value) in [ + ("domain", domain), + ("path", cookie.path.as_str()), + ("name", cookie.name.as_str()), + ("value", cookie.value.as_str()), + ] { + anyhow::ensure!( + !value.contains(['\t', '\r', '\n']), + "cookie {field} cannot be represented in a Netscape cookie file" + ); + } + + if cookie.http_only { + output.push_str("#HttpOnly_"); + } + if !cookie.host_only { + output.push('.'); + } + output.push_str(domain); + let tail_match = if cookie.host_only { "FALSE" } else { "TRUE" }; + let secure = if cookie.secure { "TRUE" } else { "FALSE" }; + let expires = cookie + .expires + .map(|expires| expires.unix_timestamp()) + .unwrap_or(0); + writeln!( + output, + "\t{tail_match}\t{}\t{secure}\t{expires}\t{}\t{}", + cookie.path, cookie.name, cookie.value + ) + .expect("writing to a String cannot fail"); + Ok(()) +} + pub(crate) fn parse_netscape_cookie_file( reader: impl std::io::BufRead, ) -> Result> { diff --git a/moli-cookie-cache/src/tests.rs b/moli-cookie-cache/src/tests.rs index fcbf048fe1..24df178c38 100644 --- a/moli-cookie-cache/src/tests.rs +++ b/moli-cookie-cache/src/tests.rs @@ -187,6 +187,87 @@ fn netscape_cookie_parser_imports_httponly_tailmatch_cookie() -> Result<()> { Ok(()) } +#[test] +fn netscape_cookie_writer_round_trips_representable_cookies() -> Result<()> { + let temp = TempDir::new("netscape-write-roundtrip"); + let target = temp.path.join("cookies.txt"); + let mut domain_cookie = stored_cookie("session", "fixture"); + domain_cookie.host_only = false; + domain_cookie.path = "/account/".to_owned(); + domain_cookie.secure = true; + domain_cookie.http_only = true; + domain_cookie.expires = Some(OffsetDateTime::now_utc() + time::Duration::days(1)); + let host_cookie = stored_cookie("host", "session value"); + + let report = save_cookie_file(&target, vec![domain_cookie, host_cookie])?; + + assert_eq!(report.written, 2); + assert_eq!(report.skipped_expired, 0); + assert_eq!(report.skipped_partitioned, 0); + let contents = fs::read_to_string(&target)?; + assert!(contents.starts_with("# Netscape HTTP Cookie File\n")); + assert!(contents.contains("#HttpOnly_.example.com\tTRUE\t/account/\tTRUE\t")); + let loaded = load_cookie_file(&target)?; + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].domain, "example.com"); + assert!(!loaded[0].host_only); + assert_eq!(loaded[0].path, "/account/"); + assert!(loaded[0].secure); + assert!(loaded[0].http_only); + assert_eq!(loaded[0].name, "session"); + assert_eq!(loaded[0].value, "fixture"); + assert!(loaded[0].expires.is_some()); + assert!(loaded[1].host_only); + assert_eq!(loaded[1].name, "host"); + assert_eq!(loaded[1].value, "session value"); + assert_eq!(loaded[1].expires, None); + Ok(()) +} + +#[test] +fn netscape_cookie_writer_omits_expired_and_partitioned_cookies() -> Result<()> { + let temp = TempDir::new("netscape-write-filter"); + let target = temp.path.join("cookies.txt"); + let mut expired = stored_cookie("expired", "gone"); + expired.expires = Some(OffsetDateTime::now_utc() - time::Duration::days(1)); + let mut partitioned = stored_cookie("partitioned", "scoped"); + partitioned.partition_key = Some(StoredCookiePartitionKey::site( + "https://top.example".to_owned(), + false, + )); + + let report = save_cookie_file( + &target, + vec![expired, partitioned, stored_cookie("plain", "kept")], + )?; + + assert_eq!(report.written, 1); + assert_eq!(report.skipped_expired, 1); + assert_eq!(report.skipped_partitioned, 1); + let loaded = load_cookie_file(&target)?; + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].name, "plain"); + Ok(()) +} + +#[test] +fn netscape_cookie_writer_rejects_unrepresentable_fields() { + let temp = TempDir::new("netscape-write-invalid"); + let target = temp.path.join("cookies.txt"); + let cookie = stored_cookie("session", "line\tbreak"); + + let error = save_cookie_file(&target, vec![cookie]) + .expect_err("tabs cannot be encoded in Netscape cookie records"); + + assert!( + error + .to_string() + .contains("cookie value cannot be represented"), + "error={error:#}" + ); + assert!(!target.exists()); +} + #[test] fn netscape_cookie_parser_preserves_trailing_slash_path_scope() -> Result<()> { let cookies = parse_netscape_cookie_file(std::io::Cursor::new( diff --git a/moli/src/app.rs b/moli/src/app.rs index 4b8745bd06..f05dfcf48d 100644 --- a/moli/src/app.rs +++ b/moli/src/app.rs @@ -58,7 +58,7 @@ pub async fn run_cli_with_config( let fetched_document = match fetch_result { Ok(document) => document, Err(error) => { - finalize_fetch_browser(browser); + finalize_fetch_browser_after_failure(browser, args.cookie_jar.as_deref()); return Err(with_fetch_context(error, &args.url)); } }; @@ -67,7 +67,7 @@ pub async fn run_cli_with_config( FetchedDocument::Page(page) => page, FetchedDocument::Raw(raw_document) => { if readiness.has_page_waits() || args.delay_ms > 0 { - finalize_fetch_browser(browser); + finalize_fetch_browser_after_failure(browser, args.cookie_jar.as_deref()); return Err(with_fetch_context( anyhow!( "raw non-HTML document fetch does not support page wait options" @@ -83,7 +83,8 @@ pub async fn run_cli_with_config( .context("failed to write raw fetch output") .map_err(|error| with_fetch_context(error, &args.url))?; let _ = stdout.flush(); - finalize_fetch_browser(browser); + finalize_fetch_browser(browser, args.cookie_jar.as_deref()) + .map_err(|error| with_fetch_context(error, &args.url))?; return Ok(()); } }; @@ -95,7 +96,7 @@ pub async fn run_cli_with_config( "failed to close fetched page after readiness failure" ); } - finalize_fetch_browser(browser); + finalize_fetch_browser_after_failure(browser, args.cookie_jar.as_deref()); return Err(with_fetch_context(error, &args.url)); } @@ -118,7 +119,8 @@ pub async fn run_cli_with_config( if let Err(error) = page.close_async().await { tracing::warn!(error = %error, "failed to close fetched page before browser shutdown"); } - finalize_fetch_browser(browser); + finalize_fetch_browser(browser, args.cookie_jar.as_deref()) + .map_err(|error| with_fetch_context(error, &args.url))?; } Commands::Serve(_) => { if config.browser.fetch().obey_robots() { @@ -235,12 +237,34 @@ fn load_cookie_state_cookies(config: &AppConfig) -> Result) { + if let Err(error) = finalize_fetch_browser(browser, cookie_jar) { + tracing::warn!( + error = %error, + "failed to finalize cookie jar after fetch failure" + ); + } +} + +fn finalize_fetch_browser(browser: Browser, cookie_jar: Option<&str>) -> Result<()> { + let save_result = cookie_jar.map_or(Ok(()), |path| -> Result<()> { + let report = cookie_cache::save_browser_cookie_file(&browser, path) + .with_context(|| format!("failed to save cookie jar `{path}`"))?; + if report.skipped_partitioned > 0 { + tracing::warn!( + path, + skipped = report.skipped_partitioned, + "omitted partitioned cookies from Netscape cookie jar" + ); + } + Ok(()) + }); // Fetch is a one-shot CLI path, but the browser must still be dropped in an // orderly way. Letting network threads survive until process exit can race // OpenSSL global cleanup with libcurl transfers still in progress. // Browser::drop owns profile cookie writeback when --profile-dir is set. drop(browser); + save_result } #[cfg(test)] diff --git a/moli/src/cli.rs b/moli/src/cli.rs index 2a319cad1c..02cb62c3f2 100644 --- a/moli/src/cli.rs +++ b/moli/src/cli.rs @@ -61,6 +61,10 @@ pub struct FetchArgs { #[arg(long, value_name = "TEXT")] pub body: Option, + /// Write final unpartitioned cookies to a Netscape cookie file. + #[arg(long, value_name = "PATH")] + pub cookie_jar: Option, + #[arg(long)] pub noscript: bool, diff --git a/moli/src/cookie_cache.rs b/moli/src/cookie_cache.rs index 89adc4809b..dbec286806 100644 --- a/moli/src/cookie_cache.rs +++ b/moli/src/cookie_cache.rs @@ -22,6 +22,16 @@ pub fn load_browser_cookie_file(browser: &Browser, path: impl AsRef) -> Re browser.import_cookies(load_cookie_file(path)?) } +pub fn save_browser_cookie_file( + browser: &Browser, + path: impl AsRef, +) -> Result { + let cookies = browser + .cookies() + .context("failed to snapshot browser cookies for cookie jar")?; + moli_cookie_cache::save_cookie_file(path, cookies) +} + pub fn save_browser_cookie_cache(browser: &Browser, path: impl AsRef) -> Result<()> { save_cookie_cache( path, diff --git a/moli/tests/cli.rs b/moli/tests/cli.rs index 2456fd01ab..db588acd78 100644 --- a/moli/tests/cli.rs +++ b/moli/tests/cli.rs @@ -76,6 +76,7 @@ fn parses_explicit_fetch_command_with_compatibility_flags() { dump: Some(DumpFormat::SemanticTree), method: "GET".to_owned(), body: None, + cookie_jar: None, headers: vec![ RequestHeaderArg { name: "X-Test".to_owned(), @@ -416,6 +417,7 @@ fn infers_fetch_mode_from_bare_url() { dump: None, method: "GET".to_owned(), body: None, + cookie_jar: None, headers: vec![], noscript: false, with_base: false, @@ -458,6 +460,7 @@ fn parses_bare_dump_with_explicit_fetch_command_and_defaults_to_html() { dump: Some(DumpFormat::Html), method: "GET".to_owned(), body: None, + cookie_jar: None, headers: vec![], noscript: false, with_base: false, @@ -501,6 +504,7 @@ fn parses_header_flag_with_explicit_fetch_command() { dump: None, method: "GET".to_owned(), body: None, + cookie_jar: None, headers: vec![RequestHeaderArg { name: "X-Test".to_owned(), value: "one".to_owned(), @@ -1187,6 +1191,8 @@ fn parses_fetch_network_policy_flags() { "/tmp/browser-cookies.txt", "--cookie-file", "/tmp/extra-cookies.txt", + "--cookie-jar", + "/tmp/final-cookies.txt", "--block-private-networks", "--block-cidrs", "198.18.0.0/15,203.0.113.0/24", @@ -1220,6 +1226,7 @@ fn parses_fetch_network_policy_flags() { "/tmp/extra-cookies.txt".to_owned() ] ); + assert_eq!(args.cookie_jar.as_deref(), Some("/tmp/final-cookies.txt")); assert!(args.common.block_private_networks); assert_eq!( args.common.block_cidrs.as_deref(), diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs index 71366e4a44..2dbc5dca62 100644 --- a/moli/tests/fetch_cli.rs +++ b/moli/tests/fetch_cli.rs @@ -1225,6 +1225,51 @@ fn cli_cookie_file_imports_netscape_cookie_before_fetch() -> Result<()> { Ok(()) } +#[test] +fn cli_cookie_jar_writes_set_cookie_for_later_import() -> Result<()> { + let runtime = tokio::runtime::Runtime::new()?; + let server = runtime.block_on(FixtureServer::spawn())?; + let url = server.url("/cookie"); + let cookie_jar = unique_temp_file_path("cookie-jar-writeback", "cookies.txt")?; + let cookie_jar_arg = cookie_jar.to_string_lossy().into_owned(); + + let first = run_fetch_cli_with_dump_and_args(&url, "html", &["--cookie-jar", &cookie_jar_arg])?; + assert!( + first.status.success(), + "first moli fetch failed: stdout={}\nstderr={}", + String::from_utf8_lossy(&first.stdout), + String::from_utf8_lossy(&first.stderr) + ); + let first_stdout = clean_output(&first.stdout); + assert!( + first_stdout.contains("
cookie=missing
"), + "stdout={first_stdout}" + ); + let jar_contents = std::fs::read_to_string(&cookie_jar)?; + assert!(jar_contents.starts_with("# Netscape HTTP Cookie File\n")); + assert!( + jar_contents.contains("\tsession\tfixture\n"), + "cookie jar={jar_contents}" + ); + + let second = + run_fetch_cli_with_dump_and_args(&url, "html", &["--cookie-file", &cookie_jar_arg])?; + runtime.block_on(server.shutdown()); + + assert!( + second.status.success(), + "second moli fetch failed: stdout={}\nstderr={}", + String::from_utf8_lossy(&second.stdout), + String::from_utf8_lossy(&second.stderr) + ); + let second_stdout = clean_output(&second.stdout); + assert!( + second_stdout.contains("
cookie=seen
"), + "stdout={second_stdout}" + ); + Ok(()) +} + #[test] fn cli_profile_dir_persists_cookies_after_successful_fetch() -> Result<()> { let runtime = tokio::runtime::Runtime::new()?;