mirror of
https://github.com/stalwartlabs/stalwart.git
synced 2026-10-03 16:01:50 +00:00
Update sieve zstd dictionary
This commit is contained in:
Generated
+2
@@ -7902,6 +7902,8 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
[[package]]
|
||||
name = "sieve-rs"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fabd36845a9418dd73ca312f5e87f57f6662cf61d125f40a33c5349fdc24cfb"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bumpalo",
|
||||
|
||||
@@ -36,9 +36,6 @@ exclude = ["fuzz"]
|
||||
[workspace.dependencies]
|
||||
rkyv = { version = "0.8.18", features = ["little_endian", "unaligned"] }
|
||||
|
||||
[patch.crates-io]
|
||||
sieve-rs = { path = "../sieve" }
|
||||
|
||||
[workspace.lints.clippy]
|
||||
result_unit_err = "allow"
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ use common::{
|
||||
};
|
||||
use mail_parser::MessageParser;
|
||||
use registry::schema::enums::Permission;
|
||||
use sieve::compiler::grammar::actions::action_redirect::Notify;
|
||||
use std::{borrow::Cow, future::Future};
|
||||
use store::ahash::AHashMap;
|
||||
use types::blob_hash::BlobHash;
|
||||
@@ -62,6 +63,7 @@ pub struct AutogeneratedMessage {
|
||||
pub sender_address: String,
|
||||
pub recipients: Vec<String>,
|
||||
pub message: Vec<u8>,
|
||||
pub notify: Notify,
|
||||
}
|
||||
|
||||
pub trait MailDelivery: Sync + Send {
|
||||
|
||||
@@ -21,8 +21,9 @@ use common::{
|
||||
use mail_builder::headers::date::Date;
|
||||
use mail_parser::{HeaderName, MessageParser};
|
||||
use sieve::{
|
||||
Arena, Context, Envelope, Handler, Input, Mailbox, Recipient, Reply, ScriptArena, Sieve,
|
||||
SieveAction, Status,
|
||||
Arena, Context, Envelope, Handler, Input, Mailbox, MessageSource, Recipient, Reply,
|
||||
ScriptArena, Sieve, SieveAction, Status,
|
||||
compiler::grammar::actions::action_redirect::Notify,
|
||||
runtime::{Script, Variable},
|
||||
};
|
||||
use std::{borrow::Cow, sync::Arc};
|
||||
@@ -116,7 +117,18 @@ impl<'x> SieveHandler<'x> {
|
||||
}
|
||||
}
|
||||
|
||||
fn send_message(&mut self, recipient: Recipient<'_>, message_id: usize) {
|
||||
fn send_message(
|
||||
&mut self,
|
||||
source: MessageSource,
|
||||
recipient: Recipient<'_>,
|
||||
notify: Notify,
|
||||
message_id: usize,
|
||||
) {
|
||||
let sender_address = if source == MessageSource::Vacation {
|
||||
""
|
||||
} else {
|
||||
self.mail_from
|
||||
};
|
||||
let Some(message) = self.messages.get(message_id) else {
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::UnexpectedError),
|
||||
@@ -161,7 +173,7 @@ impl<'x> SieveHandler<'x> {
|
||||
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::SendMessage),
|
||||
From = self.mail_from.to_string(),
|
||||
From = sender_address.to_string(),
|
||||
To = recipients
|
||||
.iter()
|
||||
.map(|r| trc::Value::String(r.as_str().into()))
|
||||
@@ -175,9 +187,10 @@ impl<'x> SieveHandler<'x> {
|
||||
raw_message.extend_from_slice(message.raw_message.as_ref());
|
||||
|
||||
self.autogenerated.push(AutogeneratedMessage {
|
||||
sender_address: self.mail_from.to_string(),
|
||||
sender_address: sender_address.to_string(),
|
||||
recipients,
|
||||
message: raw_message,
|
||||
notify,
|
||||
});
|
||||
self.do_redirect = true;
|
||||
}
|
||||
@@ -324,11 +337,13 @@ impl<'x> Handler<'x> for SieveHandler<'x> {
|
||||
}
|
||||
}
|
||||
SieveAction::SendMessage {
|
||||
source,
|
||||
recipient,
|
||||
notify,
|
||||
message_id,
|
||||
..
|
||||
} => {
|
||||
self.send_message(recipient, message_id);
|
||||
self.send_message(source, recipient, notify, message_id);
|
||||
}
|
||||
SieveAction::CreatedMessage { message, .. } => {
|
||||
self.messages.push(SieveMessage {
|
||||
|
||||
@@ -14,6 +14,7 @@ use crate::{
|
||||
rcpt_spam_percentage,
|
||||
spool::{QueueParams, SmtpSpool},
|
||||
},
|
||||
scripts::notify_flags,
|
||||
};
|
||||
use common::Server;
|
||||
use email::message::delivery::{IngestMessage, IngestRecipient, LocalDeliveryStatus, MailDelivery};
|
||||
@@ -108,6 +109,12 @@ impl MessageWrapper {
|
||||
for rcpt in autogenerated.recipients {
|
||||
message.expand_and_add_recipient(rcpt, server).await;
|
||||
}
|
||||
let flags = notify_flags(&autogenerated.notify);
|
||||
if flags != 0 {
|
||||
for rcpt in &mut message.message.recipients {
|
||||
rcpt.flags |= flags;
|
||||
}
|
||||
}
|
||||
|
||||
// Queue Message
|
||||
message.message.size = autogenerated.message.len() as u64;
|
||||
|
||||
@@ -16,18 +16,16 @@ use common::{
|
||||
};
|
||||
use mail_parser::{Encoding, Message, MessagePart, PartType};
|
||||
use sieve::{
|
||||
Arena, Context, Handler, Input, Mailbox, MatchAs, Recipient, Reply, Sieve, SieveAction, Status,
|
||||
compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret},
|
||||
Arena, Context, Handler, Input, Mailbox, MatchAs, MessageSource as SieveMessageSource,
|
||||
Recipient, Reply, Sieve, SieveAction, Status,
|
||||
compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, Ret},
|
||||
runtime::{RuntimeError, Script, Variable},
|
||||
};
|
||||
use smtp_proto::{
|
||||
MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE,
|
||||
RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS,
|
||||
};
|
||||
use smtp_proto::{MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS};
|
||||
use std::{borrow::Cow, future::Future, time::Instant};
|
||||
use trc::SieveEvent;
|
||||
|
||||
use super::{ScriptParameters, ScriptResult};
|
||||
use super::{ScriptParameters, ScriptResult, notify_flags};
|
||||
|
||||
const IMPLICIT_KEEP: usize = usize::MAX;
|
||||
const DISCARD: usize = usize::MAX - 1;
|
||||
@@ -63,6 +61,7 @@ enum PendingWork<'x> {
|
||||
arguments: Vec<Variable<'x>>,
|
||||
},
|
||||
SendMessage {
|
||||
source: SieveMessageSource,
|
||||
recipients: Vec<String>,
|
||||
notify: Notify,
|
||||
return_of_content: Ret,
|
||||
@@ -154,6 +153,7 @@ impl<'x> Handler<'x> for SmtpHandler<'x> {
|
||||
});
|
||||
}
|
||||
SieveAction::SendMessage {
|
||||
source,
|
||||
recipient,
|
||||
notify,
|
||||
return_of_content,
|
||||
@@ -177,6 +177,7 @@ impl<'x> Handler<'x> for SmtpHandler<'x> {
|
||||
};
|
||||
|
||||
self.pending = Some(PendingWork::SendMessage {
|
||||
source,
|
||||
recipients,
|
||||
notify,
|
||||
return_of_content,
|
||||
@@ -315,6 +316,7 @@ impl RunScript for Server {
|
||||
.await
|
||||
}
|
||||
Some(PendingWork::SendMessage {
|
||||
source,
|
||||
recipients,
|
||||
notify,
|
||||
return_of_content,
|
||||
@@ -326,6 +328,7 @@ impl RunScript for Server {
|
||||
ctx.message().raw_message(),
|
||||
¶ms,
|
||||
&script_id,
|
||||
source,
|
||||
recipients,
|
||||
notify,
|
||||
return_of_content,
|
||||
@@ -442,6 +445,7 @@ trait QueueSieveMessage {
|
||||
original_raw_message: &[u8],
|
||||
params: &ScriptParameters<'_>,
|
||||
script_id: &str,
|
||||
source: SieveMessageSource,
|
||||
recipients: Vec<String>,
|
||||
notify: Notify,
|
||||
return_of_content: Ret,
|
||||
@@ -457,6 +461,7 @@ impl QueueSieveMessage for Server {
|
||||
original_raw_message: &[u8],
|
||||
params: &ScriptParameters<'_>,
|
||||
script_id: &str,
|
||||
source: SieveMessageSource,
|
||||
recipients: Vec<String>,
|
||||
notify: Notify,
|
||||
return_of_content: Ret,
|
||||
@@ -466,32 +471,18 @@ impl QueueSieveMessage for Server {
|
||||
let session_id = params.session_id;
|
||||
|
||||
// Build message
|
||||
let mut message = self.new_message(
|
||||
params.return_path.as_str(),
|
||||
MessageSource::Autogenerated,
|
||||
session_id,
|
||||
);
|
||||
let return_path = if source == SieveMessageSource::Vacation {
|
||||
""
|
||||
} else {
|
||||
params.return_path.as_str()
|
||||
};
|
||||
let mut message = self.new_message(return_path, MessageSource::Autogenerated, session_id);
|
||||
for rcpt in recipients {
|
||||
message.expand_and_add_recipient(rcpt, self).await;
|
||||
}
|
||||
|
||||
// Set notify flags
|
||||
let mut flags = 0;
|
||||
match notify {
|
||||
Notify::Never => {
|
||||
flags = RCPT_NOTIFY_NEVER;
|
||||
}
|
||||
Notify::Items(items) => {
|
||||
for item in items {
|
||||
flags |= match item {
|
||||
NotifyItem::Success => RCPT_NOTIFY_SUCCESS,
|
||||
NotifyItem::Failure => RCPT_NOTIFY_FAILURE,
|
||||
NotifyItem::Delay => RCPT_NOTIFY_DELAY,
|
||||
};
|
||||
}
|
||||
}
|
||||
Notify::Default => (),
|
||||
}
|
||||
let flags = notify_flags(¬ify);
|
||||
if flags > 0 {
|
||||
for rcpt in &mut message.message.recipients {
|
||||
rcpt.flags |= flags;
|
||||
|
||||
@@ -12,7 +12,12 @@ use common::{
|
||||
};
|
||||
|
||||
use mail_parser::Message;
|
||||
use sieve::{Envelope, SpamStatus, runtime::Variable};
|
||||
use sieve::{
|
||||
Envelope, SpamStatus,
|
||||
compiler::grammar::actions::action_redirect::{Notify, NotifyItem},
|
||||
runtime::Variable,
|
||||
};
|
||||
use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS};
|
||||
|
||||
pub mod envelope;
|
||||
pub mod event_loop;
|
||||
@@ -134,3 +139,18 @@ impl Default for ScriptParameters<'_> {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn notify_flags(notify: &Notify) -> u64 {
|
||||
match notify {
|
||||
Notify::Never => RCPT_NOTIFY_NEVER,
|
||||
Notify::Items(items) => items.iter().fold(0, |flags, item| {
|
||||
flags
|
||||
| match item {
|
||||
NotifyItem::Success => RCPT_NOTIFY_SUCCESS,
|
||||
NotifyItem::Failure => RCPT_NOTIFY_FAILURE,
|
||||
NotifyItem::Delay => RCPT_NOTIFY_DELAY,
|
||||
}
|
||||
}),
|
||||
Notify::Default => 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,15 +21,12 @@ mail-builder = { version = "0.5" }
|
||||
mail-parser = { version = "0.11", features = ["full_encoding"] }
|
||||
rkyv = { version = "0.8.17", features = ["little_endian", "unaligned"] }
|
||||
serde_json = "1.0"
|
||||
sieve-rs = { version = "1.0", features = ["rkyv", "serde"] }
|
||||
sieve-rs = { version = "1.0" }
|
||||
smtp-proto = { version = "0.2.3", features = ["rkyv"] }
|
||||
xxhash-rust = { version = "0.8.18", features = ["xxh3"] }
|
||||
zstd = { version = "0.13", features = ["zdict_builder", "experimental"] }
|
||||
|
||||
[patch.crates-io]
|
||||
hickory-proto = { git = "https://github.com/hickory-dns/hickory-dns", rev = "e645086f7863eaab8d961421a348b06ea613ebd3" }
|
||||
hickory-net = { git = "https://github.com/hickory-dns/hickory-dns", rev = "e645086f7863eaab8d961421a348b06ea613ebd3" }
|
||||
hickory-resolver = { git = "https://github.com/hickory-dns/hickory-dns", rev = "e645086f7863eaab8d961421a348b06ea613ebd3" }
|
||||
sieve-rs = { path = "../../../sieve" }
|
||||
|
||||
[profile.release]
|
||||
|
||||
@@ -8,10 +8,7 @@ use calcard::{
|
||||
ICalendarValue, Uri,
|
||||
},
|
||||
};
|
||||
use common::DavName;
|
||||
use groupware::calendar::{
|
||||
CalendarEvent, CalendarEventData, EVENT_DRAFT, EVENT_INVITE_OTHERS, EventPreferences,
|
||||
};
|
||||
use groupware::calendar::{CalendarEventContent, CalendarEventData, EventPreferences};
|
||||
|
||||
use crate::corpus::{Corpus, Rng, Stats, archive, collect_files, scrub};
|
||||
|
||||
@@ -63,15 +60,14 @@ pub fn build(dir: &Path, keep_text: bool, stats: &mut Stats) -> std::io::Result<
|
||||
match parser.entry() {
|
||||
Entry::ICalendar(real) if !real.components.is_empty() => {
|
||||
seed += 1;
|
||||
let size = real.to_string().len();
|
||||
if keep_text {
|
||||
corpus.push_both(sample(&mut Rng::new(seed), real, size));
|
||||
corpus.push_both(sample(&mut Rng::new(seed), real));
|
||||
} else {
|
||||
let mut scrubbed = real.clone();
|
||||
scrub_ical(&mut Rng::new(seed ^ SCRUB_SEED), &mut scrubbed);
|
||||
corpus.push(
|
||||
sample(&mut Rng::new(seed), scrubbed, size),
|
||||
sample(&mut Rng::new(seed), real, size),
|
||||
sample(&mut Rng::new(seed), scrubbed),
|
||||
sample(&mut Rng::new(seed), real),
|
||||
);
|
||||
}
|
||||
stats.read += 1;
|
||||
@@ -90,29 +86,17 @@ pub fn build(dir: &Path, keep_text: bool, stats: &mut Stats) -> std::io::Result<
|
||||
Ok(corpus)
|
||||
}
|
||||
|
||||
fn sample(rng: &mut Rng, ical: ICalendar, size: usize) -> Vec<u8> {
|
||||
archive(&event(rng, ical, size))
|
||||
fn sample(rng: &mut Rng, ical: ICalendar) -> Vec<u8> {
|
||||
archive(&event(rng, ical))
|
||||
}
|
||||
|
||||
fn event(rng: &mut Rng, ical: ICalendar, size: usize) -> CalendarEvent {
|
||||
let name = ical
|
||||
.uids()
|
||||
.next()
|
||||
.map(|uid| uid.to_string())
|
||||
.unwrap_or_else(|| rng.token(36));
|
||||
fn event(rng: &mut Rng, ical: ICalendar) -> CalendarEventContent {
|
||||
let mut next_email_alarm = None;
|
||||
let mut data =
|
||||
CalendarEventData::new(ical, Tz::Floating, MAX_EXPANSIONS, &mut next_email_alarm);
|
||||
stabilise(&mut data);
|
||||
let created = timestamp(rng);
|
||||
let display_name_len = rng.range(6, 24);
|
||||
|
||||
CalendarEvent {
|
||||
names: vec![DavName {
|
||||
name: format!("{name}.ics"),
|
||||
parent_id: rng.below(4) as u32,
|
||||
}],
|
||||
display_name: rng.chance(20).then(|| rng.token(display_name_len)),
|
||||
CalendarEventContent {
|
||||
data,
|
||||
preferences: if rng.chance(15) {
|
||||
vec![EventPreferences {
|
||||
@@ -124,18 +108,7 @@ fn event(rng: &mut Rng, ical: ICalendar, size: usize) -> CalendarEvent {
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
flags: if rng.chance(10) {
|
||||
EVENT_DRAFT
|
||||
} else if rng.chance(30) {
|
||||
EVENT_INVITE_OTHERS
|
||||
} else {
|
||||
0
|
||||
},
|
||||
dead_properties: Default::default(),
|
||||
size: size as u32,
|
||||
created,
|
||||
modified: created + rng.below(86400 * 30) as i64,
|
||||
schedule_tag: rng.chance(20).then(|| rng.below(64) as u32),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,10 +122,6 @@ fn stabilise(data: &mut CalendarEventData) {
|
||||
data.alarms = alarms.into_boxed_slice();
|
||||
}
|
||||
|
||||
fn timestamp(rng: &mut Rng) -> i64 {
|
||||
1_750_000_000 + rng.below(86400 * 365) as i64
|
||||
}
|
||||
|
||||
fn scrub_ical(rng: &mut Rng, ical: &mut ICalendar) {
|
||||
for component in ical.components.iter_mut() {
|
||||
for entry in component.entries.iter_mut() {
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
use serde_json::Value;
|
||||
use sieve::Sieve;
|
||||
|
||||
use crate::corpus::Rng;
|
||||
|
||||
const MAX_KEPT_LEN: usize = 64;
|
||||
@@ -36,27 +33,158 @@ const MAILBOXES: [&str; 8] = [
|
||||
"Junk Mail",
|
||||
];
|
||||
|
||||
pub fn sanitize(script: &Sieve, rng: &mut Rng) -> Option<Sieve> {
|
||||
let mut value = serde_json::to_value(script).ok()?;
|
||||
pub fn sanitize(source: &[u8], rng: &mut Rng) -> Option<Vec<u8>> {
|
||||
let source = std::str::from_utf8(source).ok()?;
|
||||
let mut out = String::with_capacity(source.len());
|
||||
let mut replaced = false;
|
||||
let mut rest = source;
|
||||
|
||||
for constant in value.get_mut("constants")?.as_array_mut()? {
|
||||
let text = constant.as_str()?;
|
||||
if is_realistic(text) {
|
||||
continue;
|
||||
while let Some(token) = next_token(rest) {
|
||||
out.push_str(&rest[..token.start]);
|
||||
match token.kind {
|
||||
Token::Comment => out.push_str(&rest[token.start..token.end]),
|
||||
Token::Quoted(literal) => {
|
||||
if is_realistic(&literal) {
|
||||
out.push_str(&rest[token.start..token.end]);
|
||||
} else {
|
||||
out.push('"');
|
||||
for ch in replacement(rng).chars() {
|
||||
if ch == '"' || ch == '\\' {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(ch);
|
||||
}
|
||||
out.push('"');
|
||||
replaced = true;
|
||||
}
|
||||
}
|
||||
Token::MultiLine(body) => {
|
||||
if is_realistic(&body) {
|
||||
out.push_str(&rest[token.start..token.end]);
|
||||
} else {
|
||||
out.push_str("text:\r\n");
|
||||
out.push_str(&replacement(rng));
|
||||
out.push_str("\r\n.\r\n");
|
||||
replaced = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
*constant = Value::String(replacement(rng));
|
||||
replaced = true;
|
||||
rest = &rest[token.end..];
|
||||
}
|
||||
|
||||
if !replaced {
|
||||
return Some(script.clone());
|
||||
if replaced {
|
||||
out.push_str(rest);
|
||||
Some(out.into_bytes())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
enum Token {
|
||||
Comment,
|
||||
Quoted(String),
|
||||
MultiLine(String),
|
||||
}
|
||||
|
||||
struct Located {
|
||||
kind: Token,
|
||||
start: usize,
|
||||
end: usize,
|
||||
}
|
||||
|
||||
fn next_token(source: &str) -> Option<Located> {
|
||||
let bytes = source.as_bytes();
|
||||
let mut at = 0;
|
||||
|
||||
while at < bytes.len() {
|
||||
let start = at;
|
||||
match bytes[at] {
|
||||
b'#' => {
|
||||
let end = source[at..]
|
||||
.find('\n')
|
||||
.map_or(source.len(), |offset| at + offset);
|
||||
return Located::at(Token::Comment, start, end).into();
|
||||
}
|
||||
b'/' if bytes.get(at + 1) == Some(&b'*') => {
|
||||
let end = source[at + 2..]
|
||||
.find("*/")
|
||||
.map_or(source.len(), |offset| at + 2 + offset + 2);
|
||||
return Located::at(Token::Comment, start, end).into();
|
||||
}
|
||||
b'"' => {
|
||||
let (literal, end) = read_quoted(source, at);
|
||||
return Located::at(Token::Quoted(literal), start, end).into();
|
||||
}
|
||||
_ => {
|
||||
if let Some((body, end)) = read_multiline(source, at) {
|
||||
return Located::at(Token::MultiLine(body), start, end).into();
|
||||
}
|
||||
at += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let sanitized: Sieve = serde_json::from_value(value).ok()?;
|
||||
(sanitized.constant_count() == script.constant_count()
|
||||
&& sanitized.instruction_count() == script.instruction_count())
|
||||
.then_some(sanitized)
|
||||
None
|
||||
}
|
||||
|
||||
fn read_quoted(source: &str, at: usize) -> (String, usize) {
|
||||
let bytes = source.as_bytes();
|
||||
let mut literal = String::new();
|
||||
let mut cursor = at + 1;
|
||||
|
||||
while cursor < bytes.len() {
|
||||
match bytes[cursor] {
|
||||
b'\\' if cursor + 1 < bytes.len() => {
|
||||
literal.push_str(&source[cursor + 1..cursor + 2]);
|
||||
cursor += 2;
|
||||
}
|
||||
b'"' => return (literal, cursor + 1),
|
||||
_ => {
|
||||
let width = source[cursor..].chars().next().map_or(1, char::len_utf8);
|
||||
literal.push_str(&source[cursor..cursor + width]);
|
||||
cursor += width;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(literal, source.len())
|
||||
}
|
||||
|
||||
fn read_multiline(source: &str, at: usize) -> Option<(String, usize)> {
|
||||
const MARKER: &str = "text:";
|
||||
|
||||
if !source[at..]
|
||||
.get(..MARKER.len())?
|
||||
.eq_ignore_ascii_case(MARKER)
|
||||
|| source[..at]
|
||||
.chars()
|
||||
.next_back()
|
||||
.is_some_and(|ch| ch.is_alphanumeric() || ch == '_')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let opening = at + MARKER.len() + source[at + MARKER.len()..].find('\n')?;
|
||||
let mut body = String::new();
|
||||
let mut cursor = opening + 1;
|
||||
|
||||
for line in source[cursor..].split_inclusive('\n') {
|
||||
let content = line.trim_end_matches(['\r', '\n']);
|
||||
cursor += line.len();
|
||||
if content == "." {
|
||||
return Some((body, cursor));
|
||||
}
|
||||
body.push_str(content.strip_prefix("..").unwrap_or(content));
|
||||
body.push_str("\r\n");
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
impl Located {
|
||||
fn at(kind: Token, start: usize, end: usize) -> Self {
|
||||
Located { kind, start, end }
|
||||
}
|
||||
}
|
||||
|
||||
fn is_realistic(constant: &str) -> bool {
|
||||
@@ -84,3 +212,51 @@ fn replacement(rng: &mut Rng) -> String {
|
||||
_ => rng.pick(&SENTENCES).to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sanitized(source: &str) -> Option<String> {
|
||||
sanitize(source.as_bytes(), &mut Rng::new(1))
|
||||
.map(|output| String::from_utf8(output).expect("utf-8 output"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_realistic_scripts_untouched() {
|
||||
assert_eq!(
|
||||
sanitized("require [\"fileinto\"];\r\nfileinto \"INBOX/Lists\";\r\n"),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaces_junk_literals() {
|
||||
let output = sanitized("fileinto \"aaaaaaaaaaaaaaaa\";\r\n").expect("replaced");
|
||||
assert!(output.starts_with("fileinto \""), "{output}");
|
||||
assert!(output.ends_with("\";\r\n"), "{output}");
|
||||
assert!(!output.contains("aaaa"), "{output}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_quotes_inside_comments() {
|
||||
assert_eq!(sanitized("# \"aaaaaaaaaaaa\"\r\nstop;\r\n"), None);
|
||||
assert_eq!(sanitized("/* \"aaaaaaaaaaaa\" */\r\nstop;\r\n"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_escaped_quotes_in_place() {
|
||||
assert_eq!(sanitized("set \"a\" \"say \\\"hello\\\"\";\r\n"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaces_junk_multiline_blocks() {
|
||||
let output = sanitized(
|
||||
"notify text:\r\nxxxxxxxxxxxxxxxxxxxx\r\nxxxxxxxxxxxxxxxxxxxx\r\n.\r\nstop;\r\n",
|
||||
)
|
||||
.expect("replaced");
|
||||
assert!(output.starts_with("notify text:\r\n"), "{output}");
|
||||
assert!(output.ends_with(".\r\nstop;\r\n"), "{output}");
|
||||
assert!(!output.contains("xxxx"), "{output}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,7 @@ use calcard::{
|
||||
Entry, Parser,
|
||||
vcard::{VCard, VCardParameterName, VCardParameterValue, VCardProperty, VCardValue},
|
||||
};
|
||||
use common::DavName;
|
||||
use groupware::contact::ContactCard;
|
||||
use groupware::contact::ContactCardContent;
|
||||
|
||||
use crate::corpus::{Corpus, Rng, Stats, archive, collect_files, scrub};
|
||||
|
||||
@@ -34,16 +33,12 @@ pub fn build(dir: &Path, keep_text: bool, stats: &mut Stats) -> std::io::Result<
|
||||
match parser.entry() {
|
||||
Entry::VCard(real) if !real.entries.is_empty() => {
|
||||
seed += 1;
|
||||
let size = real.to_string().len();
|
||||
if keep_text {
|
||||
corpus.push_both(archive(&contact(&mut Rng::new(seed), real, size)));
|
||||
corpus.push_both(archive(&contact(real)));
|
||||
} else {
|
||||
let mut scrubbed = real.clone();
|
||||
scrub_vcard(&mut Rng::new(seed ^ SCRUB_SEED), &mut scrubbed);
|
||||
corpus.push(
|
||||
archive(&contact(&mut Rng::new(seed), scrubbed, size)),
|
||||
archive(&contact(&mut Rng::new(seed), real, size)),
|
||||
);
|
||||
corpus.push(archive(&contact(scrubbed)), archive(&contact(real)));
|
||||
}
|
||||
stats.read += 1;
|
||||
found += 1;
|
||||
@@ -61,24 +56,10 @@ pub fn build(dir: &Path, keep_text: bool, stats: &mut Stats) -> std::io::Result<
|
||||
Ok(corpus)
|
||||
}
|
||||
|
||||
fn contact(rng: &mut Rng, card: VCard, size: usize) -> ContactCard {
|
||||
let name = card
|
||||
.uid()
|
||||
.map(|uid| uid.to_string())
|
||||
.unwrap_or_else(|| rng.token(36));
|
||||
let created = 1_750_000_000 + rng.below(86400 * 365) as i64;
|
||||
|
||||
ContactCard {
|
||||
names: vec![DavName {
|
||||
name: format!("{name}.vcf"),
|
||||
parent_id: rng.below(3) as u32,
|
||||
}],
|
||||
display_name: None,
|
||||
fn contact(card: VCard) -> ContactCardContent {
|
||||
ContactCardContent {
|
||||
card,
|
||||
dead_properties: Default::default(),
|
||||
created,
|
||||
modified: created + rng.below(86400 * 30) as i64,
|
||||
size: size as u32,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -55,14 +55,11 @@ pub fn build(dir: &Path, stats: &mut Stats) -> std::io::Result<Corpus> {
|
||||
};
|
||||
|
||||
let mut rng = Rng::new(index as u64 + 1);
|
||||
let Some(script) = constants::sanitize(&script, &mut rng) else {
|
||||
stats.skipped += 1;
|
||||
continue;
|
||||
};
|
||||
let Ok(script) = script.to_bytes() else {
|
||||
stats.skipped += 1;
|
||||
continue;
|
||||
};
|
||||
let script = constants::sanitize(&raw, &mut rng)
|
||||
.and_then(|sanitized| compiler.compile(&sanitized).ok())
|
||||
.filter(|sanitized| sanitized.code_len() == script.code_len())
|
||||
.unwrap_or(script)
|
||||
.to_bytes();
|
||||
|
||||
let name_len = rng.range(4, 20);
|
||||
let sample = archive(&SieveScript {
|
||||
|
||||
@@ -642,13 +642,17 @@ fn metadata(rng: &mut Rng) -> Box<[Metadata]> {
|
||||
}];
|
||||
if rng.chance(50) {
|
||||
entries.push(Metadata::QueueSize {
|
||||
key: rng.domain().into_bytes().into_boxed_slice(),
|
||||
key: quota_key(rng),
|
||||
id: rng.next_u64(),
|
||||
});
|
||||
entries.push(Metadata::QueueCount {
|
||||
key: rng.domain().into_bytes().into_boxed_slice(),
|
||||
key: quota_key(rng),
|
||||
id: rng.next_u64(),
|
||||
});
|
||||
}
|
||||
entries.into_boxed_slice()
|
||||
}
|
||||
|
||||
fn quota_key(rng: &mut Rng) -> u128 {
|
||||
(u128::from(rng.next_u64()) << 64) | u128::from(rng.next_u64())
|
||||
}
|
||||
|
||||
@@ -47,14 +47,9 @@ pub fn build(compiler: &Compiler, corpus: &mut Corpus, stats: &mut Stats) {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Ok(script) = script.to_bytes() else {
|
||||
stats.skipped += 1;
|
||||
continue;
|
||||
};
|
||||
|
||||
object.blob_hash = BlobHash::generate(&source);
|
||||
object.size = source.len() as u32;
|
||||
object.script = script;
|
||||
object.script = script.to_bytes();
|
||||
|
||||
corpus.push_both(archive(&object));
|
||||
stats.read += 1;
|
||||
|
||||
Binary file not shown.
@@ -60,7 +60,7 @@ pub async fn test(test: &TestServer) {
|
||||
// Await vacation response
|
||||
assert_message_delivery(
|
||||
&mut smtp_rx,
|
||||
MockMessage::new("<jdoe@example.com>", ["<bill@remote.org>"], "@Kokomo"),
|
||||
MockMessage::new("<>", ["<bill@remote.org>"], "@Kokomo"),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -149,7 +149,7 @@ pub async fn test(test: &TestServer) {
|
||||
|
||||
assert_message_delivery(
|
||||
&mut smtp_rx,
|
||||
MockMessage::new("<jdoe@example.com>", ["<jane_smith@remote.org>"], "@Kokomo"),
|
||||
MockMessage::new("<>", ["<jane_smith@remote.org>"], "@Kokomo"),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user