mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 16:19:13 +00:00
666 lines
25 KiB
Go
666 lines
25 KiB
Go
package cloudlink
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/mail"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/rs/zerolog/log"
|
|
|
|
"github.com/warmbly/warmbly/internal/app/cipher"
|
|
"github.com/warmbly/warmbly/internal/config"
|
|
"github.com/warmbly/warmbly/internal/errx"
|
|
"github.com/warmbly/warmbly/internal/events"
|
|
"github.com/warmbly/warmbly/internal/models"
|
|
"github.com/warmbly/warmbly/internal/repository"
|
|
)
|
|
|
|
// Pump pacing. A lease also tells Cloud this instance is alive, so the
|
|
// interval stays well inside models.NodeLivenessWindow.
|
|
const (
|
|
commandPumpInterval = 30 * time.Second
|
|
commandLeaseLimit = 50
|
|
commandClaimLease = 2 * time.Minute
|
|
commandRetryAfter = time.Minute
|
|
// commandAnswerGrace is how long past NotAfter an action may still report its outcome.
|
|
commandAnswerGrace = 15 * time.Minute
|
|
// ownMoveWindow hides a removal caused by an action this instance just ran.
|
|
ownMoveWindow = 15 * time.Minute
|
|
reportBatch = 100
|
|
reportMaxDelay = time.Hour
|
|
commandKeepFor = 7 * 24 * time.Hour
|
|
// messageKeepFor outlasts the longest warmup retention, after which Cloud has had the message deleted.
|
|
messageKeepFor = (config.RetentionDaysMax + 30) * 24 * time.Hour
|
|
pruneEvery = time.Hour
|
|
maxErrorMessage = 200
|
|
)
|
|
|
|
// Reasons an instance refuses a command, reported to Cloud as error_code.
|
|
const (
|
|
refuseMailboxNotExecuted = "mailbox_not_executed_here"
|
|
refuseMailboxInactive = "mailbox_inactive"
|
|
refuseWorkerOffline = "worker_offline"
|
|
refuseInvalidCommand = "invalid_command"
|
|
refuseUnknownHandle = "unknown_message_handle"
|
|
refuseNoActions = "no_actions_permitted"
|
|
)
|
|
|
|
type execAccounts interface {
|
|
GetByID(ctx context.Context, id uuid.UUID) (*models.Email, *errx.Error)
|
|
}
|
|
|
|
type execPublisher interface {
|
|
PublishSendEmail(ctx context.Context, workerID uuid.UUID, params *events.SendEmailParams) error
|
|
PublishWarmupAction(ctx context.Context, workerID uuid.UUID, action *models.WarmupEmailAction) error
|
|
}
|
|
|
|
type execAdmission interface {
|
|
ReserveOutbound(ctx context.Context, in repository.OutboundReservation) (uuid.UUID, error)
|
|
CancelOutbound(ctx context.Context, task, mailbox, worker, nonce uuid.UUID) error
|
|
}
|
|
|
|
type execLiveness interface {
|
|
IsWorkerLive(ctx context.Context, workerID uuid.UUID) (bool, error)
|
|
}
|
|
|
|
// payloadSealer keeps a command's content sealed with the workspace key at rest.
|
|
type payloadSealer interface {
|
|
Seal(ctx context.Context, orgID uuid.UUID, plain string) (string, error)
|
|
Open(ctx context.Context, orgID uuid.UUID, sealed string) (string, error)
|
|
}
|
|
|
|
type cipherSealer struct{ c cipher.CipherService }
|
|
|
|
func (s cipherSealer) Seal(ctx context.Context, orgID uuid.UUID, plain string) (string, error) {
|
|
c, err := s.c.Cipher(ctx, orgID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return c.Encrypt(ctx, plain)
|
|
}
|
|
|
|
func (s cipherSealer) Open(ctx context.Context, orgID uuid.UUID, sealed string) (string, error) {
|
|
c, err := s.c.Cipher(ctx, orgID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return c.Decrypt(ctx, sealed)
|
|
}
|
|
|
|
// ExecutionDeps is what the command pump needs from the consumer process.
|
|
type ExecutionDeps struct {
|
|
Repo repository.CloudLinkExecRepository
|
|
Publisher events.Publisher
|
|
Admission repository.OutboundAdmissionRepository
|
|
Liveness execLiveness
|
|
Cipher cipher.CipherService
|
|
}
|
|
|
|
// Execution runs Cloud's warmup for mailboxes this instance keeps the only
|
|
// credential to: it leases commands, executes them through the instance's
|
|
// own worker, reports their outcomes, and reports warmup mail that arrives.
|
|
type Execution struct {
|
|
svc *service
|
|
repo repository.CloudLinkExecRepository
|
|
accounts execAccounts
|
|
publisher execPublisher
|
|
admission execAdmission
|
|
liveness execLiveness
|
|
sealer payloadSealer
|
|
now func() time.Time
|
|
lastPrune time.Time
|
|
}
|
|
|
|
// NewExecution returns nil when the service is not this package's own.
|
|
func NewExecution(svc Service, deps ExecutionDeps) *Execution {
|
|
s, ok := svc.(*service)
|
|
if !ok || deps.Repo == nil || deps.Publisher == nil || deps.Admission == nil {
|
|
return nil
|
|
}
|
|
s.WireExecution(deps.Repo)
|
|
return &Execution{svc: s, repo: deps.Repo, accounts: s.emails, publisher: deps.Publisher, admission: deps.Admission,
|
|
liveness: deps.Liveness, sealer: cipherSealer{deps.Cipher}, now: time.Now}
|
|
}
|
|
|
|
// Run pumps commands and reports until ctx ends.
|
|
func (x *Execution) Run(ctx context.Context) {
|
|
if x == nil {
|
|
return
|
|
}
|
|
ticker := time.NewTicker(commandPumpInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
x.Pump(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// Pump runs one pass: lease, execute, settle, deliver.
|
|
func (x *Execution) Pump(ctx context.Context) {
|
|
pass, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancel()
|
|
x.leaseAll(pass)
|
|
x.executeDue(pass)
|
|
x.settleAdmitted(pass)
|
|
x.settleAbandoned(pass)
|
|
x.deliverReports(pass)
|
|
if x.now().Sub(x.lastPrune) > pruneEvery {
|
|
if err := x.repo.Prune(pass, commandKeepFor, messageKeepFor); err != nil {
|
|
log.Warn().Err(err).Msg("cloud commands: prune failed")
|
|
}
|
|
x.lastPrune = x.now()
|
|
}
|
|
}
|
|
|
|
func (x *Execution) leaseAll(ctx context.Context) {
|
|
links, err := x.svc.repo.ListLinks(ctx)
|
|
if err != nil {
|
|
log.Warn().Err(err).Msg("cloud commands: links could not be listed")
|
|
return
|
|
}
|
|
for i := range links {
|
|
l := &links[i]
|
|
if l.DisconnectPending {
|
|
continue
|
|
}
|
|
if active, err := x.repo.HasInstanceExecuted(ctx, l.InstanceID); err != nil || !active {
|
|
continue
|
|
}
|
|
info, xerr := x.svc.selfExecution(ctx, l, false)
|
|
if xerr != nil || !selfExecutionSupported(info) {
|
|
continue
|
|
}
|
|
var res models.PoolLinkLeaseResponse
|
|
if xerr := x.svc.clientFor(l).do(ctx, http.MethodPost, "/instance/commands/lease", models.PoolLinkLeaseRequest{Limit: commandLeaseLimit}, &res); xerr != nil {
|
|
log.Warn().Str("instance_id", l.InstanceID.String()).Str("code", xerr.Identifier).Msg("cloud commands: lease failed")
|
|
continue
|
|
}
|
|
for _, cmd := range res.Commands {
|
|
if err := x.store(ctx, l, cmd); err != nil {
|
|
log.Warn().Err(err).Str("command_id", cmd.ID.String()).Msg("cloud commands: command could not be stored")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// store keeps a leased command sealed; a replayed lease stores nothing new.
|
|
func (x *Execution) store(ctx context.Context, l *models.CloudLink, cmd models.PoolLinkCommand) error {
|
|
if cmd.ID == uuid.Nil {
|
|
return errors.New("command without id")
|
|
}
|
|
m, err := x.repo.MailboxByRemote(ctx, l.InstanceID, cmd.RemoteID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if m == nil || m.Executor.OrCloud() != models.PoolLinkExecutorInstance || m.EnrollmentState == "pending_remove" {
|
|
// Nothing to keep it against; Cloud re-leases it if this answer is lost.
|
|
if xerr := x.post(ctx, l, "/instance/commands/"+cmd.ID.String()+"/result", refused(refuseMailboxNotExecuted, "This instance does not run warmup for that mailbox."), nil); xerr != nil && !reportSettled(xerr) {
|
|
return xerr
|
|
}
|
|
return nil
|
|
}
|
|
acc, xerr := x.accounts.GetByID(ctx, m.EmailAccountID)
|
|
if xerr != nil || acc == nil || acc.OrganizationID == nil {
|
|
return errors.New("enrolled mailbox unreadable")
|
|
}
|
|
raw, err := json.Marshal(cmd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sealed, err := x.sealer.Seal(ctx, *acc.OrganizationID, string(raw))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
stored := repository.CloudCommand{ID: cmd.ID, InstanceID: l.InstanceID, EmailAccountID: m.EmailAccountID, Kind: cmd.Kind, Payload: sealed, NotAfter: cmd.NotAfter}
|
|
if cmd.Kind == models.PoolLinkCommandAction && cmd.Action != nil {
|
|
stored.Actions = cmd.Action.Actions
|
|
stored.RFCMessageID = cmd.Action.RFCMessageID
|
|
if h, err := uuid.Parse(cmd.Action.MessageHandle); err == nil {
|
|
stored.MessageHandle = &h
|
|
}
|
|
}
|
|
if cmd.Kind != models.PoolLinkCommandSend && cmd.Kind != models.PoolLinkCommandAction {
|
|
// Stored so it is answered once, as a kind this build does not run.
|
|
stored.Kind = models.PoolLinkCommandAction
|
|
}
|
|
_, err = x.repo.StoreCommand(ctx, stored)
|
|
return err
|
|
}
|
|
|
|
func (x *Execution) executeDue(ctx context.Context) {
|
|
claimed, err := x.repo.ClaimCommands(ctx, commandLeaseLimit, commandClaimLease)
|
|
if err != nil {
|
|
log.Warn().Err(err).Msg("cloud commands: claim failed")
|
|
return
|
|
}
|
|
for _, c := range claimed {
|
|
if err := x.execute(ctx, c); err != nil {
|
|
log.Warn().Err(err).Str("command_id", c.ID.String()).Msg("cloud commands: execution deferred")
|
|
_ = x.repo.RetryCommand(ctx, c.ID, x.now().Add(commandRetryAfter), err.Error())
|
|
}
|
|
}
|
|
}
|
|
|
|
func refused(code, message string) models.PoolLinkCommandResult {
|
|
return models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeRefused, ErrorCode: code, ErrorMessage: message}
|
|
}
|
|
|
|
// execute runs one claimed command; an error leaves it for the next pass.
|
|
func (x *Execution) execute(ctx context.Context, c repository.CloudCommand) error {
|
|
if !x.now().Before(c.NotAfter) {
|
|
if c.Kind == models.PoolLinkCommandSend {
|
|
begun, err := x.releaseUnstartedSend(ctx, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if begun {
|
|
// The worker began it before the deadline; its result answers the command.
|
|
return x.repo.MarkDispatched(ctx, c.ID, c.NotAfter)
|
|
}
|
|
}
|
|
return x.repo.AnswerCommand(ctx, c.ID, models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeExpired})
|
|
}
|
|
acc, xerr := x.accounts.GetByID(ctx, c.EmailAccountID)
|
|
if xerr != nil {
|
|
if xerr.Code == errx.NotFound {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseMailboxNotExecuted, "The mailbox is gone from this instance."))
|
|
}
|
|
return xerr
|
|
}
|
|
if acc.OrganizationID == nil {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseMailboxNotExecuted, "The mailbox belongs to no workspace."))
|
|
}
|
|
m, err := x.svc.repo.GetByAccount(ctx, acc.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if m == nil || m.InstanceID != c.InstanceID || m.Executor.OrCloud() != models.PoolLinkExecutorInstance || m.EnrollmentState == "pending_remove" {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseMailboxNotExecuted, "This instance no longer runs warmup for the mailbox."))
|
|
}
|
|
raw, err := x.sealer.Open(ctx, *acc.OrganizationID, c.Payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var cmd models.PoolLinkCommand
|
|
if err := json.Unmarshal([]byte(raw), &cmd); err != nil || cmd.ID != c.ID {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseInvalidCommand, "The command could not be read."))
|
|
}
|
|
if acc.Status != "active" {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseMailboxInactive, "The mailbox is not active on this instance."))
|
|
}
|
|
if acc.WorkerID == nil {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseWorkerOffline, "No worker on this instance holds the mailbox."))
|
|
}
|
|
if x.liveness != nil {
|
|
live, err := x.liveness.IsWorkerLive(ctx, *acc.WorkerID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !live {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseWorkerOffline, "The worker holding the mailbox is offline."))
|
|
}
|
|
}
|
|
switch {
|
|
case c.Kind == models.PoolLinkCommandSend && cmd.Kind == models.PoolLinkCommandSend && cmd.Send != nil:
|
|
return x.executeSend(ctx, c, cmd, acc)
|
|
case c.Kind == models.PoolLinkCommandAction && cmd.Kind == models.PoolLinkCommandAction && cmd.Action != nil:
|
|
return x.executeAction(ctx, c, cmd, acc)
|
|
}
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseInvalidCommand, "The command kind is not one this instance runs."))
|
|
}
|
|
|
|
// releaseUnstartedSend gives back a send's reservation the worker never began; true when it had begun.
|
|
func (x *Execution) releaseUnstartedSend(ctx context.Context, c repository.CloudCommand) (bool, error) {
|
|
r, err := x.repo.SendReservation(ctx, c.ID)
|
|
if err != nil || r == nil {
|
|
return false, err
|
|
}
|
|
if r.Started && !r.Released {
|
|
return true, nil
|
|
}
|
|
if r.Worker != nil && r.Nonce != nil && !r.Released {
|
|
// Refuses nothing the worker has begun: the release and BeginOutbound share one lock.
|
|
if err := x.admission.CancelOutbound(ctx, c.ID, c.EmailAccountID, *r.Worker, *r.Nonce); err != nil {
|
|
return false, err
|
|
}
|
|
if r, err = x.repo.SendReservation(ctx, c.ID); err != nil {
|
|
return false, err
|
|
}
|
|
if r != nil && r.Started && !r.Released {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, x.repo.CancelSendTask(ctx, c.ID)
|
|
}
|
|
|
|
// validSend returns the one bare recipient of a send whose envelope cannot inject headers.
|
|
func validSend(s *models.PoolLinkSendCommand) (string, bool) {
|
|
if s == nil || s.MessageID == "" || s.WarmupToken == "" || s.BodyPlain == "" {
|
|
return "", false
|
|
}
|
|
for _, v := range append([]string{s.To, s.Subject, s.MessageID, s.InReplyTo, s.ThreadID, s.WarmupToken, s.FromName}, s.References...) {
|
|
if strings.ContainsAny(v, "\r\n\x00") {
|
|
return "", false
|
|
}
|
|
}
|
|
addr, err := mail.ParseAddress(s.To)
|
|
if err != nil || addr.Address == "" || strings.Contains(s.To, ",") {
|
|
return "", false
|
|
}
|
|
return addr.Address, true
|
|
}
|
|
|
|
func (x *Execution) executeSend(ctx context.Context, c repository.CloudCommand, cmd models.PoolLinkCommand, acc *models.Email) error {
|
|
s := cmd.Send
|
|
to, ok := validSend(s)
|
|
if !ok {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseInvalidCommand, "The send command is malformed."))
|
|
}
|
|
task, err := x.repo.PrepareSendTask(ctx, c.ID, acc.ID, s.MessageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if task.Released || (task.Nonce == nil && task.Status != "active") {
|
|
return x.repo.AnswerCommand(ctx, c.ID, models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeFailed, ErrorCode: "send_released", ErrorMessage: "The send was released before it ran."})
|
|
}
|
|
nonce := task.Nonce
|
|
if nonce == nil {
|
|
n, err := x.admission.ReserveOutbound(ctx, repository.OutboundReservation{
|
|
TaskID: c.ID, MailboxID: acc.ID, OrganizationID: *acc.OrganizationID, WorkerID: *acc.WorkerID,
|
|
Provider: models.InboxProvider(acc.Provider), Recipients: []string{to},
|
|
})
|
|
if errors.Is(err, repository.ErrRecipientSuppressed) {
|
|
if cerr := x.repo.CancelSendTask(ctx, c.ID); cerr != nil {
|
|
return cerr
|
|
}
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(strings.ToLower(repository.SendAdmissionCode(err)), "The recipient is on this workspace's suppression list."))
|
|
}
|
|
if errors.Is(err, repository.ErrSendAdmissionDenied) {
|
|
// Another send's hold or a full window can clear before the deadline; expiry answers it otherwise.
|
|
return x.repo.RetryCommand(ctx, c.ID, x.now().Add(commandRetryAfter), strings.ToLower(repository.SendAdmissionCode(err)))
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
nonce = &n
|
|
}
|
|
fromName := strings.TrimSpace(acc.Name)
|
|
if fromName == "" {
|
|
fromName = acc.Email
|
|
}
|
|
params := &events.SendEmailParams{
|
|
TaskID: c.ID, EmailID: acc.ID, OrgID: *acc.OrganizationID,
|
|
To: []string{to}, InReplyTo: s.InReplyTo, References: s.References, ThreadID: s.ThreadID,
|
|
DispatchNonce: nonce.String(), Subject: s.Subject, MessageID: s.MessageID, BodyPlain: s.BodyPlain,
|
|
IsWarmup: true, WarmupToken: s.WarmupToken, FromName: fromName,
|
|
}
|
|
// A re-claimed command republishes the same reservation; the worker runs it at most once.
|
|
if err := x.publisher.PublishSendEmail(ctx, *acc.WorkerID, params); err != nil {
|
|
return fmt.Errorf("publish send: %w", err)
|
|
}
|
|
return x.repo.MarkDispatched(ctx, c.ID, c.NotAfter)
|
|
}
|
|
|
|
// knownActions are the warmup actions a worker runs; anything else is dropped.
|
|
var knownActions = []string{
|
|
models.WarmupActionFile, models.WarmupActionRescueFromSpam, models.WarmupActionMarkRead, models.WarmupActionMarkImportant,
|
|
models.WarmupActionStar, models.WarmupActionDelete, models.WarmupActionVerifyRemoval,
|
|
}
|
|
|
|
func (x *Execution) executeAction(ctx context.Context, c repository.CloudCommand, cmd models.PoolLinkCommand, acc *models.Email) error {
|
|
a := cmd.Action
|
|
var actions []string
|
|
for _, act := range a.Actions {
|
|
if slices.Contains(knownActions, act) && !slices.Contains(actions, act) {
|
|
actions = append(actions, act)
|
|
}
|
|
}
|
|
if len(actions) == 0 {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseNoActions, "The command named no action this instance runs."))
|
|
}
|
|
placement, folder := acc.WarmupFiling()
|
|
if a.Placement != "" {
|
|
placement, folder = a.Placement, a.TargetFolder
|
|
}
|
|
userID, _ := uuid.Parse(acc.UserID)
|
|
action := models.WarmupEmailAction{UserID: userID, EmailID: acc.ID, RFCMessageID: a.RFCMessageID, Actions: actions,
|
|
Placement: placement, TargetFolder: folder, Recheck: a.Recheck}
|
|
if a.MessageHandle != "" {
|
|
h, err := uuid.Parse(a.MessageHandle)
|
|
if err != nil {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseUnknownHandle, "The message handle is not one this instance issued."))
|
|
}
|
|
msg, err := x.repo.WarmupMessageByHandle(ctx, acc.ID, h)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if msg == nil {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseUnknownHandle, "The message handle is not one this instance issued."))
|
|
}
|
|
action.UserID, action.GmailID, action.UID, action.MailboxUIDValidity = msg.UserID, msg.GmailID, msg.UID, msg.UIDValidity
|
|
action.MailboxFolder, action.RFCMessageID, action.InternalID = msg.FolderPath, msg.RFCMessageID, msg.InternalID.String()
|
|
} else {
|
|
// Without a handle the only message Cloud may name is this mailbox's own copy of a send it executed.
|
|
if a.RFCMessageID == "" || strings.ContainsAny(a.RFCMessageID, "\r\n\x00") {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseUnknownHandle, "The action names no message this instance reported or sent."))
|
|
}
|
|
own, err := x.repo.IsOwnCloudSend(ctx, acc.ID, a.RFCMessageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !own {
|
|
return x.repo.AnswerCommand(ctx, c.ID, refused(refuseUnknownHandle, "The action names no message this instance reported or sent."))
|
|
}
|
|
}
|
|
action.CommandID = c.ID.String()
|
|
// The worker's admission window opens before the action can reach it, and only once.
|
|
began, err := x.repo.BeginAction(ctx, c.ID, c.NotAfter)
|
|
if err != nil || !began {
|
|
return err
|
|
}
|
|
if err := x.publisher.PublishWarmupAction(ctx, *acc.WorkerID, &action); err != nil {
|
|
if rerr := x.repo.RevertAction(ctx, c.ID); rerr != nil {
|
|
return errors.Join(fmt.Errorf("publish action: %w", err), rerr)
|
|
}
|
|
return fmt.Errorf("publish action: %w", err)
|
|
}
|
|
// Answered once the worker was admitted to run it (settleAdmitted), or by its own report.
|
|
return nil
|
|
}
|
|
|
|
// settleAdmitted answers done for actions the worker was admitted to run and that report nothing back.
|
|
func (x *Execution) settleAdmitted(ctx context.Context) {
|
|
admitted, err := x.repo.AdmittedActions(ctx, commandLeaseLimit)
|
|
if err != nil {
|
|
log.Warn().Err(err).Msg("cloud commands: admitted actions could not be listed")
|
|
return
|
|
}
|
|
for _, c := range admitted {
|
|
if err := x.repo.AnswerCommand(ctx, c.ID, models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeDone}); err != nil {
|
|
log.Warn().Err(err).Str("command_id", c.ID.String()).Msg("cloud commands: admitted action not answered")
|
|
}
|
|
}
|
|
}
|
|
|
|
// settleAbandoned answers commands whose local outcome is decided without a worker answer.
|
|
func (x *Execution) settleAbandoned(ctx context.Context) {
|
|
stuck, err := x.repo.AbandonedCommands(ctx, commandAnswerGrace, commandLeaseLimit)
|
|
if err != nil {
|
|
log.Warn().Err(err).Msg("cloud commands: abandoned commands could not be listed")
|
|
return
|
|
}
|
|
for _, c := range stuck {
|
|
result, settled, err := x.abandonedResult(ctx, c)
|
|
if err != nil || !settled {
|
|
if err != nil {
|
|
log.Warn().Err(err).Str("command_id", c.ID.String()).Msg("cloud commands: abandoned command not settled")
|
|
}
|
|
continue
|
|
}
|
|
if err := x.repo.AnswerCommand(ctx, c.ID, result); err != nil {
|
|
log.Warn().Err(err).Str("command_id", c.ID.String()).Msg("cloud commands: abandoned command not answered")
|
|
}
|
|
}
|
|
}
|
|
|
|
// abandonedResult decides what an unanswered command came to; settled is false while a begun send may still report.
|
|
func (x *Execution) abandonedResult(ctx context.Context, c repository.CloudCommand) (models.PoolLinkCommandResult, bool, error) {
|
|
if c.Kind != models.PoolLinkCommandSend {
|
|
if !c.Admitted {
|
|
return models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeFailed, ErrorCode: "not_admitted", ErrorMessage: "This instance's worker did not accept the action."}, true, nil
|
|
}
|
|
return models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeFailed, ErrorCode: "no_worker_answer", ErrorMessage: "The worker did not report what became of the action."}, true, nil
|
|
}
|
|
begun, err := x.releaseUnstartedSend(ctx, c)
|
|
if err != nil {
|
|
return models.PoolLinkCommandResult{}, false, err
|
|
}
|
|
if !begun {
|
|
return models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeFailed, ErrorCode: "send_released", ErrorMessage: "The send was released before it ran."}, true, nil
|
|
}
|
|
if x.now().Sub(c.NotAfter) < 24*time.Hour {
|
|
return models.PoolLinkCommandResult{}, false, nil
|
|
}
|
|
// A begun send keeps the generic unknown-send path; Cloud is told it has no result.
|
|
return models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeFailed, ErrorCode: "send_unresolved", ErrorMessage: "The send ended without a result from this instance's worker."}, true, nil
|
|
}
|
|
|
|
func (x *Execution) deliverReports(ctx context.Context) {
|
|
reports, err := x.repo.ClaimReports(ctx, reportBatch, commandClaimLease)
|
|
if err != nil {
|
|
log.Warn().Err(err).Msg("cloud commands: outbox claim failed")
|
|
return
|
|
}
|
|
links := map[uuid.UUID]*models.CloudLink{}
|
|
for _, r := range reports {
|
|
l, ok := links[r.InstanceID]
|
|
if !ok {
|
|
l, err = x.svc.repo.GetByInstance(ctx, r.InstanceID)
|
|
if err != nil {
|
|
_ = x.repo.ReportFailed(ctx, r.ID, x.now().Add(commandRetryAfter), "link unreadable")
|
|
continue
|
|
}
|
|
links[r.InstanceID] = l
|
|
}
|
|
if l == nil {
|
|
// The link is gone; nothing is left to tell.
|
|
_ = x.repo.ReportDelivered(ctx, r.ID)
|
|
continue
|
|
}
|
|
var path string
|
|
switch {
|
|
case r.Kind == "result" && r.CommandID != nil:
|
|
path = "/instance/commands/" + r.CommandID.String() + "/result"
|
|
case r.Kind == "observation" && r.RemoteID != nil:
|
|
path = "/instance/mailboxes/" + r.RemoteID.String() + "/warmup-observations"
|
|
default:
|
|
_ = x.repo.ReportDelivered(ctx, r.ID)
|
|
continue
|
|
}
|
|
xerr := x.post(ctx, l, path, r.Body, nil)
|
|
if xerr == nil || reportSettled(xerr) {
|
|
if xerr != nil {
|
|
log.Warn().Str("report_id", r.ID.String()).Str("code", xerr.Identifier).Msg("cloud commands: Cloud declined a report; not retried")
|
|
}
|
|
if err := x.repo.ReportDelivered(ctx, r.ID); err != nil {
|
|
log.Warn().Err(err).Str("report_id", r.ID.String()).Msg("cloud commands: delivered report not recorded")
|
|
}
|
|
continue
|
|
}
|
|
_ = x.repo.ReportFailed(ctx, r.ID, x.now().Add(reportBackoff(r.Attempts)), xerr.Identifier)
|
|
}
|
|
}
|
|
|
|
// reportSettled is a refusal no retry can change: the command is answered, gone, or the body refused.
|
|
func reportSettled(xerr *errx.Error) bool {
|
|
switch xerr.Code {
|
|
case errx.Conflict, errx.NotFound, errx.BadRequest, errx.Unprocessable:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func reportBackoff(attempts int) time.Duration {
|
|
d := 30 * time.Second
|
|
for i := 1; i < attempts && d < reportMaxDelay; i++ {
|
|
d *= 2
|
|
}
|
|
return min(d, reportMaxDelay)
|
|
}
|
|
|
|
func (x *Execution) post(ctx context.Context, l *models.CloudLink, path string, body, out any) *errx.Error {
|
|
return x.svc.clientFor(l).do(ctx, http.MethodPost, path, body, out)
|
|
}
|
|
|
|
// SendResolved answers the Cloud command behind a cloud_warmup task with the worker's outcome.
|
|
func (x *Execution) SendResolved(ctx context.Context, taskID uuid.UUID, result models.SendEmailResult) error {
|
|
if x == nil {
|
|
return nil
|
|
}
|
|
c, err := x.repo.CommandForTask(ctx, taskID)
|
|
if err != nil || c == nil {
|
|
return err
|
|
}
|
|
return x.repo.AnswerCommand(ctx, c.ID, sendOutcome(result))
|
|
}
|
|
|
|
func sendOutcome(result models.SendEmailResult) models.PoolLinkCommandResult {
|
|
if result.Success {
|
|
return models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeSent, MessageID: result.MessageID,
|
|
ProviderMsgID: result.ProviderMsgID, ThreadID: result.ThreadID, SentAt: result.SentAt}
|
|
}
|
|
out := models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeFailed, ErrorCode: "send_failed"}
|
|
if e := result.Error; e != nil {
|
|
if e.Code != "" {
|
|
out.ErrorCode = e.Code
|
|
}
|
|
out.ErrorMessage = e.UserTitle
|
|
if e.Failure != nil {
|
|
out.Permanent = e.Failure.Disposition == errx.SendPermanent || e.Failure.Disposition == errx.SendAuth
|
|
}
|
|
}
|
|
if len(out.ErrorMessage) > maxErrorMessage {
|
|
out.ErrorMessage = strings.ToValidUTF8(out.ErrorMessage[:maxErrorMessage], "")
|
|
}
|
|
return out
|
|
}
|
|
|
|
// RescueResolved answers a remove_from_spam command with what the worker found.
|
|
func (x *Execution) RescueResolved(ctx context.Context, accountID, internalID uuid.UUID, outcome string) error {
|
|
if x == nil || !models.ValidWarmupRescueOutcome(outcome) {
|
|
return nil
|
|
}
|
|
msg, err := x.repo.ReportedWarmupMessage(ctx, accountID, internalID)
|
|
if err != nil || msg == nil {
|
|
return err
|
|
}
|
|
c, err := x.repo.DispatchedAction(ctx, accountID, models.WarmupActionRescueFromSpam, &msg.Handle, "")
|
|
if err != nil || c == nil {
|
|
return err
|
|
}
|
|
return x.repo.AnswerCommand(ctx, c.ID, models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeDone, Rescue: outcome})
|
|
}
|
|
|
|
// RemovalChecked answers a verify_removal command with where the message was found.
|
|
func (x *Execution) RemovalChecked(ctx context.Context, accountID uuid.UUID, rfcMessageID, outcome string) error {
|
|
if x == nil || rfcMessageID == "" {
|
|
return nil
|
|
}
|
|
c, err := x.repo.DispatchedAction(ctx, accountID, models.WarmupActionVerifyRemoval, nil, rfcMessageID)
|
|
if err != nil || c == nil {
|
|
return err
|
|
}
|
|
return x.repo.AnswerCommand(ctx, c.ID, models.PoolLinkCommandResult{Outcome: models.PoolLinkOutcomeDone, Removal: outcome})
|
|
}
|