diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4bbee9ca..0e3ea3289 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,6 +253,52 @@ jobs: go test -race -v -p 1 ./internal/app/consumer -run '^TestLiveAdvancedBounceReconciliationRollbackAndReplayWithOneConnection$' -count=1 + - name: Typed mailbox history, archive checkpoint and real-driver upgrade regressions (PostgreSQL) + env: + WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable + run: | + set -o pipefail + go test -race -v -p 1 ./internal/infrastructure/db -run '^TestLiveMailTransport(CLIStartup|UpgradePreservesHistoryAndRollback)$' -count=1 | tee mailbox-transport-upgrade.log + grep -q '^--- PASS: TestLiveMailTransportCLIStartup ' mailbox-transport-upgrade.log + grep -q '^--- PASS: TestLiveMailTransportUpgradePreservesHistoryAndRollback ' mailbox-transport-upgrade.log + ! grep -q -- '--- SKIP:' mailbox-transport-upgrade.log + go test -race -v -p 1 ./internal/repository ./internal/app/orgtransfer ./internal/app/consumer -run '^TestLive(MailTransport.*|ImportMailTransport.*)$' -count=1 | tee mailbox-transport-runtime.log + for test in TestLiveMailTransportNativeDomainExcludesHistory TestLiveMailTransportImportSerializesWithConfirmedResult TestLiveImportMailTransportCheckpointCountsAndUnchangedRows TestLiveMailTransportConfirmedSendReconcilesOnlySend TestLiveImportMailTransportLateAndSpanningSuccess TestLiveImportMailTransportCredentialOnlyWritesRotateFence TestLiveImportMailTransportExportOmitsGeneration TestLiveImportMailTransportWholeArchiveOmitsGeneration TestLiveMailTransportSnapshotKeepsConfigurationAndGenerationTogether TestLiveMailTransportOperatorConfirmationDoesNotRecover; do + grep -q "^--- PASS: $test " mailbox-transport-runtime.log + done + ! grep -q -- '--- SKIP:' mailbox-transport-runtime.log + grep -q '^--- PASS: TestLiveImportMailTransportWholeArchiveSendResultLockOrder ' mailbox-transport-runtime.log + for logfile in mailbox-transport-upgrade.log mailbox-transport-runtime.log; do + test "$(grep -c '^=== RUN' "$logfile")" -gt 0 + printf '%s RUN=%s PASS=%s SKIP=0\n' "$logfile" "$(grep -c '^=== RUN' "$logfile")" "$(grep -c -- '--- PASS:' "$logfile")" + done + go test -race -v -p 1 ./internal/app/worker/wmail ./internal/app/worker/mailmanager ./internal/app/consumer -run '^Test(MailTransportGeneration.*|SyncState.*)$' -count=1 | tee mailbox-transport-worker.log + grep -q '^--- PASS: TestMailTransportGenerationRequiresFreshCompletedSyncPass ' mailbox-transport-worker.log + grep -q '^--- PASS: TestMailTransportGenerationReplacesUnchangedCredentialsAndDrains ' mailbox-transport-worker.log + ! grep -q -- '--- SKIP:' mailbox-transport-worker.log + + - name: Typed mailbox JSON and native Avro registry compatibility + env: + SR_URL: http://127.0.0.1:18181 + run: | + set -o pipefail + docker run -d --name mailbox-transport-registry -p 18181:8081 redpandadata/redpanda:v24.3.13 redpanda start --mode dev-container --smp 1 --memory 512M --reserve-memory 0M --node-id 0 --check=false + trap 'docker rm -f mailbox-transport-registry' EXIT + ready=false + for attempt in $(seq 1 60); do + if curl -fsS "$SR_URL/subjects" >/dev/null; then ready=true; break; fi + sleep 1 + done + test "$ready" = true + go test -tags kafka -race -v -p 1 ./internal/infrastructure/codec -run '^TestMailTransportCodec.*Compatibility$' -count=1 | tee mailbox-transport-codec.log + grep -q '^--- PASS: TestMailTransportCodecJSONCompatibility ' mailbox-transport-codec.log + grep -q '^--- PASS: TestMailTransportCodecRegistryCompatibility ' mailbox-transport-codec.log + for testcase in add send sync error; do + grep -q -- "--- PASS: TestMailTransportCodecRegistryCompatibility/$testcase " mailbox-transport-codec.log + done + ! grep -q -- '--- SKIP:' mailbox-transport-codec.log + printf 'codec RUN=%s PASS=%s SKIP=0\n' "$(grep -c '^=== RUN' mailbox-transport-codec.log)" "$(grep -c -- '--- PASS:' mailbox-transport-codec.log)" + - name: Mailbox sync admission, replay, truthful state and upgrade regressions (PostgreSQL) env: WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable diff --git a/docs/content/docs/development/instance-health.mdx b/docs/content/docs/development/instance-health.mdx index 01172135d..15a601d32 100644 --- a/docs/content/docs/development/instance-health.mdx +++ b/docs/content/docs/development/instance-health.mdx @@ -71,6 +71,20 @@ Two neighbouring endpoints round out the surface, both under **Instance** in the The backend collects with an eight-second overall timeout, three seconds per source and at most four concurrent collectors. It shares a one-minute snapshot across admins, then filters source permissions for each response. HTTP responses are not cacheable. On collection failure it can retain a source's last good measurement for at most five minutes, marked stale and with its original observation/evidence clocks. After that it is unavailable. Source samples are bounded; measured/expected scope counts and incomplete coverage expose truncation. Counts in partial samples may be lower bounds, never extrapolated fleet totals. +Unresolved mailbox reports are stored evidence, not proof of an ongoing provider failure. New transport reports explicitly distinguish `sync`, `send` and `unknown` operations. Their `created_at` remains the latest failure observation; `observation_source` distinguishes a valid worker clock from receipt fallback, and `received_at` records backend receipt separately. Missing, invalid or future worker clocks fall back to receipt time. Older redelivery cannot move the latest observation backward. Upgrading does not rewrite historical clocks or guess the operation from a nullable task ID. + +Inbound sync cannot prove sending recovered. A completed provider pass automatically reconciles only earlier, explicitly sync-scoped native transport reports. Provider-confirmed sends independently reconcile earlier native send-scoped reports, with confirmed-task provenance, in the existing send-result transaction. Each operation retains its own monotonic success clock, so delayed older native failures stay resolved history instead of reopening an outage or emitting a fresh warning. Failure timestamps have second precision; recovery must be more than one second later. Credential and policy errors still use their existing recovery flows. + +Transport recovery additionally requires the worker's immutable, instance-local mailbox generation. The control plane reads that generation and the mailbox configuration and credentials in one database snapshot. It is part of the worker's execution identity, so an unchanged-credential restore still drains and replaces the old mailbox transport rather than relabeling it. Only a freshly completed provider sync pass stamps its generation; imported seeds, progress, partial or deferred passes, heartbeats and resumed checkpoints cannot create new proof. Sending captures the loaded transport's generation before execution and retains it in the genuine result. A task execution or dispatch marker alone is not proof of the restored configuration, and cannot establish sync recovery. + +Workspace import rotates this generation and invalidates derived clocks transactionally for actually inserted or overwritten mailboxes, including actual mailbox credential-table writes. Skipped, foreign, rolled-back and unaffected rows keep their generation and clocks. This is a restore-incarnation boundary, not a version for every ordinary settings edit or token refresh. The generation comparison and recovery write lock the same mailbox fence as restore, preserving task-before-mailbox lock order. A late pre-import or spanning result, even with a recent timestamp, cannot recreate current recovery. Genuine send outcomes still settle normally, including accounting and audit; mismatch never discards them or changes restrictive holds. Operator confirmation and imported confirmation history do not authorize transport recovery, even if metadata contains a generation. + +Missing generation remains unproven. Old workers continue to send and sync through upgraded consumers, but cannot advance these derived recovery clocks. New typed workers recover automatically with no customer confirmation, worker re-enrollment or shared-token permission change. JSON and Kafka/Avro fields are additive; the native registry regression checks both directions under FULL_TRANSITIVE compatibility. Upgrade the database and every backend and consumer before workers. Local regression results do not establish production provider recovery. + +Legacy and imported reports with unknown observation provenance stay outside native active-report deduplication and automatic recovery. Their IDs, timestamps, task references and resolutions remain available. Untyped worker events have unknown operation scope, even when their observation clock is valid. Neither sync nor sending alone can automatically clear ambiguous operation evidence. No new customer action is required for new typed reports. Transport recovery does not prove recipient delivery, inbox currentness or permission to replay an uncertain send. Sync passes can complete while the arrival queue is still behind. Unknown-send safety holds are not cleared by report reconciliation. + +Quiesce old mailbox-error writers and result consumers before applying migration 288: old account-plus-code conflict targets are incompatible with the new native unique index. Roll out schema and all backend/result consumers before typed workers. Old workers remain supported through new consumers, but their untyped reports stay unknown. Do not run old result consumers alongside typed workers; old consumers can still apply account-wide resolution. Workspace import resets report operation and observation provenance and invalidates derived destination clocks only for mailboxes actually written. Whole-instance database backups retain the original provenance and clocks. Downgrade refuses any set of unresolved reports that old uniqueness cannot represent, including multiple historical reports of the same operation, rather than discarding evidence or inventing recovery. A refused real-driver downgrade leaves migration metadata dirty and the typed schema intact. Verify the preserved schema and roll forward; do not delete reports or clear safety holds to force a downgrade. + The local task dispatcher also waits five minutes before revisiting a task whose handler returned successfully but left it pending with an already-due schedule. This lets newer due work run without treating the no-op as a failure. It preserves any later retry floor, the original scheduled time, send outcomes and safety holds. Tasks that completed, became active or were rescheduled into the future do not receive this extra wait. A retry wait is not evidence of provider recovery. Failed attempts keep their existing retry policy. During a mixed-version rollout, older local dispatchers can still clear successful no-op waits. Verify dispatch fairness only after every service running the local dispatcher has been upgraded. diff --git a/docs/content/docs/guides/workspace-export-import.mdx b/docs/content/docs/guides/workspace-export-import.mdx index d127702ae..223618fd5 100644 --- a/docs/content/docs/guides/workspace-export-import.mdx +++ b/docs/content/docs/guides/workspace-export-import.mdx @@ -47,6 +47,7 @@ Read the report, pick the groups to apply, and confirm. - Archives are kept for 7 days and then deleted automatically. - The passphrase is never stored. If you lose it, the credentials in that archive cannot be recovered. - An import runs as one transaction. If anything fails, nothing lands. +- Mailbox overwrite preserves report history, send audit and restrictive holds. Recovery is checked automatically against the destination's newly loaded mailbox transport; an old or in-flight result cannot clear restored transport warnings. Skipped mailboxes are unaffected. - To move a whole self-hosted install, use `warmblyctl backup` and `warmblyctl restore` instead. @@ -95,6 +96,10 @@ The pending warmup-filing queue stays on its source instance. The destination re Warmup statistics travel as history. Per-task charge and refund markers reset on import so settling an old task cannot refund another send's destination statistics. Whole-instance database backups retain these markers. +Mailbox transport-error reports travel as history, retaining every report ID, observation timestamp, task reference and recorded resolution. Their operation and observation provenance reset to unknown, including overwrite imports and older archives without those fields. Imported and legacy historical reports do not share the uniqueness rule for new, locally observed reports and are not automatically cleared by a successful sync or send. Unknown history means recovery is unverified, not that a provider is still failing. + +Instance-local transport-success clocks do not travel with an archive. The mailbox transport generation is omitted from exported rows, manifest columns and stored-result/confirmation JSON envelopes; other result and audit evidence remains intact. Importing a new mailbox or overwriting an existing one creates its own destination generation and invalidates only that mailbox's derived transport clocks in the same transaction. Skipped mailboxes and mailboxes not in the archive keep their generation and clocks. This does not clear restrictive send holds, confirmed-task evidence or accounting history. + Send-recovery audit rows also travel as history, including operator confirmations and payload-free contradictory outcome metadata. Their local provenance resets on import. A source-instance confirmation cannot become an executable result or release a conflict hold on the destination. Unknown outcomes stay restrictive until independent authoritative worker results or a fresh local administrator/provider confirmation resolves them; importing an archive never authorizes a resend. Recovery uses available authoritative results automatically, without a new customer sending or connection step. Only unresolved evidence needs administrator verification, not customer log investigation. Move recovery history only to a destination with compatible recovery code and schema. An older destination can reject the new conflict evidence type or misinterpret historical confirmations even when it can read the archive. Do not discard audit rows to force an import or downgrade. See [conflict-audit upgrade ordering](/development/send-safety/#conflict-audit-upgrade-ordering). diff --git a/internal/app/consumer/event_email_error.go b/internal/app/consumer/event_email_error.go index 1e9ab5831..a1ba93c8b 100644 --- a/internal/app/consumer/event_email_error.go +++ b/internal/app/consumer/event_email_error.go @@ -3,6 +3,7 @@ package jobs import ( "context" "errors" + "time" "github.com/google/uuid" "github.com/rs/zerolog/log" @@ -275,21 +276,25 @@ func (s *JobsService) HandleEmailServerError(ctx context.Context, event models.E recorded := true if s.EmailAccountErrorRepository != nil { errorRecord := &repository.CreateEmailAccountError{ - EmailAccountID: emailAccountID, - UserID: userID, - ErrorCode: event.ErrorCode, - Severity: "WARNING", - ResolveMethod: event.ResolveMethod, - Title: event.UserTitle, - Message: event.Message, - UserMessage: ptrString(event.UserMessage), - ActionRequired: ptrString(event.ActionRequired), - TaskID: taskID, + EmailAccountID: emailAccountID, + UserID: userID, + ErrorCode: event.ErrorCode, + Severity: "WARNING", + ResolveMethod: event.ResolveMethod, + Title: event.UserTitle, + Message: event.Message, + UserMessage: ptrString(event.UserMessage), + ActionRequired: ptrString(event.ActionRequired), + TaskID: taskID, + ObservedAt: serverErrorObservedAt(event.Timestamp), + Operation: serverErrorOperation(event.Operation, taskID, event.TaskID != ""), + TransportGeneration: event.TransportGeneration, } row, xerr := s.EmailAccountErrorRepository.CreateOnce(ctx, errorRecord) if xerr != nil { log.Error().Str("error", xerr.Message).Msg("Failed to store server error") + return xerr } recorded = xerr == nil && row != nil } @@ -312,6 +317,21 @@ func (s *JobsService) HandleEmailServerError(ctx context.Context, event models.E return nil } +func serverErrorObservedAt(timestamp int64) *time.Time { + if timestamp <= 0 || timestamp > time.Now().Unix() { + return nil + } + observed := time.Unix(timestamp, 0).UTC() + return &observed +} + +func serverErrorOperation(operation models.MailTransportOperation, taskID *uuid.UUID, taskProvided bool) models.MailTransportOperation { + if operation == models.MailTransportSync && taskProvided || operation == models.MailTransportSend && (taskID == nil || *taskID == uuid.Nil) { + return models.MailTransportUnknown + } + return operation.Conservative() +} + // ptrString returns a pointer to a string, or nil if empty func ptrString(s string) *string { if s == "" { diff --git a/internal/app/consumer/event_email_error_test.go b/internal/app/consumer/event_email_error_test.go index bb6d20a26..2416f811f 100644 --- a/internal/app/consumer/event_email_error_test.go +++ b/internal/app/consumer/event_email_error_test.go @@ -3,6 +3,7 @@ package jobs import ( "context" "testing" + "time" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/errx" @@ -15,16 +16,94 @@ type stubCreateRepo struct { repository.EmailAccountErrorRepository createdOnce int suppress bool + observedAt *time.Time + taskID *uuid.UUID + operation models.MailTransportOperation + fail bool } func (s *stubCreateRepo) CreateOnce(_ context.Context, in *repository.CreateEmailAccountError) (*repository.EmailAccountError, *errx.Error) { s.createdOnce++ + s.observedAt = in.ObservedAt + s.taskID = in.TaskID + s.operation = in.Operation + if s.fail { + return nil, errx.InternalError() + } if s.suppress { return nil, nil } return &repository.EmailAccountError{ErrorCode: in.ErrorCode}, nil } +func TestServerErrorOperationDoesNotInferSyncFromMissingTask(t *testing.T) { + for _, tc := range []struct { + operation models.MailTransportOperation + task string + want models.MailTransportOperation + }{ + {"", "", models.MailTransportUnknown}, + {"", uuid.NewString(), models.MailTransportUnknown}, + {models.MailTransportSync, "", models.MailTransportSync}, + {models.MailTransportSend, uuid.NewString(), models.MailTransportSend}, + {models.MailTransportSync, uuid.NewString(), models.MailTransportUnknown}, + {models.MailTransportSync, "invalid", models.MailTransportUnknown}, + {models.MailTransportSend, "invalid", models.MailTransportUnknown}, + {models.MailTransportSend, uuid.Nil.String(), models.MailTransportUnknown}, + {"future_version", "", models.MailTransportUnknown}, + } { + repo := &stubCreateRepo{} + s := &JobsService{EmailAccountErrorRepository: repo} + if err := s.HandleEmailServerError(t.Context(), models.EmailErrorEvent{ + EmailAccountID: uuid.NewString(), UserID: uuid.NewString(), ErrorCode: "SERVER_UNREACHABLE", + Operation: tc.operation, TaskID: tc.task, + }); err != nil { + t.Fatal(err) + } + if repo.operation != tc.want { + t.Fatalf("operation=%q task=%q: got %q want %q", tc.operation, tc.task, repo.operation, tc.want) + } + } +} + +func TestServerErrorStorageFailureRemainsRetryable(t *testing.T) { + s := &JobsService{EmailAccountErrorRepository: &stubCreateRepo{fail: true}} + if err := s.HandleEmailServerError(t.Context(), models.EmailErrorEvent{ + EmailAccountID: uuid.NewString(), UserID: uuid.NewString(), ErrorCode: "SERVER_UNREACHABLE", + }); err == nil { + t.Fatal("storage failure was acknowledged without preserving report") + } +} + +func TestServerErrorKeepsWorkerOccurrenceWhenResultDeliveryIsLate(t *testing.T) { + repo := &stubCreateRepo{} + s := &JobsService{EmailAccountErrorRepository: repo} + stale := time.Now().Add(-2 * time.Hour).UTC().Truncate(time.Second) + taskID := uuid.New() + if err := s.HandleEmailServerError(context.Background(), models.EmailErrorEvent{ + EmailAccountID: uuid.NewString(), UserID: uuid.NewString(), TaskID: taskID.String(), + ErrorCode: "SERVER_UNREACHABLE", Timestamp: stale.Unix(), + }); err != nil { + t.Fatal(err) + } + if repo.observedAt == nil || !repo.observedAt.Equal(stale) || repo.taskID == nil || *repo.taskID != taskID { + t.Fatalf("delayed send report lost its worker time or task scope: %v %v", repo.observedAt, repo.taskID) + } + if err := s.HandleEmailServerError(context.Background(), models.EmailErrorEvent{ + EmailAccountID: uuid.NewString(), UserID: uuid.NewString(), ErrorCode: "SERVER_UNREACHABLE", + }); err != nil { + t.Fatal(err) + } + if repo.observedAt != nil || repo.taskID != nil { + t.Fatal("legacy event fabricated worker occurrence or send scope") + } + for _, timestamp := range []int64{-1, 0, time.Now().Add(time.Hour).Unix(), 1<<63 - 1} { + if serverErrorObservedAt(timestamp) != nil { + t.Fatalf("invalid worker clock accepted as provider occurrence: %d", timestamp) + } + } +} + // A mail server that refuses the same command on every pass is relayed on // every pass. Recording each one unconditionally filled a mailbox's error // list with an identical row a minute for as long as the refusal lasted diff --git a/internal/app/consumer/event_send_result.go b/internal/app/consumer/event_send_result.go index 4ae088e3e..99f453fa7 100644 --- a/internal/app/consumer/event_send_result.go +++ b/internal/app/consumer/event_send_result.go @@ -51,6 +51,11 @@ func (s *JobsService) applyEmailSent(ctx context.Context, result models.SendEmai log.Warn().Str("task_id", result.TaskID.String()).Msg("email sent result for unknown task") return nil } + if s.EmailAccountErrorRepository != nil && models.ValidMailTransportTime(result.SentAt, time.Now()) { + if err := s.EmailAccountErrorRepository.RecordTransportSuccess(ctx, task.EmailAccountID, models.MailTransportSend, result.SentAt, &result.TaskID, result.TransportGeneration); err != nil { + return err + } + } // The worker reports the Message-ID the provider put on the wire, which is // not always the one the control plane minted: Graph and Gmail re-stamp it. Take the // worker's answer whenever it differs, because everything that matches a diff --git a/internal/app/consumer/event_sync_state.go b/internal/app/consumer/event_sync_state.go index 17d381df4..12bb0ac0c 100644 --- a/internal/app/consumer/event_sync_state.go +++ b/internal/app/consumer/event_sync_state.go @@ -26,6 +26,14 @@ func (s *JobsService) HandleSyncState(ctx context.Context, e *models.JobEventSyn prev = saved } + // Persist recovery before state, so a failed proof write remains retryable. + if e.State.LastSyncedAt != nil && e.State.Deferred == 0 && e.State.BackfillCursor.GoogleRecovery == nil && + (prev == nil || prev.LastSyncedAt == nil || e.State.TransportGeneration != prev.TransportGeneration || !models.ValidMailTransportTime(*prev.LastSyncedAt, time.Now()) || e.State.LastSyncedAt.After(*prev.LastSyncedAt)) { + if err := s.resolveTransientMailErrors(ctx, e.EmailID, e.State.LastSyncedAt, e.State.TransportGeneration); err != nil { + return err + } + } + if err := s.EmailSyncStateRepository.Put(ctx, e.UserID, e.EmailID, &e.State); err != nil { if s.dropForDeletedMailbox(ctx, e.UserID, e.EmailID, err) { return nil @@ -34,12 +42,6 @@ func (s *JobsService) HandleSyncState(ctx context.Context, e *models.JobEventSyn return err } - // Progress relays are not recovery evidence; require a newer successful check. - if e.State.LastSyncedAt != nil && e.State.Deferred == 0 && e.State.BackfillCursor.GoogleRecovery == nil && - (prev == nil || prev.LastSyncedAt == nil || e.State.LastSyncedAt.After(*prev.LastSyncedAt)) { - s.resolveTransientMailErrors(ctx, e.EmailID, e.State.LastSyncedAt) - } - if s.StreamingPublisher == nil { return nil } @@ -106,19 +108,24 @@ var transientMailErrorCodes = []string{ string(errx.MailErrorCodeImapUnknown), } -func (s *JobsService) resolveTransientMailErrors(ctx context.Context, emailID uuid.UUID, syncedAt *time.Time) { +func (s *JobsService) resolveTransientMailErrors(ctx context.Context, emailID uuid.UUID, syncedAt *time.Time, generation uuid.UUID) error { if s.EmailAccountErrorRepository == nil { - return + return nil } // Only errors raised before this pass ran. The bus can redeliver an older // event after a newer one, and without the bound a stale success would // clear a failure that happened after it, leaving the mailbox looking // healthy while it was not. - if syncedAt == nil { - return + if syncedAt == nil || generation == uuid.Nil { + return nil } - if err := s.EmailAccountErrorRepository.ResolveByCodesBefore(ctx, emailID, transientMailErrorCodes, *syncedAt, "sync recovered"); err != nil { + if !models.ValidMailTransportTime(*syncedAt, time.Now()) { + return nil + } + if err := s.EmailAccountErrorRepository.RecordTransportSuccess(ctx, emailID, models.MailTransportSync, *syncedAt, nil, generation); err != nil { log.Warn().Str("error", err.Message).Str("email_id", emailID.String()). Msg("could not clear the mailbox's transient errors after a successful sync") + return err } + return nil } diff --git a/internal/app/consumer/event_sync_state_test.go b/internal/app/consumer/event_sync_state_test.go index 9d077256d..125bfc878 100644 --- a/internal/app/consumer/event_sync_state_test.go +++ b/internal/app/consumer/event_sync_state_test.go @@ -11,6 +11,8 @@ import ( "github.com/warmbly/warmbly/internal/repository" ) +var syncTestGeneration = uuid.New() + // stubSyncStateRepo accepts what the worker relays. type stubSyncStateRepo struct { repository.EmailSyncStateRepository @@ -34,6 +36,8 @@ type stubErrorRepo struct { resolvedBy string resolvedBefore time.Time calls int + operation models.MailTransportOperation + fail bool } func (s *stubErrorRepo) ResolveByCodesBefore(_ context.Context, _ uuid.UUID, codes []string, before time.Time, by string) *errx.Error { @@ -41,9 +45,17 @@ func (s *stubErrorRepo) ResolveByCodesBefore(_ context.Context, _ uuid.UUID, cod s.resolvedCodes = append(s.resolvedCodes, codes...) s.resolvedBefore = before s.resolvedBy = by + if s.fail { + return errx.InternalError() + } return nil } +func (s *stubErrorRepo) RecordTransportSuccess(ctx context.Context, id uuid.UUID, operation models.MailTransportOperation, at time.Time, _ *uuid.UUID, _ uuid.UUID) *errx.Error { + s.operation = operation + return s.ResolveByCodesBefore(ctx, id, transientMailErrorCodes, at, string(operation)+" recovered") +} + // Nothing but a credential reconnect ever resolved an error row, so a mailbox // whose server was briefly unreachable kept a red "needs attention" for good // and never returned to healthy. A relayed sync state means the worker @@ -61,7 +73,7 @@ func TestSyncStateClearsTransientMailErrors(t *testing.T) { if err := s.HandleSyncState(context.Background(), &models.JobEventSyncState{ UserID: uuid.New(), EmailID: emailID, - State: models.SyncState{BackfillStatus: models.SyncBackfillComplete, LastSyncedAt: &syncedAt}, + State: models.SyncState{TransportGeneration: syncTestGeneration, BackfillStatus: models.SyncBackfillComplete, LastSyncedAt: &syncedAt}, }); err != nil { t.Fatalf("HandleSyncState: %v", err) } @@ -73,6 +85,9 @@ func TestSyncStateClearsTransientMailErrors(t *testing.T) { if errRepo.calls != 1 { t.Fatalf("resolved errors %d times, want once per relayed state", errRepo.calls) } + if errRepo.operation != models.MailTransportSync { + t.Fatal("sync recovery did not carry explicit operation scope") + } want := map[string]bool{"SERVER_UNREACHABLE": true, "CONNECTION_LOST": true, "RESOURCE_NOT_FOUND": true, "IMAP_UNKNOWN": true} for _, code := range errRepo.resolvedCodes { if !want[code] { @@ -99,7 +114,7 @@ func TestSyncStateLeavesActionableErrorsAlone(t *testing.T) { if err := s.HandleSyncState(context.Background(), &models.JobEventSyncState{ UserID: uuid.New(), EmailID: uuid.New(), - State: models.SyncState{LastSyncedAt: &syncedAt}, + State: models.SyncState{TransportGeneration: syncTestGeneration, LastSyncedAt: &syncedAt}, }); err != nil { t.Fatalf("HandleSyncState: %v", err) } @@ -138,3 +153,45 @@ func TestSyncStateWithoutATimestampResolvesNothing(t *testing.T) { t.Errorf("resolved errors %d times from a state with no timestamp to bound it by", errRepo.calls) } } + +func TestSyncStateRejectsIncompleteAndInvalidSuccessClocks(t *testing.T) { + old := time.Now().Add(-time.Hour) + future := time.Now().Add(time.Hour) + zero := time.Time{} + for _, state := range []models.SyncState{ + {TransportGeneration: syncTestGeneration, LastSyncedAt: &future}, {TransportGeneration: syncTestGeneration, LastSyncedAt: &zero}, + {TransportGeneration: syncTestGeneration, LastSyncedAt: &old, Deferred: 1}, + {TransportGeneration: syncTestGeneration, LastSyncedAt: &old, BackfillCursor: models.SyncCursor{GoogleRecovery: &models.GoogleHistoryRecovery{}}}, + } { + errRepo := &stubErrorRepo{} + s := &JobsService{EmailSyncStateRepository: &stubSyncStateRepo{}, EmailAccountErrorRepository: errRepo} + if err := s.HandleSyncState(t.Context(), &models.JobEventSyncState{EmailID: uuid.New(), State: state}); err != nil { + t.Fatal(err) + } + if errRepo.calls != 0 { + t.Fatal("incomplete/future state was treated as recovery") + } + } +} + +func TestSyncStateRecoveryRetriesBeforeStateIsStored(t *testing.T) { + at := time.Now().Add(-time.Minute) + older := at.Add(-time.Minute) + errRepo := &stubErrorRepo{fail: true} + stateRepo := &stubSyncStateRepo{saved: &models.SyncState{TransportGeneration: syncTestGeneration, LastSyncedAt: &older}} + s := &JobsService{EmailSyncStateRepository: stateRepo, EmailAccountErrorRepository: errRepo} + event := &models.JobEventSyncState{EmailID: uuid.New(), State: models.SyncState{TransportGeneration: syncTestGeneration, LastSyncedAt: &at}} + if err := s.HandleSyncState(t.Context(), event); err == nil { + t.Fatal("recovery failure was acknowledged") + } + if stateRepo.put != nil { + t.Fatal("state stored before recovery, suppressing an idempotent retry") + } + errRepo.fail = false + if err := s.HandleSyncState(t.Context(), event); err != nil { + t.Fatal(err) + } + if errRepo.calls != 2 { + t.Fatal("recovery failure suppressed retry") + } +} diff --git a/internal/app/consumer/mail_transport_live_test.go b/internal/app/consumer/mail_transport_live_test.go new file mode 100644 index 000000000..0cd7a69b8 --- /dev/null +++ b/internal/app/consumer/mail_transport_live_test.go @@ -0,0 +1,75 @@ +package jobs + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func TestLiveMailTransportConfirmedSendReconcilesOnlySend(t *testing.T) { + for _, rollback := range []bool{false, true} { + t.Run(map[bool]string{false: "confirmed", true: "rollback"}[rollback], func(t *testing.T) { + d := liveDB(t) + s := liveJobsService(d) + s.EmailAccountErrorRepository = repository.NewEmailAccountErrorRepository(d) + f := newSendResultFixture(t, d) + task := f.stampSend(t, s) + var generation uuid.UUID + if err := d.QueryRow(t.Context(), `SELECT transport_generation FROM email_accounts WHERE id=$1`, f.mailbox).Scan(&generation); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-time.Hour) + for _, op := range []models.MailTransportOperation{models.MailTransportUnknown, models.MailTransportSync, models.MailTransportSend} { + data := &repository.CreateEmailAccountError{TransportGeneration: generation, EmailAccountID: f.mailbox, UserID: f.user, Operation: op, ObservedAt: &old, ErrorCode: "SERVER_UNREACHABLE", Severity: "WARNING", ResolveMethod: "RETRY", Title: "Transport", Message: "Transport report"} + if op == models.MailTransportSend { + data.TaskID = &task + } + if _, err := s.EmailAccountErrorRepository.CreateOnce(t.Context(), data); err != nil { + t.Fatal(err) + } + } + result := models.SendEmailResult{TransportGeneration: generation, TaskID: task, SentAt: old.Add(time.Minute), MessageID: ""} + if rollback { + // A real write after recovery fails, forcing the parent reconciliation to roll back. + result.MessageID = string(make([]byte, 256)) + if err := s.HandleEmailSent(t.Context(), result); err == nil { + t.Fatal("invalid database text unexpectedly reconciled") + } + } else if err := s.HandleEmailSent(t.Context(), result); err != nil { + t.Fatal(err) + } + rows, err := s.EmailAccountErrorRepository.GetByAccountID(t.Context(), f.mailbox, true) + if err != nil { + t.Fatal(err) + } + want := 2 + if rollback { + want = 3 + } + if len(rows) != want { + t.Fatalf("unresolved=%d want=%d", len(rows), want) + } + var n int + if err := d.QueryRow(t.Context(), `SELECT count(*) FROM email_account_transport_recovery WHERE email_account_id=$1`, f.mailbox).Scan(&n); err != nil { + t.Fatal(err) + } + if rollback && n != 0 || !rollback && n != 1 { + t.Fatalf("recovery transaction did not settle atomically: %d", n) + } + if !rollback { + var source uuid.UUID + if err := d.QueryRow(t.Context(), `SELECT source_task_id FROM email_account_transport_recovery WHERE email_account_id=$1 AND operation='send'`, f.mailbox).Scan(&source); err != nil || source != task { + t.Fatal("confirmed task provenance missing", err) + } + for _, row := range rows { + if row.Operation == models.MailTransportSend { + t.Fatal("send transport did not recover") + } + } + } + }) + } +} diff --git a/internal/app/consumer/send_recovery_resolution_live_test.go b/internal/app/consumer/send_recovery_resolution_live_test.go index 49e92f41c..9eedf1052 100644 --- a/internal/app/consumer/send_recovery_resolution_live_test.go +++ b/internal/app/consumer/send_recovery_resolution_live_test.go @@ -36,6 +36,11 @@ func TestLiveUnknownSendOperatorResolutionReconcilesAndAudits(t *testing.T) { if _, xerr := repo.Update(ctx, f.org.String(), f.mailbox.String(), &models.UpdateEmail{SendRecoveryResolution: resolution}); xerr != nil { t.Fatal(xerr) } + if sent { + if _, err := h.Pool.Exec(ctx, `UPDATE send_recovery_resolutions r SET confirmed_result=jsonb_set(confirmed_result,'{transport_generation}',to_jsonb(e.transport_generation::text)) FROM email_accounts e WHERE r.recovery_task_id=$1 AND e.id=$2`, id, f.mailbox); err != nil { + t.Fatal(err) + } + } if _, xerr := repo.Update(ctx, f.org.String(), f.mailbox.String(), &models.UpdateEmail{SendRecoveryResolution: resolution}); xerr == nil { t.Fatal("duplicate confirmation overwrote audit") } @@ -45,8 +50,13 @@ func TestLiveUnknownSendOperatorResolutionReconcilesAndAudits(t *testing.T) { } // A replacement consumer replays the durable confirmation without a broker event. replacement := liveJobsService(h) + replacement.EmailAccountErrorRepository = repository.NewEmailAccountErrorRepository(h) replacement.reconcileRecordedSendResults(ctx) replacement.reconcileRecordedSendResults(ctx) + var clocks int + if err := h.QueryRow(ctx, `SELECT COUNT(*) FROM email_account_transport_recovery WHERE email_account_id=$1`, f.mailbox).Scan(&clocks); err != nil || clocks != 0 { + t.Fatal("operator confirmation manufactured transport recovery", clocks, err) + } var state string var applied bool var audits, attempts, counted int @@ -83,6 +93,10 @@ func TestLiveUnknownSendOperatorResolutionReconcilesAndAudits(t *testing.T) { } } +func TestLiveMailTransportOperatorConfirmationDoesNotRecover(t *testing.T) { + TestLiveUnknownSendOperatorResolutionReconcilesAndAudits(t) +} + func TestLiveRecordedWorkerOutcomeSurvivesLostEvent(t *testing.T) { h := liveDB(t) f := newSendResultFixture(t, h) diff --git a/internal/app/email/loader.go b/internal/app/email/loader.go index b66abdb50..83e2949fd 100644 --- a/internal/app/email/loader.go +++ b/internal/app/email/loader.go @@ -328,6 +328,26 @@ func (s *emailService) releaseDeadWorker(ctx context.Context, accountID uuid.UUI // left zero: it is avro-excluded and the worker rebuilds it locally from its own // oauth config. func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Email) (*models.AddWorkerEmail, error) { + if snapshots, ok := s.emailRepository.(repository.EmailTransportSnapshot); ok { + var payload *models.AddWorkerEmail + err := snapshots.WithTransportSnapshot(ctx, func(repo repository.EmailRepository) error { + current, xerr := repo.GetByID(ctx, acc.ID) + if xerr != nil { + return xerr + } + if current == nil || current.Status != "active" { + return nil + } + var err error + payload, err = s.buildAddWorkerEmailSnapshot(ctx, current, repo) + return err + }) + return payload, err + } + return s.buildAddWorkerEmailSnapshot(ctx, acc, s.emailRepository) +} + +func (s *emailService) buildAddWorkerEmailSnapshot(ctx context.Context, acc *models.Email, repo repository.EmailRepository) (*models.AddWorkerEmail, error) { userID, err := uuid.Parse(acc.UserID) if err != nil { return nil, err @@ -341,14 +361,15 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai return nil, err } out := &models.AddWorkerEmail{ - ID: acc.ID, - UserID: userID, - OrganizationID: acc.OrganizationID, - Email: acc.Email, - FirstName: first, - LastName: last, - Type: provider, - Sync: sync, + TransportGeneration: acc.TransportGeneration, + ID: acc.ID, + UserID: userID, + OrganizationID: acc.OrganizationID, + Email: acc.Email, + FirstName: first, + LastName: last, + Type: provider, + Sync: sync, // Only SMTP/IMAP acts on this; Gmail and Graph file their own copy. SaveToSent: &saveToSent, } @@ -363,7 +384,7 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai // A mailbox under an administrator's grant has no stored credential // either; the worker draws tokens the control plane mints per use. if acc.AuthMethod == models.MailAuthDelegated { - d, xerr := s.emailRepository.GetDelegation(ctx, acc.ID) + d, xerr := repo.GetDelegation(ctx, acc.ID) if xerr != nil { return nil, xerr } @@ -400,7 +421,7 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai switch provider { case models.InboxProviderGoogle: - creds, cerr := s.emailRepository.GetOAuthCredentials(ctx, acc.ID) + creds, cerr := repo.GetOAuthCredentials(ctx, acc.ID) if cerr != nil { return nil, cerr } @@ -409,13 +430,13 @@ func (s *emailService) buildAddWorkerEmail(ctx context.Context, acc *models.Emai LastHistoryID: s.lastHistoryFor(ctx, userID, acc.ID, acc.LastID), } case models.InboxProviderOutlook: - creds, cerr := s.emailRepository.GetOAuthCredentials(ctx, acc.ID) + creds, cerr := repo.GetOAuthCredentials(ctx, acc.ID) if cerr != nil { return nil, cerr } out.Graph.Token = oauthToken(creds) case models.InboxProviderSMTPIMAP: - creds, cerr := s.emailRepository.GetSMTPCredentials(ctx, acc.ID) + creds, cerr := repo.GetSMTPCredentials(ctx, acc.ID) if cerr != nil { return nil, cerr } diff --git a/internal/app/orgtransfer/export.go b/internal/app/orgtransfer/export.go index 8719587bd..3b91b2e05 100644 --- a/internal/app/orgtransfer/export.go +++ b/internal/app/orgtransfer/export.go @@ -3,11 +3,13 @@ package orgtransfer import ( "archive/zip" "bufio" + "bytes" "context" "encoding/json" "errors" "fmt" "io" + "slices" "strings" "time" @@ -128,20 +130,24 @@ func (s *service) ExportTo( buf := bufio.NewWriterSize(entry, 256*1024) mt := ManifestTable{ - Name: t.Name, - Group: t.Group, - Columns: columnNames(cols), + Name: t.Name, + Group: t.Group, + Columns: slices.DeleteFunc(columnNames(cols), func(column string) bool { + return slices.Contains(t.ExcludeOnExport, column) + }), } if archiveKey != nil { for _, sc := range t.Secrets { - mt.SecretColumns = append(mt.SecretColumns, sc.Column) + if !slices.Contains(t.ExcludeOnExport, sc.Column) { + mt.SecretColumns = append(mt.SecretColumns, sc.Column) + } } } // Rows only need decoding when something on them has to change. Most // tables have neither secrets nor blobs, so they stream through // untouched, which is what makes a million-row inbox export cheap. - passthrough := len(t.Secrets) == 0 && len(t.Blobs) == 0 + passthrough := len(t.Secrets) == 0 && len(t.Blobs) == 0 && len(t.ExcludeOnExport) == 0 && len(t.ExcludeJSONOnExport) == 0 err = s.repo.StreamScoped(ctx, t.Name, t.Scope, orgID, func(row []byte) error { out := row @@ -186,7 +192,7 @@ func (s *service) ExportTo( return manifest, nil } -// exportRow rewrites one row's secret and blob columns. +// exportRow removes instance-local columns and rewrites portable secrets and blobs. func (s *service) exportRow( ctx context.Context, t *Table, @@ -200,6 +206,22 @@ func (s *service) exportRow( return nil, fmt.Errorf("decode %s row: %w", t.Name, err) } + for _, column := range t.ExcludeOnExport { + delete(obj, column) + } + + for _, exclusion := range t.ExcludeJSONOnExport { + raw, ok := obj[exclusion.Column] + if !ok { + continue + } + stripped, err := excludeExportJSONKeys(raw, exclusion.Keys) + if err != nil { + return nil, fmt.Errorf("exclude %s.%s JSON keys: %w", t.Name, exclusion.Column, err) + } + obj[exclusion.Column] = stripped + } + for _, sc := range t.Secrets { raw, ok := obj[sc.Column] if !ok { @@ -255,6 +277,47 @@ func (s *service) exportRow( return json.Marshal(obj) } +func excludeExportJSONKeys(raw json.RawMessage, keys []string) (json.RawMessage, error) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 { + return raw, nil + } + switch raw[0] { + case '{': + var object map[string]json.RawMessage + if err := json.Unmarshal(raw, &object); err != nil { + return nil, err + } + for key, value := range object { + if slices.Contains(keys, key) { + delete(object, key) + continue + } + stripped, err := excludeExportJSONKeys(value, keys) + if err != nil { + return nil, err + } + object[key] = stripped + } + return json.Marshal(object) + case '[': + var array []json.RawMessage + if err := json.Unmarshal(raw, &array); err != nil { + return nil, err + } + for i, value := range array { + stripped, err := excludeExportJSONKeys(value, keys) + if err != nil { + return nil, err + } + array[i] = stripped + } + return json.Marshal(array) + default: + return raw, nil + } +} + // openSecret returns the plaintext behind one stored value, using whichever // key domain sealed it. func (s *service) openSecret(ctx context.Context, sc SecretColumn, stored string, orgCipher *cipher.Cipher) (string, error) { diff --git a/internal/app/orgtransfer/export_test.go b/internal/app/orgtransfer/export_test.go index 7c5d636c8..dac070185 100644 --- a/internal/app/orgtransfer/export_test.go +++ b/internal/app/orgtransfer/export_test.go @@ -1,11 +1,61 @@ package orgtransfer import ( + "context" "encoding/json" + "reflect" "slices" "testing" ) +func TestMailTransportExportJSONExclusionPreservesEvidence(t *testing.T) { + row := []byte(`{"registered":{"transport_generation":"proof","task_id":"kept","success":false,"message_id":"accepted","error":{"code":"KEPT","message":"transport_generation is quoted text","details":[{"transport_generation":"nested","count":9007199254740993},null,true,"transport_generation"]},"future":[{"result":{"transport_generation":"proof","sent_at":"kept"}}]},"unregistered":{"transport_generation":"user-content"},"historical":"kept"}`) + out, err := (&service{}).exportRow(context.Background(), &Table{Name: "fixture", ExcludeJSONOnExport: []JSONColumnExclusion{{Column: "registered", Keys: []string{"transport_generation"}}}}, row, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + var actual map[string]json.RawMessage + if err := json.Unmarshal(out, &actual); err != nil { + t.Fatal(err) + } + expected := []byte(`{"task_id":"kept","success":false,"message_id":"accepted","error":{"code":"KEPT","message":"transport_generation is quoted text","details":[{"count":9007199254740993},null,true,"transport_generation"]},"future":[{"result":{"sent_at":"kept"}}]}`) + assertExportJSONEqual(t, actual["registered"], expected) + assertExportJSONEqual(t, actual["unregistered"], []byte(`{"transport_generation":"user-content"}`)) + for _, raw := range []string{"null", "[]", `"ordinary scalar"`, "9007199254740993"} { + out, err := excludeExportJSONKeys(json.RawMessage(raw), []string{"transport_generation"}) + if err != nil || string(out) != raw { + t.Fatal("scalar/null/array changed", raw, err) + } + } + for table, column := range map[string]string{"tasks": "send_executor_result", "warmup_tasks": "dispatch_result", "send_recovery_resolutions": "confirmed_result"} { + if !reflect.DeepEqual(TableByName[table].ExcludeJSONOnExport, []JSONColumnExclusion{{Column: column, Keys: []string{"transport_generation"}}}) { + t.Fatal("missing or broad JSON exclusion", table) + } + } +} + +func TestMailTransportExportExclusionIsRegistryDirected(t *testing.T) { + row := []byte(`{"id":"portable","local_only":"do-not-export","worker_id":"source-worker","historical":"preserved"}`) + out, err := (&service{}).exportRow(context.Background(), &Table{Name: "fixture", ExcludeOnExport: []string{"local_only"}, ResetOnImport: []string{"worker_id"}}, row, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + var fields map[string]string + if err := json.Unmarshal(out, &fields); err != nil { + t.Fatal(err) + } + if _, ok := fields["local_only"]; ok { + t.Fatal("configured excluded field exported") + } + if len(fields) != 3 || fields["worker_id"] != "source-worker" || fields["historical"] != "preserved" || fields["id"] != "portable" { + t.Fatal("exclusion changed unrelated/history/reset-on-import fields") + } + mailbox := TableByName["email_accounts"] + if !slices.Equal(mailbox.ExcludeOnExport, []string{"transport_generation"}) || !slices.Contains(mailbox.ResetOnImport, "transport_generation") || !slices.Contains(mailbox.ResetColumnsOnOverwrite, "transport_generation") { + t.Fatal("generation export/import policy incomplete") + } +} + func TestCampaignReplayIntentResetsOnWorkspaceImport(t *testing.T) { for _, tbl := range Tables { if tbl.Name == "campaign_tasks" { diff --git a/internal/app/orgtransfer/import.go b/internal/app/orgtransfer/import.go index 10f84ef8a..c248d6f2d 100644 --- a/internal/app/orgtransfer/import.go +++ b/internal/app/orgtransfer/import.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "slices" "sort" "strings" @@ -260,6 +261,33 @@ func (s *service) importTable( } } } + for _, c := range t.ResetColumnsOnOverwrite { + if writable[c] && reset[c] && !slices.Contains(resetOnOverwrite, c) { + resetOnOverwrite = append(resetOnOverwrite, c) + } + } + var invalidations []repository.ImportInvalidation + for _, relation := range t.InvalidateOnImport { + columns, err := s.repo.TableColumns(ctx, relation.Table) + if err != nil { + return 0, nil, fmt.Errorf("inspect checkpoint relation %s: %w", relation.Table, err) + } + if len(columns) > 0 { + if len(relation.ResetColumns) > 0 { + var available []string + for _, c := range relation.ResetColumns { + if slices.ContainsFunc(columns, func(info repository.ColumnInfo) bool { return info.Name == c }) { + available = append(available, c) + } + } + if len(available) == 0 { + continue + } + relation.ResetColumns = available + } + invalidations = append(invalidations, relation) + } + } insertCols := make([]string, 0, len(mt.Columns)) unknown := 0 @@ -328,7 +356,7 @@ func (s *service) importTable( if foreign > 0 { return fmt.Errorf("%s: %w", t.Name, ErrForeignRows) } - n, err := s.repo.InsertBatch(ctx, tx, t.Name, insertCols, batch, ic.conflict, pk, t.OwnerScope(), orgID, resetOnOverwrite) + n, err := s.repo.InsertBatch(ctx, tx, t.Name, insertCols, batch, ic.conflict, pk, t.OwnerScope(), orgID, resetOnOverwrite, invalidations) if err != nil { return err } diff --git a/internal/app/orgtransfer/mail_transport_credentials_live_test.go b/internal/app/orgtransfer/mail_transport_credentials_live_test.go new file mode 100644 index 000000000..24d1d497b --- /dev/null +++ b/internal/app/orgtransfer/mail_transport_credentials_live_test.go @@ -0,0 +1,53 @@ +package orgtransfer + +import ( + "github.com/warmbly/warmbly/internal/models" + "testing" + "time" +) + +func TestLiveImportMailTransportCredentialOnlyWritesRotateFence(t *testing.T) { + for _, table := range []string{"email_accounts_oauth", "email_accounts_smtp_imap"} { + for _, conflict := range []models.OrgImportConflict{models.OrgImportConflictSkip, models.OrgImportConflictOverwrite} { + t.Run(table+"/"+string(conflict), func(t *testing.T) { + svc, pool, f, mailbox, _, _ := overwriteTransportFixture(t) + row := map[string]any{"email_account_id": mailbox} + switch table { + case "email_accounts_oauth": + row["access_token"], row["refresh_token"], row["expires_at"] = "new-access", "new-refresh", time.Now().Add(time.Hour) + if _, err := pool.Exec(t.Context(), `INSERT INTO email_accounts_oauth(email_account_id,access_token,refresh_token,expires_at) VALUES($1,'old','old',NOW())`, mailbox); err != nil { + t.Fatal(err) + } + case "email_accounts_smtp_imap": + for _, c := range []string{"smtp_host", "smtp_user", "smtp_password", "imap_host", "imap_user", "imap_password"} { + row[c] = "fixture" + } + row["smtp_port"], row["imap_port"] = 587, 993 + if _, err := pool.Exec(t.Context(), `INSERT INTO email_accounts_smtp_imap(email_account_id,smtp_host,smtp_port,smtp_user,smtp_password,imap_host,imap_port,imap_user,imap_password) VALUES($1,'old',25,'old','old','old',143,'old','old')`, mailbox); err != nil { + t.Fatal(err) + } + } + before := importTransportGeneration(t, pool, mailbox) + if _, err := pool.Exec(t.Context(), `INSERT INTO email_account_transport_recovery(email_account_id,operation,recovered_at) VALUES($1,'sync',NOW()-INTERVAL '1 hour')`, mailbox); err != nil { + t.Fatal(err) + } + result, err := svc.ImportFrom(t.Context(), f.org, tenantArchive(t, map[string][]map[string]any{table: {row}}), ImportOptions{Conflict: conflict, ActorUserID: f.owner}, nil) + if err != nil { + t.Fatal(err) + } + var clocks int + if err := pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM email_account_transport_recovery WHERE email_account_id=$1`, mailbox).Scan(&clocks); err != nil { + t.Fatal(err) + } + current := importTransportGeneration(t, pool, mailbox) + if conflict == models.OrgImportConflictSkip { + if current != before || clocks != 1 || result.RowCounts[table] != 0 { + t.Fatal("skipped credentials changed fence/count/clock") + } + } else if current == before || clocks != 0 || result.RowCounts[table] != 1 || result.RowCounts["email_accounts"] != 0 { + t.Fatal("actual credential-only write did not rotate/invalidate accurately") + } + }) + } + } +} diff --git a/internal/app/orgtransfer/mail_transport_export_live_test.go b/internal/app/orgtransfer/mail_transport_export_live_test.go new file mode 100644 index 000000000..b05884900 --- /dev/null +++ b/internal/app/orgtransfer/mail_transport_export_live_test.go @@ -0,0 +1,147 @@ +package orgtransfer + +import ( + "archive/zip" + "bytes" + "encoding/json" + "io" + "slices" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/encrypt" +) + +func TestLiveImportMailTransportExportOmitsGeneration(t *testing.T) { + for _, encrypted := range []bool{false, true} { + for _, conflict := range []models.OrgImportConflict{models.OrgImportConflictSkip, models.OrgImportConflictOverwrite} { + label := "plaintext/" + string(conflict) + if encrypted { + label = "encrypted/" + string(conflict) + } + t.Run(label, func(t *testing.T) { + svc, pool, f, mailbox, task, _ := overwriteTransportFixture(t) + native := svc.(*service) + enc, err := encrypt.NewEncrypter(make([]byte, 32)) + if err != nil { + t.Fatal(err) + } + native.creds = enc + password, err := enc.Encrypt("local-export-password") + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(t.Context(), `INSERT INTO email_accounts_smtp_imap(email_account_id,smtp_host,smtp_port,smtp_user,smtp_password,imap_host,imap_port,imap_user,imap_password) VALUES($1,'smtp.test',587,'fixture',$2,'imap.test',993,'fixture',$2)`, mailbox, password); err != nil { + t.Fatal(err) + } + source := importTransportGeneration(t, pool, mailbox) + if _, err := pool.Exec(t.Context(), `UPDATE email_accounts SET send_recovery_hold=true,send_recovery_reason='authentication',send_recovery_task_id=$2 WHERE id=$1`, mailbox, task); err != nil { + t.Fatal(err) + } + groups := []models.OrgDataGroup{models.OrgDataGroupCore, models.OrgDataGroupSending} + passphrase := "" + if encrypted { + passphrase = "local generation export test passphrase" + } + var archive bytes.Buffer + manifest, err := svc.ExportTo(t.Context(), f.org, ExportOptions{Groups: groups, Passphrase: passphrase}, &archive, nil) + if err != nil { + t.Fatal(err) + } + if (manifest.Secrets != nil) != encrypted { + t.Fatal("wrong archive secrets mode") + } + zr, err := zip.NewReader(bytes.NewReader(archive.Bytes()), int64(archive.Len())) + if err != nil { + t.Fatal(err) + } + entries := indexEntries(zr) + stored, err := readManifest(entries) + if err != nil { + t.Fatal(err) + } + for _, m := range []*Manifest{manifest, stored} { + tables := manifestTables(m) + mt, ok := tables["email_accounts"] + if !ok || mt.Rows != 1 || slices.Contains(mt.Columns, "transport_generation") || !slices.Contains(mt.Columns, "name") || !slices.Contains(mt.Columns, "worker_id") || !slices.Contains(mt.Columns, "auth_checked_at") { + t.Fatal("manifest lost portable columns or exported generation") + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(raw, []byte("transport_generation")) || bytes.Contains(raw, []byte(source.String())) { + t.Fatal("manifest leaked generation") + } + } + entry := entries[dataPath("email_accounts")] + if entry == nil { + t.Fatal("mailbox data missing") + } + r, err := entry.Open() + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(r) + r.Close() + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte("transport_generation")) || bytes.Contains(data, []byte(source.String())) { + t.Fatal("exported row leaked source generation") + } + var row map[string]json.RawMessage + if err := json.Unmarshal(bytes.TrimSpace(data), &row); err != nil { + t.Fatal(err) + } + if jsonString(row["id"]) != mailbox.String() || jsonString(row["name"]) != "Local" || jsonString(row["provider"]) != "smtp_imap" { + t.Fatal("ordinary row columns changed") + } + if importTransportGeneration(t, pool, mailbox) != source { + t.Fatal("export mutated source fence") + } + destination := uuid.New() + if _, err := pool.Exec(t.Context(), `UPDATE email_accounts SET transport_generation=$2,name='Destination',send_recovery_hold=true,send_recovery_reason='authentication',send_recovery_task_id=$3 WHERE id=$1`, mailbox, destination, task); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(t.Context(), `INSERT INTO email_account_transport_recovery(email_account_id,operation,recovered_at) VALUES($1,'sync',NOW()-INTERVAL '1 hour'),($1,'send',NOW()-INTERVAL '1 hour')`, mailbox); err != nil { + t.Fatal(err) + } + imported, err := svc.ImportFrom(t.Context(), f.org, bytes.NewReader(archive.Bytes()), ImportOptions{Groups: groups, Passphrase: passphrase, Conflict: conflict, ActorUserID: f.owner}, nil) + if err != nil { + t.Fatal(err) + } + current := importTransportGeneration(t, pool, mailbox) + var clocks int + var hold bool + var name string + if err := pool.QueryRow(t.Context(), `SELECT (SELECT COUNT(*) FROM email_account_transport_recovery WHERE email_account_id=$1),send_recovery_hold AND send_recovery_reason='authentication' AND send_recovery_task_id=$2,name FROM email_accounts WHERE id=$1`, mailbox, task).Scan(&clocks, &hold, &name); err != nil { + t.Fatal(err) + } + if !hold { + t.Fatal("portable export/import changed restrictive hold") + } + if conflict == models.OrgImportConflictSkip { + if current != destination || clocks != 2 || name != "Destination" || imported.RowCounts["email_accounts"] != 0 { + t.Fatal("skip changed destination fence/clock/count") + } + } else { + if current == source || current == destination || current == uuid.Nil || clocks != 0 || name != "Local" || imported.RowCounts["email_accounts"] != 1 { + t.Fatal("overwrite did not establish independent destination fence/clock/count") + } + if encrypted { + var restored string + if err := pool.QueryRow(t.Context(), `SELECT smtp_password FROM email_accounts_smtp_imap WHERE email_account_id=$1`, mailbox).Scan(&restored); err != nil { + t.Fatal(err) + } + plain, err := enc.Decrypt(restored) + if err != nil || plain != "local-export-password" { + t.Fatal("encrypted credentials no longer portable", err) + } + } + } + }) + } + } +} diff --git a/internal/app/orgtransfer/mail_transport_generation_live_test.go b/internal/app/orgtransfer/mail_transport_generation_live_test.go new file mode 100644 index 000000000..99b002043 --- /dev/null +++ b/internal/app/orgtransfer/mail_transport_generation_live_test.go @@ -0,0 +1,132 @@ +package orgtransfer + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func TestLiveImportMailTransportLateAndSpanningSuccess(t *testing.T) { + for _, operation := range []models.MailTransportOperation{models.MailTransportSync, models.MailTransportSend} { + for _, proof := range []string{"late", "spanning", "clock-skew", "legacy"} { + t.Run(string(operation)+"/"+proof, func(t *testing.T) { + svc, pool, f, mailbox, task, repo := overwriteTransportFixture(t) + generation := importTransportGeneration(t, pool, mailbox) + failedAt := time.Now().UTC().Add(-3 * time.Minute) + var reportTask *uuid.UUID + if operation == models.MailTransportSend { + reportTask = &task + } + report, xerr := repo.CreateOnce(t.Context(), &repository.CreateEmailAccountError{TransportGeneration: generation, EmailAccountID: mailbox, UserID: f.owner, TaskID: reportTask, Operation: operation, ObservedAt: &failedAt, ErrorCode: "SERVER_UNREACHABLE", Severity: "WARNING", ResolveMethod: "RETRY", Title: "Transport", Message: "Preserved report"}) + if xerr != nil || report == nil { + t.Fatal("native report", xerr) + } + if _, err := pool.Exec(t.Context(), `UPDATE email_accounts SET send_recovery_hold=true,send_recovery_reason='authentication',send_recovery_task_id=$2 WHERE id=$1`, mailbox, task); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(t.Context(), `INSERT INTO warmup_statistics(email_account_id,date,emails_sent,emails_replied,target_volume) VALUES($1,CURRENT_DATE,7,3,10)`, mailbox); err != nil { + t.Fatal(err) + } + if proof == "legacy" { + generation = uuid.Nil + } + if _, err := pool.Exec(t.Context(), `UPDATE tasks SET send_result_state=NULL,send_result_applied_at=NULL WHERE id=$1`, task); err != nil { + t.Fatal(err) + } + started, complete := make(chan struct{}), make(chan struct{}) + result := make(chan error, 1) + go func() { + captured := generation + close(started) + <-complete + at := time.Now().UTC().Add(-time.Millisecond) + if proof == "late" { + at = at.Add(-time.Hour) + } + if proof == "clock-skew" { + at = at.Add(-24 * time.Hour) + } + if operation == models.MailTransportSync { + if err := repo.RecordTransportSuccess(t.Context(), mailbox, operation, at, nil, captured); err != nil { + result <- err + } else { + result <- nil + } + return + } + outcome := models.SendEmailResult{TaskID: task, Success: true, MessageID: "accepted@example.test", SentAt: at, TransportGeneration: captured} + result <- repository.NewTaskRepository(pool).(repository.SendResultRecovery).ApplySendResult(t.Context(), outcome, func(ctx context.Context) error { + if err := repo.RecordTransportSuccess(ctx, mailbox, operation, at, &task, captured); err != nil { + return err + } + return nil + }) + }() + <-started + archive := tenantArchive(t, map[string][]map[string]any{"email_accounts": {{"id": mailbox, "user_id": f.owner, "organization_id": f.org, "email": mailbox.String() + "@restored.local", "name": "Restored", "signature_plain": "", "signature_html": "", "provider": "smtp_imap"}}}) + imported, err := svc.ImportFrom(t.Context(), f.org, archive, ImportOptions{Conflict: models.OrgImportConflictOverwrite, ActorUserID: f.owner}, nil) + close(complete) + if err != nil || imported.RowCounts["email_accounts"] != 1 { + t.Fatal("actual overwrite failed", err) + } + if err := <-result; err != nil { + t.Fatal("genuine result discarded", err) + } + current := importTransportGeneration(t, pool, mailbox) + if current == generation || current == uuid.Nil { + t.Fatal("generation did not rotate") + } + var clocks int + if err := pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM email_account_transport_recovery WHERE email_account_id=$1`, mailbox).Scan(&clocks); err != nil || clocks != 0 { + t.Fatal("old completion recreated current recovery", clocks, err) + } + var unresolved bool + if err := pool.QueryRow(t.Context(), `SELECT resolved_at IS NULL AND message='Preserved report' AND operation=$2 AND task_id IS NOT DISTINCT FROM $3::uuid FROM email_account_errors WHERE id=$1`, report.ID, operation, reportTask).Scan(&unresolved); err != nil || !unresolved { + t.Fatal("old completion changed report evidence", err) + } + if operation == models.MailTransportSend { + var confirmed bool + if err := pool.QueryRow(t.Context(), `SELECT send_result_state='sent' AND send_result_applied_at IS NOT NULL FROM tasks WHERE id=$1`, task).Scan(&confirmed); err != nil || !confirmed { + t.Fatal("confirmed task audit lost", err) + } + } + var taskID *uuid.UUID + if operation == models.MailTransportSend { + freshTask := uuid.New() + taskID = &freshTask + if _, err := pool.Exec(t.Context(), `INSERT INTO tasks(id,email_account_id,task_type,status,message_id) VALUES($1,$2,'email','completed','')`, freshTask, mailbox); err != nil { + t.Fatal(err) + } + } + freshAt := time.Now().UTC().Add(-time.Millisecond) + if taskID != nil { + outcome := models.SendEmailResult{TaskID: *taskID, Success: true, MessageID: "fresh-accepted@example.test", SentAt: freshAt, TransportGeneration: current} + if err := repository.NewTaskRepository(pool).(repository.SendResultRecovery).ApplySendResult(t.Context(), outcome, func(ctx context.Context) error { + if xerr := repo.RecordTransportSuccess(ctx, mailbox, operation, freshAt, taskID, current); xerr != nil { + return xerr + } + return nil + }); err != nil { + t.Fatal("fresh current native result", err) + } + } else if xerr := repo.RecordTransportSuccess(t.Context(), mailbox, operation, freshAt, nil, current); xerr != nil { + t.Fatal(xerr) + } + if err := pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM email_account_transport_recovery WHERE email_account_id=$1 AND operation=$2`, mailbox, operation).Scan(&clocks); err != nil || clocks != 1 { + t.Fatal("fresh current success failed to recover", clocks, err) + } + if err := pool.QueryRow(t.Context(), `SELECT resolved_at IS NOT NULL FROM email_account_errors WHERE id=$1`, report.ID).Scan(&unresolved); err != nil || !unresolved { + t.Fatal("fresh current success did not resolve native report", err) + } + var preserved bool + if err := pool.QueryRow(t.Context(), `SELECT e.send_recovery_hold AND e.send_recovery_reason='authentication' AND e.send_recovery_task_id=$2 AND s.emails_sent=7 AND s.emails_replied=3 FROM email_accounts e JOIN warmup_statistics s ON s.email_account_id=e.id AND s.date=CURRENT_DATE WHERE e.id=$1`, mailbox, task).Scan(&preserved); err != nil || !preserved { + t.Fatal("generation recovery changed restrictive hold/accounting", err) + } + }) + } + } +} diff --git a/internal/app/orgtransfer/mail_transport_import_matrix_live_test.go b/internal/app/orgtransfer/mail_transport_import_matrix_live_test.go new file mode 100644 index 000000000..ce54040ad --- /dev/null +++ b/internal/app/orgtransfer/mail_transport_import_matrix_live_test.go @@ -0,0 +1,180 @@ +package orgtransfer + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +func TestLiveImportMailTransportCheckpointCountsAndUnchangedRows(t *testing.T) { + for _, conflict := range []models.OrgImportConflict{models.OrgImportConflictSkip, models.OrgImportConflictOverwrite} { + t.Run(string(conflict), func(t *testing.T) { + svc, pool, f, mailbox, task, repo := overwriteTransportFixture(t) + generationBefore := importTransportGeneration(t, pool, mailbox) + other, inserted := uuid.New(), uuid.New() + for _, id := range []uuid.UUID{other} { + if _, err := pool.Exec(t.Context(), `INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider) VALUES($1,$2,$3,$4,'Unchanged','','','smtp_imap')`, id, f.owner, f.org, id.String()+"@test.local"); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM email_accounts WHERE id=ANY($1::uuid[])`, []uuid.UUID{other, inserted}) + }) + at := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + if _, err := pool.Exec(t.Context(), `INSERT INTO email_account_transport_recovery(email_account_id,operation,recovered_at) SELECT a,op,$2 FROM unnest($1::uuid[]) a CROSS JOIN unnest(ARRAY['sync','send']) op`, []uuid.UUID{mailbox, other}, at); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(t.Context(), `UPDATE email_accounts SET send_recovery_hold=true,send_recovery_task_id=$2,send_recovery_reason='unknown',auth_checked_at=$3 WHERE id=$1`, mailbox, task, at); err != nil { + t.Fatal(err) + } + row := func(id uuid.UUID) map[string]any { + return map[string]any{"id": id, "user_id": f.owner, "organization_id": f.org, "email": "restored-" + id.String() + "@test.local", "name": "Restored", "signature_plain": "", "signature_html": "", "provider": "smtp_imap"} + } + archive := tenantArchive(t, map[string][]map[string]any{"email_accounts": {row(mailbox), row(inserted)}}) + for pass := range 2 { + result, err := svc.ImportFrom(t.Context(), f.org, archive, ImportOptions{Conflict: conflict, ActorUserID: f.owner}, nil) + if err != nil { + t.Fatal(err) + } + want := int64(2) + if conflict == models.OrgImportConflictSkip { + want = int64(1 - pass) + } + if result.RowCounts["email_accounts"] != want { + t.Fatal("invalidation corrupted actual import count", result.RowCounts, want) + } + current := importTransportGeneration(t, pool, mailbox) + if (current == generationBefore) != (conflict == models.OrgImportConflictSkip) { + t.Fatal("generation reset did not follow actual overwrite") + } + generationBefore = current + var kept, affected int + if err := pool.QueryRow(t.Context(), `SELECT COUNT(*) FILTER (WHERE email_account_id=$1),COUNT(*) FILTER (WHERE email_account_id=$2) FROM email_account_transport_recovery`, other, mailbox).Scan(&kept, &affected); err != nil || kept != 2 || affected != map[bool]int{true: 2, false: 0}[conflict == models.OrgImportConflictSkip] { + t.Fatal("affected/unaffected checkpoint selection", kept, affected, err) + } + } + var held, taskPreserved bool + var checked time.Time + if err := pool.QueryRow(t.Context(), `SELECT send_recovery_hold,send_recovery_task_id=$2,auth_checked_at FROM email_accounts WHERE id=$1`, mailbox, task).Scan(&held, &taskPreserved, &checked); err != nil || !held || !taskPreserved || !checked.Equal(at) { + t.Fatal("import changed unrelated hold/auth state", err) + } + if err := repo.RecordTransportSuccess(t.Context(), mailbox, models.MailTransportSend, at.Add(time.Minute), &task, importTransportGeneration(t, pool, mailbox)); err != nil { + t.Fatal("confirmed source task audit lost", err) + } + }) + } +} + +func TestLiveImportMailTransportWaitsForPriorRecovery(t *testing.T) { + svc, pool, f, mailbox, task, _ := overwriteTransportFixture(t) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,285))`, mailbox.String()); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, `INSERT INTO email_account_transport_recovery(email_account_id,operation,recovered_at,source_task_id) VALUES($1,'send',NOW()-INTERVAL '1 hour',$2)`, mailbox, task); err != nil { + t.Fatal(err) + } + archive := tenantArchive(t, map[string][]map[string]any{"email_accounts": {{"id": mailbox, "user_id": f.owner, "organization_id": f.org, "email": mailbox.String() + "@test.local", "name": "Restored", "signature_plain": "", "signature_html": "", "provider": "smtp_imap"}}}) + started, finished := make(chan struct{}, 1), make(chan error, 1) + go func() { + result, err := svc.ImportFrom(ctx, f.org, archive, ImportOptions{Conflict: models.OrgImportConflictOverwrite, ActorUserID: f.owner}, func(_ int, stage string) { + if strings.Contains(stage, "importing email_accounts") { + started <- struct{}{} + } + }) + if err == nil && result.RowCounts["email_accounts"] != 1 { + err = errors.New("import count changed") + } + finished <- err + }() + select { + case <-started: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + var waiting bool + if err := pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event='advisory' AND query LIKE '%hashtextextended(keys.key%')`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + break + } + select { + case <-ticker.C: + case <-ctx.Done(): + t.Fatal("import did not serialize with prior success", ctx.Err()) + } + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + select { + case err := <-finished: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + var clocks int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM email_account_transport_recovery WHERE email_account_id=$1`, mailbox).Scan(&clocks); err != nil || clocks != 0 { + t.Fatal("concurrently committed prior clock escaped invalidation", clocks, err) + } +} + +func TestLiveImportMailTransportCheckpointForeignAndRollback(t *testing.T) { + for _, late := range []bool{false, true} { + label := "foreign_mailbox" + if late { + label = "later_failure_rolls_back" + } + t.Run(label, func(t *testing.T) { + svc, pool, f, mailbox, _, _ := overwriteTransportFixture(t) + generationBefore := importTransportGeneration(t, pool, mailbox) + foreign := newTenantFixture(t, pool) + at := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + if _, err := pool.Exec(t.Context(), `INSERT INTO email_account_transport_recovery(email_account_id,operation,recovered_at) VALUES($1,'sync',$2),($1,'send',$2)`, mailbox, at); err != nil { + t.Fatal(err) + } + row := map[string]any{"id": mailbox, "user_id": f.owner, "organization_id": f.org, "name": "Attempted restore", "email": mailbox.String() + "@test.local", "signature_plain": "", "signature_html": "", "provider": "smtp_imap"} + data := map[string][]map[string]any{"email_accounts": {row}} + if late { + data["contacts"] = []map[string]any{{"id": foreign.contact, "organization_id": f.org, "email": "foreign@test.local"}} + } else { + foreignMailbox := uuid.New() + if _, err := pool.Exec(t.Context(), `INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider) VALUES($1,$2,$3,$4,'Foreign','','','smtp_imap')`, foreignMailbox, foreign.owner, foreign.org, foreignMailbox.String()+"@test.local"); err != nil { + t.Fatal(err) + } + row["id"] = foreignMailbox + } + if _, err := svc.ImportFrom(t.Context(), f.org, tenantArchive(t, data), ImportOptions{Conflict: models.OrgImportConflictOverwrite, ActorUserID: f.owner}, nil); !errors.Is(err, ErrForeignRows) { + t.Fatal("foreign import accepted", err) + } + var clocks int + var name string + if err := pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM email_account_transport_recovery WHERE email_account_id=$1 AND recovered_at=$2`, mailbox, at).Scan(&clocks); err != nil || clocks != 2 { + t.Fatal("rejected/rolled-back import lost clocks", clocks, err) + } + if err := pool.QueryRow(t.Context(), `SELECT name FROM email_accounts WHERE id=$1`, mailbox).Scan(&name); err != nil || name != "Local" { + t.Fatal("rejected/rolled-back import changed mailbox", name, err) + } + if importTransportGeneration(t, pool, mailbox) != generationBefore { + t.Fatal("foreign/rolled-back import rotated generation") + } + }) + } +} diff --git a/internal/app/orgtransfer/mail_transport_live_test.go b/internal/app/orgtransfer/mail_transport_live_test.go new file mode 100644 index 000000000..ae2e9cb3e --- /dev/null +++ b/internal/app/orgtransfer/mail_transport_live_test.go @@ -0,0 +1,46 @@ +package orgtransfer + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +func TestLiveImportMailTransportEvidenceResetsUnknown(t *testing.T) { + svc, pool := liveImportService(t) + dest := newTenantFixture(t, pool) + mailbox, task, report := uuid.New(), uuid.New(), uuid.New() + if _, err := pool.Exec(t.Context(), `INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider) VALUES($1,$2,$3,$4,'Imported','','','smtp_imap')`, mailbox, dest.owner, dest.org, mailbox.String()+"@test.local"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _, _ = pool.Exec(context.Background(), `DELETE FROM email_accounts WHERE id=$1`, mailbox) }) + at := time.Now().Add(-time.Hour).UTC().Truncate(time.Microsecond) + archive := tenantArchive(t, map[string][]map[string]any{ + "tasks": {{"id": task, "email_account_id": mailbox, "task_type": "warmup", "status": "completed", "message_id": "", "completed_at": at}}, + "email_account_errors": {{"id": report, "email_account_id": mailbox, "user_id": dest.owner, "task_id": task, "error_code": "SERVER_UNREACHABLE", "severity": "WARNING", "resolve_method": "RETRY", "title": "Historical", "message": "Transport report", "operation": "send", "observation_source": "worker", "received_at": at, "created_at": at}}, + }) + if _, err := svc.ImportFrom(t.Context(), dest.org, archive, ImportOptions{Conflict: models.OrgImportConflictOverwrite, ActorUserID: dest.owner}, nil); err != nil { + t.Fatal(err) + } + var op, source string + var received *time.Time + var taskID uuid.UUID + var created time.Time + if err := pool.QueryRow(t.Context(), `SELECT operation,observation_source,received_at,task_id,created_at FROM email_account_errors WHERE id=$1`, report).Scan(&op, &source, &received, &taskID, &created); err != nil { + t.Fatal(err) + } + if op != "unknown" || source != "unknown" || received != nil || taskID != task || !created.Equal(at) { + t.Fatalf("import changed history or retained operation authority: %s %s %v %v %v", op, source, received, taskID, created) + } + if ExcludedTables["email_account_transport_recovery"] == "" { + t.Fatal("success authority missing from transfer exclusions") + } + for _, table := range Tables { + if table.Name == "email_account_transport_recovery" { + t.Fatal("instance-local success evidence travels") + } + } +} diff --git a/internal/app/orgtransfer/mail_transport_lock_order_live_test.go b/internal/app/orgtransfer/mail_transport_lock_order_live_test.go new file mode 100644 index 000000000..ee1522f60 --- /dev/null +++ b/internal/app/orgtransfer/mail_transport_lock_order_live_test.go @@ -0,0 +1,256 @@ +package orgtransfer + +import ( + "bytes" + "context" + "encoding/json" + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +type observedWholeImportRepository struct { + repository.OrgTransferRepository + pid chan uint32 +} + +func (r *observedWholeImportRepository) Begin(ctx context.Context) (pgx.Tx, error) { + tx, err := r.OrgTransferRepository.Begin(ctx) + if err == nil { + r.pid <- tx.Conn().PgConn().PID() + } + return tx, err +} + +func TestLiveImportMailTransportWholeArchiveSendResultLockOrder(t *testing.T) { + for _, lane := range []string{"import_first", "import_first_external_task_row", "result_first"} { + t.Run(lane, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + svc, pool, f, mailbox, task, reports := overwriteTransportFixture(t) + generation := importTransportGeneration(t, pool, mailbox) + at := time.Now().UTC().Add(-time.Second).Truncate(time.Microsecond) + result := models.SendEmailResult{TaskID: task, Success: true, SentAt: at, TransportGeneration: generation, MessageID: "native-lock-proof", ProviderMsgID: "provider-lock-proof"} + raw, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE tasks SET send_result_state=NULL,send_executor_result=$2 WHERE id=$1`, task, raw); err != nil { + t.Fatal(err) + } + warm := uuid.New() + if _, err := pool.Exec(ctx, `INSERT INTO tasks(id,email_account_id,task_type,status,message_id,send_result_state,send_result_applied_at) VALUES($1,$2,'warmup','completed','warm-history','sent',NOW())`, warm, mailbox); err != nil { + t.Fatal(err) + } + warmResult := result + warmResult.TaskID = warm + warmRaw, err := json.Marshal(warmResult) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `INSERT INTO warmup_tasks(task_id,dispatch_result) VALUES($1,$2)`, warm, warmRaw); err != nil { + t.Fatal(err) + } + audit := uuid.New() + if _, err := pool.Exec(ctx, `INSERT INTO send_recovery_resolutions(id,organization_id,email_account_id,recovery_task_id,previous_reason,evidence_type,confirmation_reference,confirmed_result,recorded_locally_at) VALUES($1,$2,$3,$4,'unknown','operator_confirmed_sent','Preserve archive lock proof',$5,NOW())`, audit, f.org, mailbox, task, raw); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE email_accounts SET send_recovery_hold=true,send_recovery_reason='authentication',send_recovery_task_id=$2 WHERE id=$1`, mailbox, task); err != nil { + t.Fatal(err) + } + var archive bytes.Buffer + manifest, err := svc.ExportTo(ctx, f.org, ExportOptions{}, &archive, nil) + if err != nil { + t.Fatal(err) + } + if len(manifest.Groups) != len(models.AllOrgDataGroups) { + t.Fatal("not whole archive") + } + mts := manifestTables(manifest) + if mts["tasks"].Rows != 2 || mts["warmup_tasks"].Rows != 1 || mts["send_recovery_resolutions"].Rows != 1 { + t.Fatal("missing populated send history") + } + importPID := make(chan uint32, 1) + service := svc.(*service) + service.repo = &observedWholeImportRepository{OrgTransferRepository: service.repo, pid: importPID} + cfg, err := pgxpool.ParseConfig(os.Getenv("WARMBLY_TEST_DB")) + if err != nil { + t.Fatal(err) + } + cfg.MaxConns = 1 + cfg.MinConns = 0 + cfg.ConnConfig.RuntimeParams["application_name"] = "whole-archive-result-" + task.String() + resultPool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer resultPool.Close() + conn, err := resultPool.Acquire(ctx) + if err != nil { + t.Fatal(err) + } + resultPID := conn.Conn().PgConn().PID() + conn.Release() + recovery := repository.NewTaskRepository(resultPool).(repository.SendResultRecovery) + importDone, resultDone := make(chan error, 1), make(chan error, 1) + atTasks, resumeImport, atCallback, resumeResult := make(chan struct{}, 1), make(chan struct{}), make(chan struct{}, 1), make(chan struct{}) + startImport := func() { + go func() { + _, err := svc.ImportFrom(ctx, f.org, bytes.NewReader(archive.Bytes()), ImportOptions{Conflict: models.OrgImportConflictOverwrite, ActorUserID: f.owner}, func(_ int, stage string) { + if stage == "importing tasks" && lane != "result_first" { + atTasks <- struct{}{} + select { + case <-resumeImport: + case <-ctx.Done(): + } + } + }) + importDone <- err + }() + } + startResult := func() { + go func() { + resultDone <- recovery.ApplySendResult(ctx, result, func(inner context.Context) error { + if err := reports.RecordTransportSuccess(inner, mailbox, models.MailTransportSend, at, &task, generation); err != nil { + return err + } + if lane == "result_first" { + atCallback <- struct{}{} + select { + case <-resumeResult: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) + }() + } + wait := func(q string, args ...any) { + t.Helper() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + var yes bool + if err := pool.QueryRow(ctx, q, args...).Scan(&yes); err != nil { + t.Fatal(err) + } + if yes { + return + } + select { + case <-ticker.C: + case <-ctx.Done(): + t.Fatal("lock state not reached", ctx.Err()) + } + } + } + snapshot := func(label string, pid uint32) { + t.Helper() + var state []byte + if err := pool.QueryRow(ctx, `SELECT jsonb_build_object('pid',$1::int,'blockers',pg_blocking_pids($1),'locks',(SELECT jsonb_agg(jsonb_build_object('type',locktype,'mode',mode,'granted',granted,'relation',relation::regclass::text,'classid',classid,'objid',objid,'transactionid',transactionid)) FROM pg_locks WHERE pid=$1))`, pid).Scan(&state); err != nil { + t.Fatal(err) + } + t.Logf("%s %s", label, state) + } + var ip uint32 + if lane == "result_first" { + startResult() + select { + case <-atCallback: + case err := <-resultDone: + t.Fatalf("result did not enter callback: %v", err) + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + startImport() + select { + case ip = <-importPID: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + wait(`SELECT $2::int=ANY(pg_blocking_pids($1)) AND EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory' AND NOT granted)`, ip, resultPID) + snapshot("result owns269/task row; import waits269", resultPID) + snapshot("import has not reached tasks", ip) + close(resumeResult) + } else { + startImport() + select { + case <-atTasks: + case err := <-importDone: + t.Fatalf("import did not reach task boundary: %v", err) + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + select { + case ip = <-importPID: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + var rowGate pgx.Tx + var gatePID uint32 + if lane == "import_first_external_task_row" { + rowGate, err = pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer rowGate.Rollback(context.Background()) + gatePID = rowGate.Conn().PgConn().PID() + var locked uuid.UUID + if err := rowGate.QueryRow(ctx, `SELECT id FROM tasks WHERE id=$1 FOR UPDATE`, task).Scan(&locked); err != nil { + t.Fatal(err) + } + } + startResult() + wait(`SELECT $2::int=ANY(pg_blocking_pids($1)) AND EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory' AND NOT granted)`, resultPID, ip) + var rowLockBeforeMailbox bool + if err := pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND (locktype='tuple' OR relation='tasks'::regclass AND mode IN('RowShareLock','RowExclusiveLock')))`, resultPID).Scan(&rowLockBeforeMailbox); err != nil { + t.Fatal(err) + } + if rowLockBeforeMailbox { + t.Fatal("actual ApplySendResult took task row before269") + } + snapshot("import owns269/285", ip) + snapshot("Apply owns268 and tasks AccessShare only; waits269", resultPID) + close(resumeImport) + if rowGate != nil { + wait(`SELECT $2::int=ANY(pg_blocking_pids($1))`, ip, gatePID) + snapshot("import waits separate row-gate, not result", ip) + snapshot("separate row-gate has no blockers", gatePID) + if err := rowGate.Rollback(ctx); err != nil { + t.Fatal(err) + } + } + } + for _, done := range []<-chan error{importDone, resultDone} { + select { + case err := <-done: + if err != nil { + t.Fatalf("native import/result failed: %T %v", err, err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } + var held, historical bool + if err := pool.QueryRow(ctx, `SELECT send_recovery_hold AND send_recovery_reason='authentication' AND send_recovery_task_id=$2 FROM email_accounts WHERE id=$1`, mailbox, task).Scan(&held); err != nil || !held { + t.Fatalf("restrictive hold changed %v %v", held, err) + } + if err := pool.QueryRow(ctx, `SELECT recorded_locally_at IS NULL AND confirmation_reference='Preserve archive lock proof' AND confirmed_result->>'message_id'='native-lock-proof' FROM send_recovery_resolutions WHERE id=$1`, audit).Scan(&historical); err != nil || !historical { + t.Fatalf("historical evidence/local provenance changed %v %v", historical, err) + } + if lane != "result_first" { + var confirmed bool + if err := pool.QueryRow(ctx, `SELECT send_result_state='sent' AND send_result_applied_at IS NOT NULL FROM tasks WHERE id=$1`, task).Scan(&confirmed); err != nil || !confirmed { + t.Fatal("late native result did not reconcile", err) + } + } + }) + } +} diff --git a/internal/app/orgtransfer/mail_transport_overwrite_review_live_test.go b/internal/app/orgtransfer/mail_transport_overwrite_review_live_test.go new file mode 100644 index 000000000..1048d2263 --- /dev/null +++ b/internal/app/orgtransfer/mail_transport_overwrite_review_live_test.go @@ -0,0 +1,228 @@ +package orgtransfer + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func overwriteTransportFixture(t *testing.T) (Service, *pgxpool.Pool, tenantFixture, uuid.UUID, uuid.UUID, repository.EmailAccountErrorRepository) { + t.Helper() + svc, pool := liveImportService(t) + f := newTenantFixture(t, pool) + mailbox, task := uuid.New(), uuid.New() + if _, err := pool.Exec(t.Context(), `INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider) + VALUES($1,$2,$3,$4,'Local','','','smtp_imap')`, mailbox, f.owner, f.org, mailbox.String()+"@test.local"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), `DELETE FROM email_accounts WHERE id=$1`, mailbox); err != nil { + t.Error(err) + } + }) + if _, err := pool.Exec(t.Context(), `INSERT INTO tasks(id,email_account_id,task_type,status,message_id,send_result_state) + VALUES($1,$2,'email','completed','','sent')`, task, mailbox); err != nil { + t.Fatal(err) + } + return svc, pool, f, mailbox, task, repository.NewEmailAccountErrorRepository(&db.DB{Pool: pool}) +} + +func importTransportGeneration(t *testing.T, pool *pgxpool.Pool, mailbox uuid.UUID) uuid.UUID { + t.Helper() + var generation uuid.UUID + if err := pool.QueryRow(t.Context(), `SELECT transport_generation FROM email_accounts WHERE id=$1`, mailbox).Scan(&generation); err != nil { + t.Fatal(err) + } + return generation +} + +func TestLiveImportMailTransportOverwriteResetsKnownReport(t *testing.T) { + for _, operation := range []models.MailTransportOperation{models.MailTransportSync, models.MailTransportSend} { + for _, old := range []bool{true, false} { + for _, conflict := range []models.OrgImportConflict{models.OrgImportConflictSkip, models.OrgImportConflictOverwrite} { + label := string(operation) + "/new_archive/" + string(conflict) + if old { + label = string(operation) + "/old_archive/" + string(conflict) + } + t.Run(label, func(t *testing.T) { + svc, pool, f, mailbox, task, repo := overwriteTransportFixture(t) + report := uuid.New() + local := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Microsecond) + archived := local.Add(-time.Hour) + var reportTask *uuid.UUID + if operation == models.MailTransportSend { + reportTask = &task + } + if _, err := pool.Exec(t.Context(), `INSERT INTO email_account_errors(id,email_account_id,user_id,error_code,severity,resolve_method,title,message,task_id,created_at,operation,observation_source,received_at) + VALUES($1,$2,$3,'SERVER_UNREACHABLE','WARNING','RETRY','Local','Local report',$4,$5,$6,'worker',$5)`, report, mailbox, f.owner, reportTask, local, operation); err != nil { + t.Fatal(err) + } + row := map[string]any{"id": report, "email_account_id": mailbox, "user_id": f.owner, "error_code": "SERVER_UNREACHABLE", "severity": "WARNING", "resolve_method": "RETRY", "title": "Archive", "message": "Imported historical report", "task_id": reportTask, "created_at": archived} + if !old { + row["operation"] = operation + row["observation_source"] = "worker" + row["received_at"] = archived + } + archive := tenantArchive(t, map[string][]map[string]any{"email_account_errors": {row}}) + if _, err := svc.ImportFrom(t.Context(), f.org, archive, ImportOptions{Conflict: conflict, ActorUserID: f.owner}, nil); err != nil { + t.Fatal(err) + } + var taskID *uuid.UUID + if operation == models.MailTransportSend { + taskID = &task + } + if err := repo.RecordTransportSuccess(t.Context(), mailbox, operation, local.Add(-30*time.Minute), taskID, importTransportGeneration(t, pool, mailbox)); err != nil { + t.Fatal(err) + } + var op, source, message string + var received, resolved *time.Time + var storedTask *uuid.UUID + var created time.Time + if err := pool.QueryRow(t.Context(), `SELECT operation,observation_source,received_at,resolved_at,task_id,created_at,message FROM email_account_errors WHERE id=$1`, report).Scan(&op, &source, &received, &resolved, &storedTask, &created, &message); err != nil { + t.Fatal(err) + } + wantOp, wantSource, wantCreated, wantMessage := "unknown", "unknown", archived, "Imported historical report" + wantReceived := received == nil + if conflict == models.OrgImportConflictSkip { + wantOp = string(operation) + wantSource = "worker" + wantCreated = local + wantMessage = "Local report" + wantReceived = received != nil && received.Equal(local) + } + taskPreserved := storedTask == nil && reportTask == nil || storedTask != nil && reportTask != nil && *storedTask == *reportTask + if op != wantOp || source != wantSource || !wantReceived || resolved != nil || !taskPreserved || !created.Equal(wantCreated) || message != wantMessage { + t.Fatalf("overwrite inherited recovery authority or lost history: operation=%s source=%s receipt_present=%v resolved=%v task_preserved=%v clock_preserved=%v message=%s", op, source, received != nil, resolved != nil, taskPreserved, created.Equal(wantCreated), message) + } + }) + } + } + } +} + +func TestLiveImportMailTransportOverwriteClearsDestinationClock(t *testing.T) { + for _, conflict := range []models.OrgImportConflict{models.OrgImportConflictSkip, models.OrgImportConflictOverwrite} { + t.Run(string(conflict), func(t *testing.T) { + svc, pool, f, mailbox, task, repo := overwriteTransportFixture(t) + at := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Microsecond) + for _, op := range []models.MailTransportOperation{models.MailTransportSync, models.MailTransportSend} { + var source *uuid.UUID + if op == models.MailTransportSend { + source = &task + } + if err := repo.RecordTransportSuccess(t.Context(), mailbox, op, at, source, importTransportGeneration(t, pool, mailbox)); err != nil { + t.Fatal(err) + } + } + if _, err := pool.Exec(t.Context(), `UPDATE email_accounts SET auth_checked_at=$2,send_recovery_hold=true,send_recovery_reason='unknown',send_recovery_task_id=$3 WHERE id=$1`, mailbox, at, task); err != nil { + t.Fatal(err) + } + archive := tenantArchive(t, map[string][]map[string]any{"email_accounts": {{"id": mailbox, "user_id": f.owner, "organization_id": f.org, "email": "restored-" + mailbox.String() + "@test.local", "name": "Archive", "signature_plain": "", "signature_html": "", "provider": "smtp_imap", "send_recovery_hold": true, "send_recovery_reason": "unknown", "send_recovery_task_id": task}}}) + if _, err := svc.ImportFrom(t.Context(), f.org, archive, ImportOptions{Conflict: conflict, ActorUserID: f.owner}, nil); err != nil { + t.Fatal(err) + } + var name, email string + var held bool + var heldTask uuid.UUID + var checked time.Time + if err := pool.QueryRow(t.Context(), `SELECT name,email,send_recovery_hold,send_recovery_task_id,auth_checked_at FROM email_accounts WHERE id=$1`, mailbox).Scan(&name, &email, &held, &heldTask, &checked); err != nil { + t.Fatal(err) + } + wantName := "Archive" + wantEmail := "restored-" + mailbox.String() + "@test.local" + if conflict == models.OrgImportConflictSkip { + wantName = "Local" + wantEmail = mailbox.String() + "@test.local" + } + if name != wantName || email != wantEmail || !held || heldTask != task || !checked.Equal(at) { + t.Fatal("not genuine overwrite or broad mailbox reset changed unrelated metadata/hold") + } + for _, op := range []models.MailTransportOperation{models.MailTransportSync, models.MailTransportSend} { + observed := at.Add(-time.Hour) + data := &repository.CreateEmailAccountError{TransportGeneration: importTransportGeneration(t, pool, mailbox), EmailAccountID: mailbox, UserID: f.owner, ErrorCode: "CONNECTION_LOST", Severity: "WARNING", ResolveMethod: "RETRY", Title: "Delayed", Message: "Typed delayed failure", Operation: op, ObservedAt: &observed} + if op == models.MailTransportSend { + data.TaskID = &task + } + created, err := repo.CreateOnce(t.Context(), data) + if err != nil { + t.Fatal(err) + } + var resolved bool + if err := pool.QueryRow(t.Context(), `SELECT resolved_at IS NOT NULL FROM email_account_errors WHERE email_account_id=$1 AND error_code='CONNECTION_LOST' AND operation=$2`, mailbox, op).Scan(&resolved); err != nil { + t.Fatal(err) + } + kept := conflict == models.OrgImportConflictSkip + if resolved != kept || (created == nil) != kept { + t.Errorf("stale destination %s clock suppressed post-overwrite failure: resolved=%v notification_suppressed=%v skip=%v", op, resolved, created == nil, kept) + } + } + }) + } +} + +func TestLiveImportMailTransportOverwriteKeepsAmbiguousHistory(t *testing.T) { + svc, pool, f, mailbox, task, repo := overwriteTransportFixture(t) + at := time.Now().UTC().Add(-time.Hour).Truncate(time.Microsecond) + legacy := uuid.New() + if _, err := pool.Exec(t.Context(), `INSERT INTO email_account_errors(id,email_account_id,user_id,error_code,severity,resolve_method,title,message,created_at) + VALUES($1,$2,$3,'SERVER_UNREACHABLE','WARNING','RETRY','Legacy','Unknown legacy report',$4)`, legacy, mailbox, f.owner, at); err != nil { + t.Fatal(err) + } + known, err := repo.CreateOnce(t.Context(), &repository.CreateEmailAccountError{TransportGeneration: importTransportGeneration(t, pool, mailbox), EmailAccountID: mailbox, UserID: f.owner, + ErrorCode: "SERVER_UNREACHABLE", Severity: "WARNING", ResolveMethod: "RETRY", Title: "Known", Message: "Known send report", + Operation: models.MailTransportSend, TaskID: &task, ObservedAt: &at}) + if err != nil || known == nil { + t.Fatalf("seed known report alongside genuine legacy unknown: %v", err) + } + archive := tenantArchive(t, map[string][]map[string]any{"email_account_errors": {{ + "id": known.ID, "email_account_id": mailbox, "user_id": f.owner, "error_code": "SERVER_UNREACHABLE", "severity": "WARNING", + "resolve_method": "RETRY", "title": "Archive", "message": "Imported known report", "task_id": task, "created_at": at, + "operation": "send", "observation_source": "worker", "received_at": at, + }}}) + if _, err := svc.ImportFrom(t.Context(), f.org, archive, ImportOptions{Conflict: models.OrgImportConflictOverwrite, ActorUserID: f.owner}, nil); err != nil { + var unchanged bool + if check := pool.QueryRow(t.Context(), `SELECT COUNT(*)=2 AND BOOL_AND(resolved_at IS NULL) + AND COUNT(*) FILTER (WHERE operation='unknown' AND message='Unknown legacy report')=1 + AND COUNT(*) FILTER (WHERE operation='send' AND message='Known send report')=1 + FROM email_account_errors WHERE email_account_id=$1`, mailbox).Scan(&unchanged); check != nil || !unchanged { + t.Fatalf("failed import did not preserve both original unresolved histories: unchanged=%v err=%v", unchanged, check) + } + t.Fatalf("safe operation reset collides with unresolved unknown history: %v", err) + } + var reports int + if err := pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM email_account_errors WHERE email_account_id=$1 AND resolved_at IS NULL`, mailbox).Scan(&reports); err != nil || reports != 2 { + t.Fatalf("history lost or invented resolution: reports=%d err=%v", reports, err) + } + for _, oldArchive := range []bool{true, false} { + row := map[string]any{"id": known.ID, "email_account_id": mailbox, "user_id": f.owner, "error_code": "SERVER_UNREACHABLE", "severity": "WARNING", "resolve_method": "RETRY", "title": "History", "message": "Historical send report", "created_at": at, "task_id": task} + if !oldArchive { + row["operation"], row["observation_source"], row["received_at"] = "send", "worker", at + } + for range 2 { + result, err := svc.ImportFrom(t.Context(), f.org, tenantArchive(t, map[string][]map[string]any{"email_account_errors": {row}}), ImportOptions{Conflict: models.OrgImportConflictOverwrite, ActorUserID: f.owner}, nil) + if err != nil || result.RowCounts["email_account_errors"] != 1 { + t.Fatal("repeated old/new history overwrite failed or wrong count", err) + } + } + } + active, err := repo.CreateOnce(t.Context(), &repository.CreateEmailAccountError{TransportGeneration: importTransportGeneration(t, pool, mailbox), EmailAccountID: mailbox, UserID: f.owner, ErrorCode: "SERVER_UNREACHABLE", Severity: "WARNING", ResolveMethod: "RETRY", Title: "Native", Message: "Native send failure", Operation: models.MailTransportSend, TaskID: &task, ObservedAt: &at}) + if err != nil || active == nil || active.ID == known.ID || active.ID == legacy { + t.Fatal("historical report collided with local dedupe", err) + } + if err := repo.RecordTransportSuccess(t.Context(), mailbox, models.MailTransportSend, at.Add(time.Minute), &task, importTransportGeneration(t, pool, mailbox)); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM email_account_errors WHERE id=ANY($1::uuid[]) AND resolved_at IS NULL AND observation_source='unknown' AND operation='unknown'`, []uuid.UUID{known.ID, legacy}).Scan(&reports); err != nil || reports != 2 { + t.Fatal("local recovery altered imported/legacy reports", reports, err) + } + var retainedTask uuid.UUID + if err := pool.QueryRow(t.Context(), `SELECT task_id FROM email_account_errors WHERE id=$1`, known.ID).Scan(&retainedTask); err != nil || retainedTask != task { + t.Fatal("history lost source task", retainedTask, err) + } +} diff --git a/internal/app/orgtransfer/mail_transport_whole_export_live_test.go b/internal/app/orgtransfer/mail_transport_whole_export_live_test.go new file mode 100644 index 000000000..acdcc6cfd --- /dev/null +++ b/internal/app/orgtransfer/mail_transport_whole_export_live_test.go @@ -0,0 +1,239 @@ +package orgtransfer + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "io" + "reflect" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +func assertExportJSONEqual(t *testing.T, actual, expected []byte) { + t.Helper() + var a, b any + for _, item := range []struct { + raw []byte + out *any + }{{actual, &a}, {expected, &b}} { + decoder := json.NewDecoder(bytes.NewReader(item.raw)) + decoder.UseNumber() + if err := decoder.Decode(item.out); err != nil { + t.Fatal(err) + } + } + if !reflect.DeepEqual(a, b) { + t.Fatal("portable evidence changed", a, b) + } +} + +func TestLiveImportMailTransportWholeArchiveOmitsGeneration(t *testing.T) { + for _, encrypted := range []bool{false, true} { + for _, sent := range []bool{false, true} { + label := "plaintext" + if encrypted { + label = "encrypted" + } + if sent { + label += "/sent" + } else { + label += "/failed" + } + t.Run(label, func(t *testing.T) { + svc, pool, f, mailbox, task, reports := overwriteTransportFixture(t) + source := importTransportGeneration(t, pool, mailbox) + now := time.Now().UTC().Add(-time.Second).Truncate(time.Microsecond) + normal := models.SendEmailResult{TransportGeneration: source, TaskID: task, Success: sent, MessageID: "accepted@fixture.test", ProviderMsgID: "provider-proof", ThreadID: "conversation-proof", SentAt: now, LegacyErrorMsg: "preserved historical text"} + if !sent { + normal.Error = &models.EmailSendError{Code: "RECIPIENT_REJECTED", Type: "permanent", Message: "Preserved failure evidence"} + } + raw, err := json.Marshal(normal) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(t.Context(), `UPDATE tasks SET send_executor_result=$2 WHERE id=$1`, task, raw); err != nil { + t.Fatal(err) + } + warmTask := uuid.New() + warm := normal + warm.TaskID = warmTask + warm.ProviderMsgID = "warm-provider-proof" + warmRaw, err := json.Marshal(warm) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(t.Context(), `INSERT INTO tasks(id,email_account_id,task_type,status,message_id,send_result_state) VALUES($1,$2,'warmup','completed','warm-message','sent')`, warmTask, mailbox); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(t.Context(), `INSERT INTO warmup_tasks(task_id,dispatch_result) VALUES($1,$2)`, warmTask, warmRaw); err != nil { + t.Fatal(err) + } + audit := uuid.New() + evidence := "operator_confirmed_sent" + if !sent { + evidence = "operator_confirmed_not_sent" + } + if _, err := pool.Exec(t.Context(), `INSERT INTO send_recovery_resolutions(id,organization_id,email_account_id,recovery_task_id,previous_reason,evidence_type,confirmation_reference,confirmed_result,recorded_locally_at) VALUES($1,$2,$3,$4,'unknown',$5,'Preserved source confirmation',$6,NOW())`, audit, f.org, mailbox, task, evidence, raw); err != nil { + t.Fatal(err) + } + syncRepo := repository.NewEmailSyncStateRepository(&db.DB{Pool: pool}) + state := models.SyncState{TransportGeneration: source, LastSyncedAt: &now, BackfillStatus: models.SyncBackfillComplete, BackfillCursor: models.SyncCursor{PageToken: "source-cursor"}, BackfillSynced: 17} + if err := syncRepo.Put(t.Context(), f.owner, mailbox, &state); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM email_sync_state WHERE email_id=$1`, mailbox) + }) + persisted, err := syncRepo.Get(t.Context(), mailbox) + if err != nil || persisted == nil || persisted.TransportGeneration != uuid.Nil || persisted.BackfillSynced != 17 { + t.Fatal("sync persistence inventory changed", err) + } + if _, err := reports.CreateOnce(t.Context(), &repository.CreateEmailAccountError{EmailAccountID: mailbox, UserID: f.owner, TaskID: &task, TransportGeneration: source, Operation: models.MailTransportSend, ErrorCode: "SERVER_UNREACHABLE", Severity: "WARNING", ResolveMethod: "RETRY", Title: "Preserved transport observation", Message: "Preserved report evidence", ObservedAt: &now}); err != nil { + t.Fatal(err) + } + passphrase := "" + if encrypted { + passphrase = "whole archive export regression passphrase" + } + var archive bytes.Buffer + manifest, err := svc.ExportTo(t.Context(), f.org, ExportOptions{Passphrase: passphrase}, &archive, nil) + if err != nil { + t.Fatal(err) + } + if (manifest.Secrets != nil) != encrypted { + t.Fatal("wrong export mode") + } + if len(manifest.Groups) != len(models.AllOrgDataGroups) { + t.Fatal("not a whole archive") + } + zr, err := zip.NewReader(bytes.NewReader(archive.Bytes()), int64(archive.Len())) + if err != nil { + t.Fatal(err) + } + entries := indexEntries(zr) + for _, entry := range zr.File { + reader, err := entry.Open() + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(reader) + reader.Close() + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte(source.String())) || bytes.Contains(data, []byte("transport_generation")) { + t.Errorf("generation leaked into whole-archive entry %s", entry.Name) + } + } + manifestRaw, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(manifestRaw, []byte(source.String())) || bytes.Contains(manifestRaw, []byte("transport_generation")) { + t.Fatal("returned manifest leaked generation") + } + if entries[dataPath("email_sync_state")] != nil || entries[dataPath("email_account_transport_recovery")] != nil { + t.Fatal("instance-local sync/recovery checkpoint exported") + } + stored, err := readManifest(entries) + if err != nil { + t.Fatal(err) + } + for _, m := range []*Manifest{manifest, stored} { + for table, want := range map[string]int64{"tasks": 2, "warmup_tasks": 1, "send_recovery_resolutions": 1, "email_account_errors": 1} { + mt, ok := manifestTables(m)[table] + if !ok || mt.Rows != want { + t.Fatal("whole export lost evidence rows", table, mt.Rows) + } + } + } + for _, item := range []struct { + table, column string + result []byte + }{{"tasks", "send_executor_result", raw}, {"warmup_tasks", "dispatch_result", warmRaw}, {"send_recovery_resolutions", "confirmed_result", raw}} { + reader, err := entries[dataPath(item.table)].Open() + if err != nil { + t.Fatal(err) + } + decoder := json.NewDecoder(reader) + found := false + for { + var row map[string]json.RawMessage + err := decoder.Decode(&row) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + value := row[item.column] + if len(value) == 0 || bytes.Equal(value, []byte("null")) { + continue + } + var expected map[string]json.RawMessage + if err := json.Unmarshal(item.result, &expected); err != nil { + t.Fatal(err) + } + delete(expected, "transport_generation") + portable, err := json.Marshal(expected) + if err != nil { + t.Fatal(err) + } + assertExportJSONEqual(t, value, portable) + found = true + } + reader.Close() + if !found { + t.Fatal("result envelope was dropped", item.table) + } + } + for _, item := range []struct { + query string + id uuid.UUID + expected []byte + }{ + {`SELECT send_executor_result FROM tasks WHERE id=$1`, task, raw}, + {`SELECT dispatch_result FROM warmup_tasks WHERE task_id=$1`, warmTask, warmRaw}, + {`SELECT confirmed_result FROM send_recovery_resolutions WHERE id=$1`, audit, raw}, + } { + var current []byte + if err := pool.QueryRow(t.Context(), item.query, item.id).Scan(¤t); err != nil { + t.Fatal(err) + } + assertExportJSONEqual(t, current, item.expected) + } + if importTransportGeneration(t, pool, mailbox) != source { + t.Fatal("export changed source generation") + } + if _, err := svc.ImportFrom(t.Context(), f.org, bytes.NewReader(archive.Bytes()), ImportOptions{Passphrase: passphrase, Conflict: models.OrgImportConflictOverwrite, ActorUserID: f.owner}, nil); err != nil { + t.Fatal(err) + } + var historical []byte + var local bool + if err := pool.QueryRow(t.Context(), `SELECT confirmed_result,recorded_locally_at IS NOT NULL FROM send_recovery_resolutions WHERE id=$1`, audit).Scan(&historical, &local); err != nil { + t.Fatal(err) + } + var expected map[string]json.RawMessage + if err := json.Unmarshal(raw, &expected); err != nil { + t.Fatal(err) + } + delete(expected, "transport_generation") + portable, err := json.Marshal(expected) + if err != nil { + t.Fatal(err) + } + assertExportJSONEqual(t, historical, portable) + if local || importTransportGeneration(t, pool, mailbox) == source { + t.Fatal("history gained authority or destination retained source proof") + } + }) + } + } +} diff --git a/internal/app/orgtransfer/spec.go b/internal/app/orgtransfer/spec.go index 7c583ca23..e5e1d0441 100644 --- a/internal/app/orgtransfer/spec.go +++ b/internal/app/orgtransfer/spec.go @@ -1,6 +1,9 @@ package orgtransfer -import "github.com/warmbly/warmbly/internal/models" +import ( + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) // KeyDomain names which key a sealed column was encrypted under. Warmbly has // two, and an archive that confuses them produces mailboxes that authenticate @@ -55,6 +58,12 @@ type SecretColumn struct { PlaintextPrefix string } +// JSONColumnExclusion names instance-local keys inside a stored JSON envelope. +type JSONColumnExclusion struct { + Column string + Keys []string +} + // Table is one exported relation and the policy for moving it. type Table struct { Name string @@ -80,6 +89,12 @@ type Table struct { // the rows. Blobs []BlobColumn + // ExcludeOnExport omits instance-local columns from rows and the manifest. + ExcludeOnExport []string + + // ExcludeJSONOnExport strips only named keys within registered JSON envelopes. + ExcludeJSONOnExport []JSONColumnExclusion + // ResetOnImport are columns left out of the insert because they name // something that exists only on the source instance: a queue handle, a // health counter, a last-seen timestamp. Omitting them is what makes the @@ -89,6 +104,11 @@ type Table struct { // ResetOnOverwrite also defaults reset columns when replacing destination rows. ResetOnOverwrite bool + // ResetColumnsOnOverwrite applies only selected instance-local defaults. + ResetColumnsOnOverwrite []string + + // InvalidateOnImport resets derived relations only for rows actually written. + InvalidateOnImport []repository.ImportInvalidation // ImportSkip exports the table for the record but never writes it back. ImportSkip bool @@ -199,7 +219,13 @@ var Tables = []Table{ }, { Name: "email_accounts", Group: models.OrgDataGroupCore, - Scope: scopeOrg, + Scope: scopeOrg, + ExcludeOnExport: []string{"transport_generation"}, + ResetColumnsOnOverwrite: []string{"transport_generation"}, + InvalidateOnImport: []repository.ImportInvalidation{{ + Table: "email_account_transport_recovery", Column: "email_account_id", Key: "id", + LockNamespaces: []int64{repository.SendResultMailboxLockNamespace, repository.MailTransportLockNamespace}, + }}, // Send cooldowns and recovery holds travel as mailbox columns; never reset them. // Worker placement is a property of the instance the mailbox runs on, // never of the mailbox. The destination assigns its own. @@ -227,6 +253,7 @@ var Tables = []Table{ // A tracking host is verified per instance and per workspace, so the // destination's own sweep verifies it again. ResetOnImport: []string{ + "transport_generation", "worker_id", "auth_checked_at", "auth_failing_since", "cold_ramp_started_at", "send_lifecycle", "send_lifecycle_since", "send_lifecycle_reason", "seed_scope", "avatar_checked_at", "tracking_domain_verified", "tracking_domain_verified_at", @@ -241,7 +268,8 @@ var Tables = []Table{ }, { Name: "email_accounts_smtp_imap", Group: models.OrgDataGroupCore, - Scope: `email_account_id IN ` + orgMailboxes, + Scope: `email_account_id IN ` + orgMailboxes, + InvalidateOnImport: mailboxCredentialInvalidations(), Secrets: []SecretColumn{ {Column: "smtp_host", Domain: KeyDomainInstance}, {Column: "smtp_user", Domain: KeyDomainInstance}, @@ -253,7 +281,8 @@ var Tables = []Table{ }, { Name: "email_accounts_oauth", Group: models.OrgDataGroupCore, - Scope: `email_account_id IN ` + orgMailboxes, + Scope: `email_account_id IN ` + orgMailboxes, + InvalidateOnImport: mailboxCredentialInvalidations(), Secrets: []SecretColumn{ {Column: "access_token", Domain: KeyDomainInstance}, {Column: "refresh_token", Domain: KeyDomainInstance}, @@ -779,6 +808,7 @@ var Tables = []Table{ // ---------- send pipeline ---------- { Name: "tasks", Group: models.OrgDataGroupSending, + ExcludeJSONOnExport: []JSONColumnExclusion{{Column: "send_executor_result", Keys: []string{"transport_generation"}}}, // A placement probe's task stays behind: a pending one would send from // the destination to the source instance's seeds. Scope: `email_account_id IN ` + orgMailboxes + ` AND task_type <> 'placement'`, @@ -792,11 +822,12 @@ var Tables = []Table{ }, { Name: "send_recovery_resolutions", Group: models.OrgDataGroupSending, - Scope: scopeOrg, - Owner: scopeOrg, - ResetOnImport: []string{"recorded_locally_at", "conflict_resolved_at"}, - ResetOnOverwrite: true, - Note: "Recovery evidence travels as history. Source-instance results and resolutions cannot execute or release a conflict hold after import; fresh local confirmation or independent authoritative worker evidence is required.", + ExcludeJSONOnExport: []JSONColumnExclusion{{Column: "confirmed_result", Keys: []string{"transport_generation"}}}, + Scope: scopeOrg, + Owner: scopeOrg, + ResetOnImport: []string{"recorded_locally_at", "conflict_resolved_at"}, + ResetOnOverwrite: true, + Note: "Recovery evidence travels as history. Source-instance results and resolutions cannot execute or release a conflict hold after import; fresh local confirmation or independent authoritative worker evidence is required.", }, { // An AI-group table, but it sits here because task_id points at tasks. @@ -824,9 +855,10 @@ var Tables = []Table{ }, { Name: "warmup_tasks", Group: models.OrgDataGroupSending, - PartnerRefs: []string{"target_account_id"}, - Scope: `task_id IN ` + orgTasks, - ResetOnImport: []string{"parent_task_id", "parent_received_id", "parent_message_id", "lineage_version", "schedule_revision", "queued_revision", "dispatch_nonce", "dispatch_worker_id", "dispatch_started_at", "dispatch_result", "warmup_charged_date", "warmup_reply_charged", "warmup_refunded_at"}, + ExcludeJSONOnExport: []JSONColumnExclusion{{Column: "dispatch_result", Keys: []string{"transport_generation"}}}, + PartnerRefs: []string{"target_account_id"}, + Scope: `task_id IN ` + orgTasks, + ResetOnImport: []string{"parent_task_id", "parent_received_id", "parent_message_id", "lineage_version", "schedule_revision", "queued_revision", "dispatch_nonce", "dispatch_worker_id", "dispatch_started_at", "dispatch_result", "warmup_charged_date", "warmup_reply_charged", "warmup_refunded_at"}, }, { Name: "warmup_tokens", Group: models.OrgDataGroupSending, @@ -860,7 +892,10 @@ var Tables = []Table{ }, { Name: "email_account_errors", Group: models.OrgDataGroupSending, - Scope: `email_account_id IN ` + orgMailboxes, + Scope: `email_account_id IN ` + orgMailboxes, + ResetOnImport: []string{"operation", "observation_source", "received_at"}, + ResetOnOverwrite: true, + Note: "Historical reports travel; operation and clock provenance reset to unknown on the destination.", }, { Name: "tracked_links", Group: models.OrgDataGroupSending, @@ -1012,53 +1047,54 @@ var Tables = []Table{ // with the reason. Kept as data so the docs page and the coverage test both // read from one list instead of restating it. var ExcludedTables = map[string]string{ - "sync_arrival_outbox": "Encrypted, org-key-owned arrival delivery queue bound to this instance's provider handles and body storage. Pending maps are excluded too so destination sync retries them; no queue secrets or blob references travel. Full instance backups retain queue, org keys and bodies together.", - "send_result_effects": "Source-instance reconciliation delivery queue. Pending webhook and notification effects must drain on the source; imports must not replay customer notifications. Full instance backups retain the durable outbox.", - "warmup_received": "Raw receiving-mailbox observations and provider thread handles are source-instance evidence, not portable send authority. Aggregated warmup placement history travels; imported threads cannot continue without new verified observations.", - "diagnostic_auth_verifications": "Bounded DKIM observations and grants bind source-instance workers, provider messages and cross-workspace diagnostic parents. They cannot authorize imported mail; destination verification starts unknown. Full instance database backups retain them.", - "warmup_pending_filings": "Provider filing awaiting acknowledgement on this instance. The destination resyncs mailbox messages.", - "unibox_pending_emails": "Unverified mailbox-sync events awaiting this instance's warmup checks. The destination resyncs provider mail with its own warmup and cloud-link state.", - "organization_encrypted_keys": "The organization's data key, wrapped by the source instance's KMS. The destination cannot unwrap it, and shipping it would put every org secret behind one exported blob.", - "api_idempotency_keys": "A short-lived replay cache for in-flight API requests.", - "realtime_events": "The websocket outbox. Every row is already delivered or expired.", - "integration_oauth_states": "In-flight OAuth handshakes, valid for minutes and bound to the source instance's redirect URL.", - "crm_sync_jobs": "The outbox of pending CRM writes on this instance; the destination's own events feed its outbox.", - "crm_sync_cursors": "Pull checkpoints for this instance; the destination starts its own pull.", - "salesforce_record_links": "Which Salesforce record each contact is, with a cached copy of it. The destination links contacts again by address the first time it syncs or shows them, and reads the record fresh.", - "salesforce_activity_queue": "Activity waiting to be logged in Salesforce, and the recent outcome of what was. What was logged is already in Salesforce; what was waiting belongs to this instance's drain.", - "salesforce_sync_state": "Where this instance's pull loop got to in each Salesforce org, and the API calls it counted today. The destination starts its own cursor when the connection first syncs.", - "oauth_authorization_codes": "Single-use authorization codes, valid for seconds.", - "oauth_developer_blocks": "An operator's decision on the source instance about who may build apps there; the destination's operators decide for theirs.", - "app_directory_listings": "A publication on the source instance's community directory, featured or hidden by its team. Publish again on the destination, where its own team decides.", - "scheduled_deletions": "Instance lifecycle state. Importing a pending deletion would schedule the destination workspace for destruction.", - "dedicated_worker_assignments": "Worker topology, which is a property of the instance rather than the workspace.", - "warmup_spam_moves": "Per-message attribution evidence for warmup mail this instance synced, kept only to decide recent tampering; the destination judges its own.", - "mailbox_owner_activity": "Five-minute buckets of sync-observed owner activity on this instance, read only to attribute recent spam moves.", - "warmup_pools": "Instance-global pool definitions shared by every workspace on the instance.", - "pool_link_codes": "In-flight link handshakes between a self-hosted instance and this cloud, valid for minutes.", - "cli_auth_codes": "In-flight `warmbly auth login` handshakes, valid for minutes. The API key an approval mints does travel, with the api_keys rows.", - "pool_link_instances": "Self-hosted instances linked to this workspace's pool allowance. The token hash only authenticates against this instance, and the enrolled mailboxes are mirrors of mailboxes that live elsewhere.", - "pool_link_mailboxes": "Which mailbox rows are warmup-only mirrors for a linked instance. They follow pool_link_instances, which does not travel.", - "cloud_link": "Workspace connections to Warmbly Cloud and legacy instance links: their encrypted tokens only work on this installation and do not travel.", - "cloud_link_mailboxes": "Which local mailboxes Warmbly Cloud warms for this instance. The enrollment belongs to the link, which does not travel.", - "warmup_conversations": "The instance's shared warmup content library, not workspace data.", - "copy_judgments": "A cache of copy judgments keyed by the hash of the words judged. The destination re-reads a step the first time its Advisor runs.", - "warmup_thread_messages": "Message identifiers this instance recognised as turns of a warmup conversation, so the reply to each is recognised too. The destination syncs provider mail afresh and rebuilds it from the warmup tokens, which do travel.", - "sessions": "Live login sessions. They are bound to the source instance's signing key and must not survive a move.", - "mailbox_erasures": "Erasure still owed for a mailbox this instance deleted: a grant to revoke at the provider, and message bodies to remove from this instance's blob store. Both name work on the instance that wrote the row, and the mailboxes are already gone.", - "login_history": "Where people signed in from, kept only to compare a new sign-in against recent ones. It belongs to the person rather than the workspace, and a destination must build its own baseline before it can call anything anomalous.", - "mailbox_imports": "Mailbox imports in progress or recently finished. They are work this instance is doing, and their rows hold credentials in flight, which live on only as the mailboxes they created.", - "mailbox_import_rows": "The rows of a mailbox import, with credentials sealed until each row is connected. They follow mailbox_imports, which does not travel.", - "contact_imports": "Contact imports in progress or recently finished. They are work this instance is doing; the contacts they created travel with the contacts group.", - "contact_import_rows": "The uploaded rows of a contact import and what became of each. They follow contact_imports, which does not travel.", - "placement_renders": "The copy a tracking comparison is sending to each seed, sealed so both halves send the same words. It lives only while the comparison runs, and a copy that had not been sent stays behind with its task.", - "inbox_follow_up_sweeps": "This instance's hourly follow-up sweep state for the workspace: where its cycle stopped (by this instance's mailbox and message row ids), how far it has checked changed conversations, and which walker holds it. The destination starts its own cycle at the newest conversation.", - "slack_user_links": "Which Slack member speaks for which Warmbly member. Slack delivers that member's messages to the instance whose Slack app the workspace installed, so each member links again after the workspace reconnects Slack on the destination.", - "slack_link_codes": "In-flight Slack account links, valid for minutes.", - "slack_agent_threads": "Which Slack thread the assistant answers in for which conversation. The Slack install they belong to does not travel; the conversations themselves do, with agent_sessions.", - "slack_inbox_threads": "Which Slack thread mirrors which inbox conversation. The Slack install and its channel do not travel; the conversations themselves do, with the unified inbox.", - "user_view_preferences": "Each member's own column layout and sort for the dashboard's lists, and their unibox scope rail arrangement. It belongs to the person rather than the workspace: members are matched by account on import and a layout names custom fields the destination may not hold yet, so everyone starts from the default view and picks their columns again.", - "campaign_send_plan_snapshots": "Today's precomputed send plan for a campaign, derived from the campaign, its leads, its mailboxes and this instance's limits, which all travel. Keyed to this instance's budget day, and naming mailboxes and workers. The destination's own background snapshotter recomputes it.", + "email_account_transport_recovery": "Source-instance sync and provider-confirmed-send recovery clocks cannot authorize destination recovery. Full instance database backups retain them.", + "sync_arrival_outbox": "Encrypted, org-key-owned arrival delivery queue bound to this instance's provider handles and body storage. Pending maps are excluded too so destination sync retries them; no queue secrets or blob references travel. Full instance backups retain queue, org keys and bodies together.", + "send_result_effects": "Source-instance reconciliation delivery queue. Pending webhook and notification effects must drain on the source; imports must not replay customer notifications. Full instance backups retain the durable outbox.", + "warmup_received": "Raw receiving-mailbox observations and provider thread handles are source-instance evidence, not portable send authority. Aggregated warmup placement history travels; imported threads cannot continue without new verified observations.", + "diagnostic_auth_verifications": "Bounded DKIM observations and grants bind source-instance workers, provider messages and cross-workspace diagnostic parents. They cannot authorize imported mail; destination verification starts unknown. Full instance database backups retain them.", + "warmup_pending_filings": "Provider filing awaiting acknowledgement on this instance. The destination resyncs mailbox messages.", + "unibox_pending_emails": "Unverified mailbox-sync events awaiting this instance's warmup checks. The destination resyncs provider mail with its own warmup and cloud-link state.", + "organization_encrypted_keys": "The organization's data key, wrapped by the source instance's KMS. The destination cannot unwrap it, and shipping it would put every org secret behind one exported blob.", + "api_idempotency_keys": "A short-lived replay cache for in-flight API requests.", + "realtime_events": "The websocket outbox. Every row is already delivered or expired.", + "integration_oauth_states": "In-flight OAuth handshakes, valid for minutes and bound to the source instance's redirect URL.", + "crm_sync_jobs": "The outbox of pending CRM writes on this instance; the destination's own events feed its outbox.", + "crm_sync_cursors": "Pull checkpoints for this instance; the destination starts its own pull.", + "salesforce_record_links": "Which Salesforce record each contact is, with a cached copy of it. The destination links contacts again by address the first time it syncs or shows them, and reads the record fresh.", + "salesforce_activity_queue": "Activity waiting to be logged in Salesforce, and the recent outcome of what was. What was logged is already in Salesforce; what was waiting belongs to this instance's drain.", + "salesforce_sync_state": "Where this instance's pull loop got to in each Salesforce org, and the API calls it counted today. The destination starts its own cursor when the connection first syncs.", + "oauth_authorization_codes": "Single-use authorization codes, valid for seconds.", + "oauth_developer_blocks": "An operator's decision on the source instance about who may build apps there; the destination's operators decide for theirs.", + "app_directory_listings": "A publication on the source instance's community directory, featured or hidden by its team. Publish again on the destination, where its own team decides.", + "scheduled_deletions": "Instance lifecycle state. Importing a pending deletion would schedule the destination workspace for destruction.", + "dedicated_worker_assignments": "Worker topology, which is a property of the instance rather than the workspace.", + "warmup_spam_moves": "Per-message attribution evidence for warmup mail this instance synced, kept only to decide recent tampering; the destination judges its own.", + "mailbox_owner_activity": "Five-minute buckets of sync-observed owner activity on this instance, read only to attribute recent spam moves.", + "warmup_pools": "Instance-global pool definitions shared by every workspace on the instance.", + "pool_link_codes": "In-flight link handshakes between a self-hosted instance and this cloud, valid for minutes.", + "cli_auth_codes": "In-flight `warmbly auth login` handshakes, valid for minutes. The API key an approval mints does travel, with the api_keys rows.", + "pool_link_instances": "Self-hosted instances linked to this workspace's pool allowance. The token hash only authenticates against this instance, and the enrolled mailboxes are mirrors of mailboxes that live elsewhere.", + "pool_link_mailboxes": "Which mailbox rows are warmup-only mirrors for a linked instance. They follow pool_link_instances, which does not travel.", + "cloud_link": "Workspace connections to Warmbly Cloud and legacy instance links: their encrypted tokens only work on this installation and do not travel.", + "cloud_link_mailboxes": "Which local mailboxes Warmbly Cloud warms for this instance. The enrollment belongs to the link, which does not travel.", + "warmup_conversations": "The instance's shared warmup content library, not workspace data.", + "copy_judgments": "A cache of copy judgments keyed by the hash of the words judged. The destination re-reads a step the first time its Advisor runs.", + "warmup_thread_messages": "Message identifiers this instance recognised as turns of a warmup conversation, so the reply to each is recognised too. The destination syncs provider mail afresh and rebuilds it from the warmup tokens, which do travel.", + "sessions": "Live login sessions. They are bound to the source instance's signing key and must not survive a move.", + "mailbox_erasures": "Erasure still owed for a mailbox this instance deleted: a grant to revoke at the provider, and message bodies to remove from this instance's blob store. Both name work on the instance that wrote the row, and the mailboxes are already gone.", + "login_history": "Where people signed in from, kept only to compare a new sign-in against recent ones. It belongs to the person rather than the workspace, and a destination must build its own baseline before it can call anything anomalous.", + "mailbox_imports": "Mailbox imports in progress or recently finished. They are work this instance is doing, and their rows hold credentials in flight, which live on only as the mailboxes they created.", + "mailbox_import_rows": "The rows of a mailbox import, with credentials sealed until each row is connected. They follow mailbox_imports, which does not travel.", + "contact_imports": "Contact imports in progress or recently finished. They are work this instance is doing; the contacts they created travel with the contacts group.", + "contact_import_rows": "The uploaded rows of a contact import and what became of each. They follow contact_imports, which does not travel.", + "placement_renders": "The copy a tracking comparison is sending to each seed, sealed so both halves send the same words. It lives only while the comparison runs, and a copy that had not been sent stays behind with its task.", + "inbox_follow_up_sweeps": "This instance's hourly follow-up sweep state for the workspace: where its cycle stopped (by this instance's mailbox and message row ids), how far it has checked changed conversations, and which walker holds it. The destination starts its own cycle at the newest conversation.", + "slack_user_links": "Which Slack member speaks for which Warmbly member. Slack delivers that member's messages to the instance whose Slack app the workspace installed, so each member links again after the workspace reconnects Slack on the destination.", + "slack_link_codes": "In-flight Slack account links, valid for minutes.", + "slack_agent_threads": "Which Slack thread the assistant answers in for which conversation. The Slack install they belong to does not travel; the conversations themselves do, with agent_sessions.", + "slack_inbox_threads": "Which Slack thread mirrors which inbox conversation. The Slack install and its channel do not travel; the conversations themselves do, with the unified inbox.", + "user_view_preferences": "Each member's own column layout and sort for the dashboard's lists, and their unibox scope rail arrangement. It belongs to the person rather than the workspace: members are matched by account on import and a layout names custom fields the destination may not hold yet, so everyone starts from the default view and picks their columns again.", + "campaign_send_plan_snapshots": "Today's precomputed send plan for a campaign, derived from the campaign, its leads, its mailboxes and this instance's limits, which all travel. Keyed to this instance's budget day, and naming mailboxes and workers. The destination's own background snapshotter recomputes it.", } // TableByName indexes Tables for lookup during import. @@ -1087,9 +1123,14 @@ func GroupTables(groups []models.OrgDataGroup) []*Table { return out } -// NormalizeGroups validates a requested group list, adds core (which every -// archive needs to be importable at all), and closes over the catalog's -// dependencies so a selection can never be one that fails on a foreign key. +func mailboxCredentialInvalidations() []repository.ImportInvalidation { + return []repository.ImportInvalidation{ + {Table: "email_accounts", Column: "id", Key: "email_account_id", ResetColumns: []string{"transport_generation"}, LockNamespaces: []int64{repository.SendResultMailboxLockNamespace, repository.MailTransportLockNamespace}}, + {Table: "email_account_transport_recovery", Column: "email_account_id", Key: "email_account_id"}, + } +} + +// NormalizeGroups validates the selection and closes over group dependencies. func NormalizeGroups(groups []models.OrgDataGroup) []models.OrgDataGroup { if len(groups) == 0 { return append([]models.OrgDataGroup(nil), models.AllOrgDataGroups...) diff --git a/internal/app/worker/event_add_email.go b/internal/app/worker/event_add_email.go index 4051bb4b1..6d022a0b7 100644 --- a/internal/app/worker/event_add_email.go +++ b/internal/app/worker/event_add_email.go @@ -4,7 +4,6 @@ import ( "context" "errors" "sync/atomic" - "time" "github.com/google/uuid" "github.com/rs/zerolog/log" @@ -140,6 +139,7 @@ func (w *WorkerService) reportLoadFailure(e *models.AddWorkerEmail, err error) { } info := mailErr.GetUserErrorInfo() event := models.EmailErrorEvent{ + Operation: models.MailTransportUnknown, EmailAccountID: e.ID.String(), UserID: e.UserID.String(), ErrorCode: string(mailErr.Code), @@ -150,7 +150,7 @@ func (w *WorkerService) reportLoadFailure(e *models.AddWorkerEmail, err error) { UserTitle: info.Title, UserMessage: info.Message, ActionRequired: info.ActionRequired, - Timestamp: time.Now().Unix(), + Timestamp: wmail.ErrorObservedTimestamp(mailErr), } if perr := w.mailManager.OnEvent(eventType, e.ID.String(), event); perr != nil { log.Error().Err(perr).Str("email_id", e.ID.String()).Msg("failed to report a mailbox that did not load") diff --git a/internal/app/worker/event_send_email.go b/internal/app/worker/event_send_email.go index c95aba5f9..84ecdddc0 100644 --- a/internal/app/worker/event_send_email.go +++ b/internal/app/worker/event_send_email.go @@ -89,6 +89,7 @@ func (w *WorkerService) HandleSendEmail(ctx context.Context, sendEmail models.Se if sendEmail.OrgID != uuid.Nil && (mail.OrgID == nil || *mail.OrgID != sendEmail.OrgID) { return errors.New("send organization does not own loaded mailbox") } + transportGeneration := mail.TransportGeneration // Decrypt subject subject := sendEmail.Subject @@ -164,7 +165,7 @@ func (w *WorkerService) HandleSendEmail(ctx context.Context, sendEmail models.Se }) w.recordSendLatency(time.Since(sendStart)) w.recordSendOutcome(result) - stored := models.SendEmailResult{TaskID: sendEmail.TaskID, Success: result.Success, MessageID: result.MessageID, ProviderMsgID: result.ProviderMsgID, ThreadID: result.ThreadID, SentAt: time.Now().UTC()} + stored := models.SendEmailResult{TransportGeneration: transportGeneration, TaskID: sendEmail.TaskID, Success: result.Success, MessageID: result.MessageID, ProviderMsgID: result.ProviderMsgID, ThreadID: result.ThreadID, SentAt: time.Now().UTC()} if result.Error != nil { stored.Error = wmail.MailErrorToSendError(result.Error) stored.LegacyErrorMsg = result.Error.Message @@ -376,18 +377,20 @@ func (w *WorkerService) sendMailboxError(taskID uuid.UUID, emailID uuid.UUID, ma userInfo := mailErr.GetUserErrorInfo() errorEvent := models.EmailErrorEvent{ - TaskID: taskID.String(), - EmailAccountID: emailID.String(), - UserID: mail.UserID.String(), - ErrorCode: string(mailErr.Code), - ErrorType: string(mailErr.Type), - ResolveMethod: string(mailErr.ResolveMethod), - Message: mailErr.Message, - UserVisible: mailErr.IsUserVisible(), - UserTitle: userInfo.Title, - UserMessage: userInfo.Message, - ActionRequired: userInfo.ActionRequired, - Timestamp: time.Now().Unix(), + TransportGeneration: mail.TransportGeneration, + Operation: models.MailTransportSend, + TaskID: taskID.String(), + EmailAccountID: emailID.String(), + UserID: mail.UserID.String(), + ErrorCode: string(mailErr.Code), + ErrorType: string(mailErr.Type), + ResolveMethod: string(mailErr.ResolveMethod), + Message: mailErr.Message, + UserVisible: mailErr.IsUserVisible(), + UserTitle: userInfo.Title, + UserMessage: userInfo.Message, + ActionRequired: userInfo.ActionRequired, + Timestamp: wmail.ErrorObservedTimestamp(mailErr), } if err := w.Produce(eventType, emailID.String(), errorEvent); err != nil { diff --git a/internal/app/worker/mailmanager/transport_generation_test.go b/internal/app/worker/mailmanager/transport_generation_test.go new file mode 100644 index 000000000..14efb360d --- /dev/null +++ b/internal/app/worker/mailmanager/transport_generation_test.go @@ -0,0 +1,55 @@ +package mailmanager + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +func TestMailTransportGenerationReplacesUnchangedCredentialsAndDrains(t *testing.T) { + m := NewMailManager(func(models.JobEventType, string, any) error { return nil }, nil, nil, nil, nil, nil, nil) + data := smtpPayload() + data.TransportGeneration = uuid.New() + at := time.Now().Add(-time.Hour) + data.Sync.State.TransportGeneration = data.TransportGeneration + data.Sync.State.LastSyncedAt = &at + if err := m.AddWMail(t.Context(), data); err != nil { + t.Fatal(err) + } + old := m.Get(data.ID) + done, ok := old.BeginExecution() + if !ok { + t.Fatal("old execution not admitted") + } + copied := *data + copied.TransportGeneration = uuid.New() + result := make(chan error, 1) + go func() { result <- m.AddWMail(t.Context(), &copied) }() + select { + case <-old.Ctx.Done(): + case <-time.After(time.Second): + t.Fatal("generation did not replace unchanged credentials") + } + if m.Get(data.ID) != old || old.TransportGeneration != data.TransportGeneration { + t.Fatal("old operation relabelled or not drained") + } + done() + select { + case err := <-result: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("replacement blocked") + } + next := m.Get(data.ID) + if next == old || next.TransportGeneration != copied.TransportGeneration { + t.Fatal("fresh incarnation missing") + } + if data.TransportGeneration == copied.TransportGeneration { + t.Fatal("source snapshot mutated") + } + m.Terminate(data.ID) +} diff --git a/internal/app/worker/wmail/err.go b/internal/app/worker/wmail/err.go index 428e89ab7..ec2e000de 100644 --- a/internal/app/worker/wmail/err.go +++ b/internal/app/worker/wmail/err.go @@ -37,6 +37,14 @@ func reportable(err error) bool { } func (w *WMail) CaptureError(err error) { + w.captureOperationError(err, models.MailTransportUnknown) +} + +func (w *WMail) captureSyncError(err error) { + w.captureOperationError(err, models.MailTransportSync) +} + +func (w *WMail) captureOperationError(err error, operation models.MailTransportOperation) { w.lifecycleMu.Lock() stopped := w.stopped w.lifecycleMu.Unlock() @@ -64,17 +72,19 @@ func (w *WMail) CaptureError(err error) { userInfo := mailErr.GetUserErrorInfo() errorEvent := models.EmailErrorEvent{ - EmailAccountID: w.ID.String(), - UserID: w.UserID.String(), - ErrorCode: string(mailErr.Code), - ErrorType: string(mailErr.Type), - ResolveMethod: string(mailErr.ResolveMethod), - Message: mailErr.Message, - UserVisible: mailErr.IsUserVisible(), - UserTitle: userInfo.Title, - UserMessage: userInfo.Message, - ActionRequired: userInfo.ActionRequired, - Timestamp: time.Now().Unix(), + TransportGeneration: w.TransportGeneration, + Operation: operation, + EmailAccountID: w.ID.String(), + UserID: w.UserID.String(), + ErrorCode: string(mailErr.Code), + ErrorType: string(mailErr.Type), + ResolveMethod: string(mailErr.ResolveMethod), + Message: mailErr.Message, + UserVisible: mailErr.IsUserVisible(), + UserTitle: userInfo.Title, + UserMessage: userInfo.Message, + ActionRequired: userInfo.ActionRequired, + Timestamp: ErrorObservedTimestamp(mailErr), } w.lifecycleMu.Lock() @@ -93,6 +103,13 @@ func (w *WMail) CaptureError(err error) { } } +func ErrorObservedTimestamp(mailErr *errx.MailError) int64 { + if mailErr.Failure != nil && !mailErr.Failure.ObservedAt.IsZero() { + return mailErr.Failure.ObservedAt.Unix() + } + return time.Now().Unix() +} + // mailErrorToJobEventType maps a mail error code to the matching job event type // so the consumer knows how to handle it. func mailErrorToJobEventType(mailErr *errx.MailError) models.JobEventType { diff --git a/internal/app/worker/wmail/err_test.go b/internal/app/worker/wmail/err_test.go index ef9238eb0..4af91c03f 100644 --- a/internal/app/worker/wmail/err_test.go +++ b/internal/app/worker/wmail/err_test.go @@ -2,11 +2,33 @@ package wmail import ( "testing" + "time" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/models" ) +func TestMailTransportErrorScopesAndWorkerObservation(t *testing.T) { + observed := time.Now().Add(-time.Hour).UTC().Truncate(time.Second) + err := errx.WithSendFailure(errx.ErrMailServerUnreachable, errx.SendFailure{ObservedAt: observed}) + w := &WMail{} + var events []models.EmailErrorEvent + w.onEvent = func(_ models.JobEventType, body any) error { + events = append(events, body.(models.EmailErrorEvent)) + return nil + } + w.CaptureError(err) + w.captureSyncError(err) + if len(events) != 2 || events[0].Operation != models.MailTransportUnknown || events[1].Operation != models.MailTransportSync { + t.Fatalf("general error was inferred sync-only: %+v", events) + } + for _, event := range events { + if event.Timestamp != observed.Unix() || event.TaskID != "" { + t.Fatalf("worker clock or context changed: %+v", event) + } + } +} + func TestTransientProviderFailuresAreNotReportedAsApplicationErrors(t *testing.T) { for _, mailErr := range []*errx.MailError{ errx.ErrMailServerUnreachable, diff --git a/internal/app/worker/wmail/lifecycle.go b/internal/app/worker/wmail/lifecycle.go index acfa8eaf6..f6e7be246 100644 --- a/internal/app/worker/wmail/lifecycle.go +++ b/internal/app/worker/wmail/lifecycle.go @@ -22,6 +22,7 @@ func ExecutionKey(data *models.AddWorkerEmail) [32]byte { return credential{t.AccessToken, t.RefreshToken} } key := struct { + TransportGeneration uuid.UUID ID, User uuid.UUID Org *uuid.UUID Provider models.InboxProvider @@ -31,7 +32,8 @@ func ExecutionKey(data *models.AddWorkerEmail) [32]byte { SMTP, IMAP *models.Service Skip []string }{ID: data.ID, User: data.UserID, Org: data.OrganizationID, Provider: data.Type, - Email: data.Email, First: data.FirstName, Last: data.LastName, Brokered: data.Brokered, + TransportGeneration: data.TransportGeneration, + Email: data.Email, First: data.FirstName, Last: data.LastName, Brokered: data.Brokered, ImapSync: data.Type == models.InboxProviderSMTPIMAP && data.ImapSync, SaveToSent: data.Type == models.InboxProviderSMTPIMAP && data.SavesSentCopy()} if data.Type == models.InboxProviderGoogle && data.Google != nil && !data.Brokered { key.Token = tokenKey(data.Google.Token) @@ -138,6 +140,7 @@ func (w *WMail) ResumeInto(data *models.AddWorkerEmail) { data.Sync = &s } state := w.tracker.state + state.TransportGeneration = uuid.Nil state.BackfillCursor.Folders = make(map[string]models.SyncFolderCursor, len(w.tracker.state.BackfillCursor.Folders)) for k, v := range w.tracker.state.BackfillCursor.Folders { state.BackfillCursor.Folders[k] = v diff --git a/internal/app/worker/wmail/start_sync.go b/internal/app/worker/wmail/start_sync.go index 0bc00e8ef..17e9a4850 100644 --- a/internal/app/worker/wmail/start_sync.go +++ b/internal/app/worker/wmail/start_sync.go @@ -87,7 +87,7 @@ func (w *WMail) syncOnce(ctx context.Context) (result *errx.MailError) { w.tracker.tickComplete = false } err := fmt.Errorf("mail sync panic: %v", r) - w.CaptureError(err) + w.captureSyncError(err) log.Error().Err(err).Str("email_id", w.ID.String()).Msg("mail sync panicked") result = &errx.MailError{Code: errx.MailErrorCodeImapUnknown, Type: errx.MailErrorWarning, Message: "mail sync pass interrupted"} } @@ -111,12 +111,12 @@ func (w *WMail) syncOnce(ctx context.Context) (result *errx.MailError) { if isTransportError(err) { w.transportFailures++ if w.transportFailures > 1 { - w.CaptureError(err) + w.captureSyncError(err) log.Debug().Err(err).Str("email_id", w.ID.String()).Int("consecutive", w.transportFailures).Msg("mail server still unreachable") return err } } - w.CaptureError(err) + w.captureSyncError(err) log.Warn().Err(err).Str("email_id", w.ID.String()).Msg("mail sync error") return err } diff --git a/internal/app/worker/wmail/sync_google.go b/internal/app/worker/wmail/sync_google.go index 6db4d9061..6759d0caf 100644 --- a/internal/app/worker/wmail/sync_google.go +++ b/internal/app/worker/wmail/sync_google.go @@ -60,7 +60,7 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError { if newHistoryID != 0 && newHistoryID != w.GoogleData.LastHistoryID { // Keep the old cursor until publication succeeds, so a failed relay is retried. if perr := w.NewHistoryID(newHistoryID); perr != nil { - w.CaptureError(perr) + w.captureSyncError(perr) return nil } w.GoogleData.LastHistoryID = newHistoryID @@ -70,7 +70,7 @@ func (w *WMail) SyncGoogle(ctx context.Context) *errx.MailError { if errors.As(err, &errMail) { return errMail } - w.CaptureError(err) + w.captureSyncError(err) return nil } @@ -350,7 +350,7 @@ func (w *WMail) googleBackfill(ctx context.Context, stats *tickStats) *errx.Mail if errors.As(err, &errMail) { return errMail } - w.CaptureError(err) + w.captureSyncError(err) return nil } for _, id := range ids { @@ -377,7 +377,7 @@ func (w *WMail) googleBackfill(ctx context.Context, stats *tickStats) *errx.Mail if errors.As(err, &errMail) { return errMail } - w.CaptureError(err) + w.captureSyncError(err) return nil } if msg == nil { @@ -536,11 +536,11 @@ func (w *WMail) emitFolder(id uuid.UUID, folder string) error { func (w *WMail) googleReconcileError(err error) *errx.MailError { var errMail *errx.MailError if !errors.As(err, &errMail) { - w.CaptureError(err) + w.captureSyncError(err) return nil } if errMail.Type != errx.MailErrorCritical && !gmailRetryable(err) { - w.CaptureError(err) + w.captureSyncError(err) } return errMail } diff --git a/internal/app/worker/wmail/sync_graph.go b/internal/app/worker/wmail/sync_graph.go index 743a533ac..b83111416 100644 --- a/internal/app/worker/wmail/sync_graph.go +++ b/internal/app/worker/wmail/sync_graph.go @@ -41,7 +41,7 @@ func (w *WMail) SyncGraph(ctx context.Context) *errx.MailError { done, err := w.graphUpgradeIDs(ctx) if errors.Is(err, repository.ErrSyncContextUnsupported) { w.GraphData.Client.SetImmutableIDMode(false) - w.CaptureError(err) + w.captureSyncError(err) done, err = true, nil } if err != nil { @@ -61,7 +61,7 @@ func (w *WMail) SyncGraph(ctx context.Context) *errx.MailError { if errors.As(err, &mailErr) { return mailErr } - w.CaptureError(err) + w.captureSyncError(err) return nil } if !stats.aborted { @@ -260,7 +260,7 @@ func (w *WMail) graphBackfill(ctx context.Context, stats *tickStats) *errx.MailE } return mailErr } - w.CaptureError(err) + w.captureSyncError(err) stats.aborted = true return nil } diff --git a/internal/app/worker/wmail/sync_graph_reconcile.go b/internal/app/worker/wmail/sync_graph_reconcile.go index 02524619e..20b946c5d 100644 --- a/internal/app/worker/wmail/sync_graph_reconcile.go +++ b/internal/app/worker/wmail/sync_graph_reconcile.go @@ -160,7 +160,7 @@ func (w *WMail) graphSyncError(err error) *errx.MailError { if errors.As(err, &merr) { return merr } - w.CaptureError(err) + w.captureSyncError(err) return nil } diff --git a/internal/app/worker/wmail/sync_state.go b/internal/app/worker/wmail/sync_state.go index d896d05df..ca8dc7db5 100644 --- a/internal/app/worker/wmail/sync_state.go +++ b/internal/app/worker/wmail/sync_state.go @@ -3,6 +3,7 @@ package wmail import ( "time" + "github.com/google/uuid" "github.com/rs/zerolog/log" "github.com/warmbly/warmbly/internal/models" ) @@ -16,11 +17,12 @@ const syncStateHeartbeat = 10 * time.Minute // of the tick; the consumer persists it and the loader hands it back on the // next assignment. type syncTracker struct { - state models.SyncState - dirty bool - lastSent time.Time - emit func(models.SyncState) error - tickComplete bool + transportGeneration uuid.UUID + state models.SyncState + dirty bool + lastSent time.Time + emit func(models.SyncState) error + tickComplete bool } func newSyncTracker(seed *models.SyncState, emit func(models.SyncState) error) *syncTracker { @@ -28,6 +30,8 @@ func newSyncTracker(seed *models.SyncState, emit func(models.SyncState) error) * if seed != nil { t.state = *seed } + // A loaded checkpoint is not a completed pass in this worker incarnation. + t.state.TransportGeneration = uuid.Nil if t.state.BackfillStatus == "" { t.state.BackfillStatus = models.SyncBackfillPending } @@ -86,6 +90,7 @@ func (t *syncTracker) setFoldersSkipped(cap, conflict int) { func (t *syncTracker) touch(now time.Time) { stamp := now t.state.LastSyncedAt = &stamp + t.state.TransportGeneration = t.transportGeneration t.flush(now) } diff --git a/internal/app/worker/wmail/transport_generation_test.go b/internal/app/worker/wmail/transport_generation_test.go new file mode 100644 index 000000000..a46ecbe10 --- /dev/null +++ b/internal/app/worker/wmail/transport_generation_test.go @@ -0,0 +1,50 @@ +package wmail + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +func TestMailTransportGenerationRequiresFreshCompletedSyncPass(t *testing.T) { + old, current := uuid.New(), uuid.New() + prior := time.Now().Add(-time.Hour) + seed := &models.SyncState{TransportGeneration: old, LastSyncedAt: &prior, BackfillStatus: models.SyncBackfillComplete} + var events []models.SyncState + tracker := newSyncTracker(seed, func(state models.SyncState) error { events = append(events, state); return nil }) + tracker.transportGeneration = current + tracker.mark() + tracker.flush(time.Now()) + if len(events) != 1 || events[0].TransportGeneration != uuid.Nil || !events[0].LastSyncedAt.Equal(prior) { + t.Fatal("seed/progress re-badged as current success") + } + mail := &WMail{TransportGeneration: current, tracker: tracker} + cancelled, cancel := context.WithCancel(t.Context()) + cancel() + mail.endTick(cancelled, &tickStats{}, true) + mail.endTick(t.Context(), &tickStats{aborted: true}, true) + mail.endTick(t.Context(), &tickStats{deferred: 1}, true) + if tracker.state.TransportGeneration != uuid.Nil { + t.Fatal("cancelled/partial/deferred pass created proof") + } + mail.endTick(t.Context(), &tickStats{}, true) + if tracker.state.TransportGeneration != current || !tracker.state.LastSyncedAt.After(prior) { + t.Fatal("completed fresh pass lacks proof") + } + data := &models.AddWorkerEmail{TransportGeneration: uuid.New()} + priorMail := &WMail{tracker: tracker} + priorMail.ResumeInto(data) + if data.TransportGeneration == current || data.Sync.State.TransportGeneration != uuid.Nil { + t.Fatal("resume copied obsolete proof or overwrote fresh mailbox generation") + } + replacement := newSyncTracker(data.Sync.State, func(models.SyncState) error { return nil }) + next := &WMail{tracker: replacement} + next.PreservePending(priorMail) + replacement.flush(time.Now().Add(syncStateHeartbeat)) + if replacement.state.TransportGeneration != uuid.Nil { + t.Fatal("pending heartbeat carried obsolete proof") + } +} diff --git a/internal/app/worker/wmail/wmail.go b/internal/app/worker/wmail/wmail.go index 260bf4584..1ada85534 100644 --- a/internal/app/worker/wmail/wmail.go +++ b/internal/app/worker/wmail/wmail.go @@ -59,6 +59,7 @@ type SmtpImapData struct { } type WMail struct { + TransportGeneration uuid.UUID lifecycleMu sync.Mutex stopped bool stopDone chan struct{} @@ -155,13 +156,14 @@ func NewWMail( mailCtx, cancel := context.WithCancel(context.Background()) mail := &WMail{ - ID: data.ID, - UserID: data.UserID, - OrgID: data.OrganizationID, - Email: data.Email, - FirstName: data.FirstName, - LastName: data.LastName, - EmailType: data.Type, + ID: data.ID, + UserID: data.UserID, + TransportGeneration: data.TransportGeneration, + OrgID: data.OrganizationID, + Email: data.Email, + FirstName: data.FirstName, + LastName: data.LastName, + EmailType: data.Type, // Unset in the payload means yes; see AddWorkerEmail.SavesSentCopy. SaveToSent: data.Type == models.InboxProviderSMTPIMAP && data.SavesSentCopy(), onEvent: func(jobType models.JobEventType, body any) error { @@ -207,6 +209,7 @@ func NewWMail( State: st, }) }) + mail.tracker.transportGeneration = mail.TransportGeneration switch data.Type { case models.InboxProviderGoogle: diff --git a/internal/infrastructure/codec/mail_transport_compatibility_test.go b/internal/infrastructure/codec/mail_transport_compatibility_test.go new file mode 100644 index 000000000..d706280a6 --- /dev/null +++ b/internal/infrastructure/codec/mail_transport_compatibility_test.go @@ -0,0 +1,138 @@ +package codec + +import ( + "encoding/json" + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +type priorTransportBody struct { + ID uuid.UUID `json:"id" avro:"id"` + TaskID uuid.UUID `json:"task_id" avro:"task_id"` + Success bool `json:"success" avro:"success"` + MessageID string `json:"message_id" avro:"message_id"` + EmailID uuid.UUID `json:"email_id" avro:"email_id"` + EmailAccountID string `json:"email_account_id" avro:"email_account_id"` + State struct { + LastSyncedAt *time.Time `json:"last_synced_at" avro:"last_synced_at"` + } `json:"state" avro:"state"` +} +type priorTransportEnvelope struct { + Type string `json:"type" avro:"type"` + Body priorTransportBody `json:"body" avro:"body"` +} + +func TestMailTransportCodecJSONCompatibility(t *testing.T) { + c := NewJSON() + generation, task := uuid.New(), uuid.New() + at := time.Now().UTC() + event := models.JobEvent{Type: models.JobEventTypeEmailSent, Body: models.SendEmailResult{TaskID: task, Success: true, MessageID: "accepted@example.test", SentAt: at, TransportGeneration: generation}} + payload, err := c.Serialize(t.Context(), "results", event) + if err != nil { + t.Fatal(err) + } + var prior priorTransportEnvelope + if err := c.Deserialize(t.Context(), "results", payload, &prior); err != nil { + t.Fatal(err) + } + if prior.Body.TaskID != task || !prior.Body.Success || prior.Body.MessageID != "accepted@example.test" { + t.Fatal("old reader lost genuine outcome") + } + old := priorTransportEnvelope{Type: string(models.JobEventTypeEmailSent), Body: priorTransportBody{TaskID: task, Success: true}} + payload, err = c.Serialize(t.Context(), "results", old) + if err != nil { + t.Fatal(err) + } + var current struct { + Body models.SendEmailResult `json:"body"` + } + if err := c.Deserialize(t.Context(), "results", payload, ¤t); err != nil { + t.Fatal(err) + } + result := current.Body + if result.TransportGeneration != uuid.Nil || !result.Success || result.TaskID != task { + t.Fatal("old writer changed evidence or manufactured proof") + } + mailbox := uuid.New() + for _, kind := range []string{"add", "sync", "error"} { + t.Run(kind, func(t *testing.T) { + var body any + var eventType models.JobEventType + switch kind { + case "add": + body = models.AddWorkerEmail{ID: mailbox, TransportGeneration: generation} + eventType = models.JobEventType(models.WorkerEventTypeAddEmail) + case "sync": + body = &models.JobEventSyncState{EmailID: mailbox, State: models.SyncState{LastSyncedAt: &at, TransportGeneration: generation}} + eventType = models.JobEventTypeSyncState + case "error": + body = models.EmailErrorEvent{EmailAccountID: mailbox.String(), TransportGeneration: generation, Operation: models.MailTransportSync} + eventType = models.JobEventTypeEmailServerError + } + var envelope any = models.JobEvent{Type: eventType, Body: body} + if kind == "add" { + envelope = models.WorkerEvent{Type: models.WorkerEventTypeAddEmail, Body: body} + } + payload, err := c.Serialize(t.Context(), "transport", envelope) + if err != nil { + t.Fatal(err) + } + var legacy struct { + Body struct { + ID uuid.UUID `json:"id"` + EmailID uuid.UUID `json:"email_id"` + EmailAccountID string `json:"email_account_id"` + } + } + if err := c.Deserialize(t.Context(), "transport", payload, &legacy); err != nil { + t.Fatal(err) + } + if legacy.Body.ID != mailbox && legacy.Body.EmailID != mailbox && legacy.Body.EmailAccountID != mailbox.String() { + t.Fatal("old reader lost identity") + } + var old map[string]any + if err := json.Unmarshal(payload, &old); err != nil { + t.Fatal(err) + } + oldBody := old["body"].(map[string]any) + delete(oldBody, "transport_generation") + delete(oldBody, "operation") + if state, ok := oldBody["state"].(map[string]any); ok { + delete(state, "transport_generation") + } + payload, err = c.Serialize(t.Context(), "transport", old) + if err != nil { + t.Fatal(err) + } + switch kind { + case "add": + var current struct{ Body models.AddWorkerEmail } + if err := c.Deserialize(t.Context(), "transport", payload, ¤t); err != nil { + t.Fatal(err) + } + if current.Body.TransportGeneration != uuid.Nil || current.Body.ID != mailbox { + t.Fatal("old add manufactured proof") + } + case "sync": + var current struct{ Body models.JobEventSyncState } + if err := c.Deserialize(t.Context(), "transport", payload, ¤t); err != nil { + t.Fatal(err) + } + if current.Body.State.TransportGeneration != uuid.Nil || current.Body.State.LastSyncedAt == nil || !current.Body.State.LastSyncedAt.Equal(at) { + t.Fatal("old sync changed checkpoint/proof") + } + case "error": + var current struct{ Body models.EmailErrorEvent } + if err := c.Deserialize(t.Context(), "transport", payload, ¤t); err != nil { + t.Fatal(err) + } + if current.Body.TransportGeneration != uuid.Nil || current.Body.Operation.Known() || current.Body.EmailAccountID != mailbox.String() { + t.Fatal("old error manufactured proof/operation") + } + } + }) + } +} diff --git a/internal/infrastructure/codec/mail_transport_registry_test.go b/internal/infrastructure/codec/mail_transport_registry_test.go new file mode 100644 index 000000000..15a856ce7 --- /dev/null +++ b/internal/infrastructure/codec/mail_transport_registry_test.go @@ -0,0 +1,222 @@ +//go:build kafka + +package codec + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/confluentinc/confluent-kafka-go/v2/schemaregistry" + "github.com/google/uuid" + "github.com/iskorotkov/avro/v2" + "github.com/warmbly/warmbly/internal/models" +) + +// The production encoder must see the original concrete envelope. +func encodeTransportSchema(ctx context.Context, c *AvroCodec, topic string, schema avro.Schema, value any) ([]byte, error) { + id, err := c.register(subjectFor(topic), schema) + if err != nil { + return nil, err + } + body, err := models.EventAvro.Marshal(schema, value) + if err != nil { + return nil, err + } + out := []byte{0, byte(id >> 24), byte(id >> 16), byte(id >> 8), byte(id)} + return append(out, body...), nil +} + +func priorTransportSchema(t *testing.T, schema avro.Schema) avro.Schema { + t.Helper() + doc, err := models.SchemaDocument(schema) + if err != nil { + t.Fatal(err) + } + var root any + if err := json.Unmarshal([]byte(doc), &root); err != nil { + t.Fatal(err) + } + var strip func(any) + strip = func(v any) { + switch x := v.(type) { + case map[string]any: + if fields, ok := x["fields"].([]any); ok { + kept := fields[:0] + for _, f := range fields { + record := f.(map[string]any) + if record["name"] == "transport_generation" || x["name"] == "EmailErrorEvent" && record["name"] == "operation" { + continue + } + strip(f) + kept = append(kept, f) + } + x["fields"] = kept + } + for k, value := range x { + if k != "fields" { + strip(value) + } + } + case []any: + for _, value := range x { + strip(value) + } + } + } + strip(root) + old, err := json.Marshal(root) + if err != nil { + t.Fatal(err) + } + parsed, err := avro.Parse(string(old)) + if err != nil { + t.Fatal(err) + } + return parsed +} + +func TestMailTransportCodecRegistryCompatibility(t *testing.T) { + if encoded := os.Getenv("WARMBLY_TRANSPORT_LEGACY_PAYLOAD"); encoded != "" { + for _, name := range []string{"AddWorkerEmail", "SendEmailResult", "JobEventSyncState"} { + models.EventAvro.Register("warmbly.events."+name, priorTransportBody{}) + } + type priorError struct { + EmailAccountID string `avro:"email_account_id"` + } + models.EventAvro.Register("warmbly.events.EmailErrorEvent", priorError{}) + payload, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + t.Fatal(err) + } + c, err := NewAvro(os.Getenv("SR_URL"), "", "") + if err != nil { + t.Fatal(err) + } + var out struct { + Type string `avro:"type"` + Body any `avro:"body"` + } + if err := c.Deserialize(t.Context(), os.Getenv("WARMBLY_TRANSPORT_TOPIC"), payload, &out); err != nil { + t.Fatal(err) + } + if os.Getenv("WARMBLY_TRANSPORT_CASE") == "error" { + body, ok := out.Body.(priorError) + if !ok || body.EmailAccountID != os.Getenv("WARMBLY_TRANSPORT_ID") { + t.Fatal("old consumer lost failure") + } + t.Log("LEGACY-CODEC-DECODED") + return + } + body, ok := out.Body.(priorTransportBody) + if !ok { + t.Fatalf("legacy body = %T", out.Body) + } + switch os.Getenv("WARMBLY_TRANSPORT_CASE") { + case "add": + if body.ID.String() != os.Getenv("WARMBLY_TRANSPORT_ID") { + t.Fatal("old worker lost mailbox") + } + case "send": + if body.TaskID.String() != os.Getenv("WARMBLY_TRANSPORT_ID") || !body.Success { + t.Fatal("old consumer lost outcome") + } + case "sync": + if body.EmailID.String() != os.Getenv("WARMBLY_TRANSPORT_ID") || body.State.LastSyncedAt == nil { + t.Fatal("old consumer lost checkpoint") + } + case "error": + if body.EmailAccountID != os.Getenv("WARMBLY_TRANSPORT_ID") { + t.Fatal("old consumer lost failure") + } + } + t.Log("LEGACY-CODEC-DECODED") + return + } + url := os.Getenv("SR_URL") + if url == "" { + t.Fatal("SR_URL must name an isolated local Schema Registry") + } + generation, task, mailbox := uuid.New(), uuid.New(), uuid.New() + at := time.Now().UTC().Truncate(time.Millisecond) + for _, tc := range []struct { + name string + value any + schema avro.Schema + target func() any + proof func(any) uuid.UUID + }{ + {"add", models.WorkerEvent{Type: models.WorkerEventTypeAddEmail, Body: &models.AddWorkerEmail{ID: mailbox, TransportGeneration: generation}}, models.WorkerEvent{}.Schema(), func() any { return &models.WorkerEvent{} }, func(v any) uuid.UUID { + return v.(*models.WorkerEvent).Body.(*models.AddWorkerEmail).TransportGeneration + }}, + {"send", models.JobEvent{Type: models.JobEventTypeEmailSent, Body: models.SendEmailResult{TaskID: task, Success: true, SentAt: at, TransportGeneration: generation}}, models.JobEvent{}.Schema(), func() any { return &models.JobEvent{} }, func(v any) uuid.UUID { return v.(*models.JobEvent).Body.(models.SendEmailResult).TransportGeneration }}, + {"sync", models.JobEvent{Type: models.JobEventTypeSyncState, Body: &models.JobEventSyncState{EmailID: mailbox, State: models.SyncState{LastSyncedAt: &at, TransportGeneration: generation}}}, models.JobEvent{}.Schema(), func() any { return &models.JobEvent{} }, func(v any) uuid.UUID { + return v.(*models.JobEvent).Body.(*models.JobEventSyncState).State.TransportGeneration + }}, + {"error", models.JobEvent{Type: models.JobEventTypeEmailServerError, Body: models.EmailErrorEvent{EmailAccountID: mailbox.String(), Operation: models.MailTransportSync, TransportGeneration: generation}}, models.JobEvent{}.Schema(), func() any { return &models.JobEvent{} }, func(v any) uuid.UUID { return v.(*models.JobEvent).Body.(models.EmailErrorEvent).TransportGeneration }}, + } { + t.Run(tc.name, func(t *testing.T) { + current, err := NewAvro(url, "", "") + if err != nil { + t.Fatal(err) + } + writer := current.(*AvroCodec) + previous := priorTransportSchema(t, tc.schema) + topic := fmt.Sprintf("transport-generation-%s-%s", tc.name, uuid.New()) + if _, err := writer.client.UpdateCompatibility(subjectFor(topic), schemaregistry.FullTransitive); err != nil { + t.Fatal(err) + } + oldPayload, err := encodeTransportSchema(t.Context(), writer, topic, previous, tc.value) + if err != nil { + t.Fatal("old native encoding", err) + } + // A cold consumer must fetch the actual registered writer schema by id. + cold, err := NewAvro(url, "", "") + if err != nil { + t.Fatal(err) + } + target := tc.target() + if err := cold.Deserialize(t.Context(), topic, oldPayload, target); err != nil { + t.Fatal("new reader/old writer", err) + } + if tc.proof(target) != uuid.Nil { + t.Fatal("old payload fabricated proof") + } + fresh, err := writer.Serialize(t.Context(), topic, tc.value) + if err != nil { + t.Fatal("FULL_TRANSITIVE registry rejected additive schema", err) + } + target = tc.target() + if err := cold.Deserialize(t.Context(), topic, fresh, target); err != nil { + t.Fatal(err) + } + if tc.proof(target) != generation { + t.Fatal("typed wire lost generation") + } + id := mailbox + if tc.name == "send" { + id = task + } + process := exec.Command(os.Args[0], "-test.run=^TestMailTransportCodecRegistryCompatibility$", "-test.v") + process.Env = append(os.Environ(), "WARMBLY_TRANSPORT_LEGACY_PAYLOAD="+base64.StdEncoding.EncodeToString(fresh), "WARMBLY_TRANSPORT_TOPIC="+topic, "WARMBLY_TRANSPORT_CASE="+tc.name, "WARMBLY_TRANSPORT_ID="+id.String()) + output, err := process.CombinedOutput() + if err != nil || !strings.Contains(string(output), "LEGACY-CODEC-DECODED") { + t.Fatalf("cold old reader/new native writer: %v\n%s", err, output) + } + // The registry checks both directions, including returning to an old writer. + older, err := NewAvro(url, "", "") + if err != nil { + t.Fatal(err) + } + if _, err := encodeTransportSchema(t.Context(), older.(*AvroCodec), topic, previous, tc.value); err != nil { + t.Fatal("old writer after upgrade rejected", err) + } + }) + } +} diff --git a/internal/infrastructure/db/mail_transport_upgrade_test.go b/internal/infrastructure/db/mail_transport_upgrade_test.go new file mode 100644 index 000000000..a7a79d858 --- /dev/null +++ b/internal/infrastructure/db/mail_transport_upgrade_test.go @@ -0,0 +1,131 @@ +package db + +import ( + "fmt" + "testing" + "time" + + "github.com/google/uuid" +) + +const mailTransportMigrationVersion = 288 + +func TestLiveMailTransportCLIStartup(t *testing.T) { + m, c, _ := conflictUpgradeDatabaseAt(t, 284) + runConflictMigrationCLI(t, c) + assertConflictMigrationVersion(t, m, conflictLatestMigration(t), false) + var typed bool + if err := c.QueryRow(t.Context(), `SELECT to_regclass('email_account_transport_recovery') IS NOT NULL AND EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND table_name='email_account_errors' AND column_name='operation')`).Scan(&typed); err != nil || !typed { + t.Fatal("actual cmd/migrate did not apply typed extension", err) + } +} + +func TestLiveMailTransportUpgradePreservesHistoryAndRollback(t *testing.T) { + for _, baseline := range []uint{284, 287} { + t.Run(fmt.Sprint(baseline), func(t *testing.T) { testMailTransportUpgrade(t, baseline) }) + } +} + +func testMailTransportUpgrade(t *testing.T, baseline uint) { + t.Helper() + m, c, _ := conflictUpgradeDatabaseAt(t, baseline) + user, org, mailbox, task := uuid.New(), uuid.New(), uuid.New(), uuid.New() + at := time.Now().Add(-24 * time.Hour).UTC().Truncate(time.Microsecond) + exec := func(sql string, args ...any) { t.Helper(); conflictUpgradeExec(t, c, sql, args...) } + exec(`INSERT INTO users(id,email,first_name,last_name) VALUES($1,$2,'Upgrade','')`, user, user.String()+"@test.local") + exec(`INSERT INTO organizations(id,name,owner_user_id) VALUES($1,'Upgrade',$2)`, org, user) + exec(`INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider,send_recovery_hold,send_recovery_reason,send_recovery_task_id) VALUES($1,$2,$3,$4,'Upgrade','','','smtp_imap',true,'unknown',$5)`, mailbox, user, org, mailbox.String()+"@test.local", task) + exec(`INSERT INTO tasks(id,email_account_id,task_type,status,message_id,scheduled_at,send_result_state) VALUES($1,$2,'warmup','completed','',NOW(),'sent')`, task, mailbox) + exec(`INSERT INTO warmup_statistics(email_account_id,date,emails_sent,emails_replied,target_volume) VALUES($1,'2001-01-01',17,4,25)`, mailbox) + mailboxBefore := conflictUpgradeSnapshot(t, c, `SELECT (to_jsonb(e)-'transport_generation')::text FROM email_accounts e LIMIT 1`) + taskBefore := conflictUpgradeSnapshot(t, c, `SELECT to_jsonb(t)::text FROM tasks t LIMIT 1`) + accountingBefore := conflictUpgradeSnapshot(t, c, `SELECT jsonb_agg(to_jsonb(s) ORDER BY email_account_id,date)::text FROM warmup_statistics s`) + legacyIDs := []uuid.UUID{uuid.New(), uuid.New(), uuid.New()} + for i, id := range legacyIDs { + var taskID *uuid.UUID + var resolved *time.Time + if i == 1 { + taskID = &task + } + if i == 2 { + resolved = &at + } + exec(`INSERT INTO email_account_errors(id,email_account_id,user_id,error_code,severity,resolve_method,title,message,task_id,created_at,resolved_at) VALUES($1,$2,$3,$4,'WARNING','RETRY','History','Report',$5,$6,$7)`, id, mailbox, user, []string{"SERVER_UNREACHABLE", "CONNECTION_LOST", "IMAP_UNKNOWN"}[i], taskID, at, resolved) + } + assertHistory := func(typed bool) { + t.Helper() + for i, id := range legacyIDs { + var created time.Time + var taskID *uuid.UUID + var resolved *time.Time + if err := c.QueryRow(t.Context(), `SELECT created_at,task_id,resolved_at FROM email_account_errors WHERE id=$1`, id).Scan(&created, &taskID, &resolved); err != nil { + t.Fatal(err) + } + if !created.Equal(at) || i == 1 && (taskID == nil || *taskID != task) || i != 1 && taskID != nil || (resolved != nil) != (i == 2) { + t.Fatal("migration changed historical clock, resolution or task context") + } + if typed { + var op, clock string + var receipt *time.Time + if err := c.QueryRow(t.Context(), `SELECT operation,observation_source,received_at FROM email_account_errors WHERE id=$1`, id).Scan(&op, &clock, &receipt); err != nil || op != "unknown" || clock != "unknown" || receipt != nil { + t.Fatalf("legacy proof guessed: %s %s %v %v", op, clock, receipt, err) + } + } + } + if mailboxBefore != conflictUpgradeSnapshot(t, c, `SELECT (to_jsonb(e)-'transport_generation')::text FROM email_accounts e LIMIT 1`) || taskBefore != conflictUpgradeSnapshot(t, c, `SELECT to_jsonb(t)::text FROM tasks t LIMIT 1`) || accountingBefore != conflictUpgradeSnapshot(t, c, `SELECT jsonb_agg(to_jsonb(s) ORDER BY email_account_id,date)::text FROM warmup_statistics s`) { + t.Fatal("migration altered holds, task evidence or accounting") + } + } + upgrade := func() { + t.Helper() + if err := m.Migrate(mailTransportMigrationVersion); err != nil { + t.Fatal(err) + } + assertConflictMigrationVersion(t, m, mailTransportMigrationVersion, false) + assertHistory(true) + } + upgrade() + if _, err := c.Exec(t.Context(), `INSERT INTO email_account_errors(email_account_id,user_id,error_code,severity,resolve_method,title,message) VALUES($1,$2,'SERVER_UNREACHABLE','WARNING','RETRY','Old writer','Report') ON CONFLICT(email_account_id,error_code) WHERE resolved_at IS NULL DO NOTHING`, mailbox, user); err == nil { + t.Fatal("old account-plus-code writer unexpectedly compatible with native-only operation index") + } + assertHistory(true) + t.Log("old account-plus-code writer refused; it must be quiesced before changing the index") + if err := m.Steps(-1); err != nil { + t.Fatal(err) + } + assertConflictMigrationVersion(t, m, 287, false) + assertHistory(false) + upgrade() + for _, op := range []string{"sync", "unknown"} { + t.Run("refuse_"+op, func(t *testing.T) { + id := uuid.New() + exec(`INSERT INTO email_account_errors(id,email_account_id,user_id,error_code,severity,resolve_method,title,message,operation,observation_source) VALUES($1,$2,$3,'SERVER_UNREACHABLE','WARNING','RETRY','History','Same-code history',$4,'unknown')`, id, mailbox, user, op) + if err := m.Steps(-1); err == nil { + t.Fatal("lossy real-driver downgrade accepted") + } + assertConflictMigrationVersion(t, m, 287, true) + assertHistory(true) + var preserved bool + if err := c.QueryRow(t.Context(), `SELECT resolved_at IS NULL AND message='Same-code history' AND operation=$2 AND observation_source='unknown' FROM email_account_errors WHERE id=$1`, id, op).Scan(&preserved); err != nil || !preserved { + t.Fatal("downgrade changed or discarded colliding history", err) + } + var clockTable, nativeIndex bool + if err := c.QueryRow(t.Context(), `SELECT to_regclass('email_account_transport_recovery') IS NOT NULL,to_regclass('idx_email_account_errors_one_unresolved_per_operation') IS NOT NULL`).Scan(&clockTable, &nativeIndex); err != nil || !clockTable || !nativeIndex { + t.Fatal("refusal partially removed typed schema", err) + } + // Only restore driver metadata after verifying the failed down left schema intact. + if err := m.Force(mailTransportMigrationVersion); err != nil { + t.Fatal(err) + } + assertConflictMigrationVersion(t, m, mailTransportMigrationVersion, false) + exec(`UPDATE email_account_errors SET resolved_at=$2,resolved_by='Independent fixture confirmation' WHERE id=$1`, id, at) + if err := m.Steps(-1); err != nil { + t.Fatal(err) + } + assertConflictMigrationVersion(t, m, 287, false) + assertHistory(false) + upgrade() + }) + } + t.Logf("real driver schema%d -> 288 -> 287 -> 288 preserves legacy evidence and holds/accounting; different/same-operation history refuses down atomically, verified schema metadata repair and safe down/reupgrade pass", baseline) +} diff --git a/internal/infrastructure/db/migrations/000288_mailbox_transport_recovery.down.sql b/internal/infrastructure/db/migrations/000288_mailbox_transport_recovery.down.sql new file mode 100644 index 000000000..3ad0803d1 --- /dev/null +++ b/internal/infrastructure/db/migrations/000288_mailbox_transport_recovery.down.sql @@ -0,0 +1,16 @@ +-- Old uniqueness cannot represent multiple unresolved native or historical reports. +-- Refuse rather than delete history, invent resolution, or erase task context. +DO $$ +BEGIN +CREATE UNIQUE INDEX idx_email_account_errors_one_unresolved_per_code + ON email_account_errors(email_account_id,error_code) + WHERE resolved_at IS NULL; +DROP INDEX idx_email_account_errors_one_unresolved_per_operation; +DROP TABLE email_account_transport_recovery; +ALTER TABLE email_accounts DROP COLUMN transport_generation; +ALTER TABLE email_account_errors + DROP COLUMN operation, + DROP COLUMN observation_source, + DROP COLUMN received_at; +END +$$; diff --git a/internal/infrastructure/db/migrations/000288_mailbox_transport_recovery.up.sql b/internal/infrastructure/db/migrations/000288_mailbox_transport_recovery.up.sql new file mode 100644 index 000000000..89def3db7 --- /dev/null +++ b/internal/infrastructure/db/migrations/000288_mailbox_transport_recovery.up.sql @@ -0,0 +1,23 @@ +BEGIN; +ALTER TABLE email_accounts ADD COLUMN transport_generation uuid NOT NULL DEFAULT gen_random_uuid(); +ALTER TABLE email_account_errors + ADD COLUMN operation text NOT NULL DEFAULT 'unknown' + CHECK (operation IN ('unknown','sync','send')), + ADD COLUMN observation_source text NOT NULL DEFAULT 'unknown' + CHECK (observation_source IN ('unknown','worker','receipt')), + ADD COLUMN received_at timestamptz; + +-- No inferred operation or historical clock backfill. +DROP INDEX idx_email_account_errors_one_unresolved_per_code; +CREATE UNIQUE INDEX idx_email_account_errors_one_unresolved_per_operation + ON email_account_errors(email_account_id,error_code,operation) + WHERE resolved_at IS NULL AND observation_source IN ('worker','receipt'); + +CREATE TABLE email_account_transport_recovery ( + email_account_id uuid NOT NULL REFERENCES email_accounts(id) ON DELETE CASCADE, + operation text NOT NULL CHECK (operation IN ('sync','send')), + recovered_at timestamptz NOT NULL, + source_task_id uuid REFERENCES tasks(id) ON DELETE SET NULL, + PRIMARY KEY(email_account_id,operation) +); +COMMIT; diff --git a/internal/models/email.go b/internal/models/email.go index 2dba3819e..6a797e71b 100644 --- a/internal/models/email.go +++ b/internal/models/email.go @@ -38,11 +38,12 @@ const ( ) type Email struct { - ID uuid.UUID `json:"id"` - UserID string `json:"user_id"` - OrganizationID *uuid.UUID `json:"organization_id,omitempty"` - WorkerID *uuid.UUID `json:"worker_id"` - Email string `json:"email"` + TransportGeneration uuid.UUID `json:"-"` + ID uuid.UUID `json:"id"` + UserID string `json:"user_id"` + OrganizationID *uuid.UUID `json:"organization_id,omitempty"` + WorkerID *uuid.UUID `json:"worker_id"` + Email string `json:"email"` Name string `json:"name"` SignaturePlain string `json:"signature_plain"` diff --git a/internal/models/event.go b/internal/models/event.go index a3e361a10..8768dd174 100644 --- a/internal/models/event.go +++ b/internal/models/event.go @@ -1,5 +1,7 @@ package models +import "github.com/google/uuid" + type WorkerEventType string const ( @@ -78,16 +80,18 @@ type JobEvent struct { // EmailErrorEvent represents an email error event sent from worker to jobsService type EmailErrorEvent struct { - TaskID string `json:"task_id" avro:"task_id"` - EmailAccountID string `json:"email_account_id" avro:"email_account_id"` - UserID string `json:"user_id" avro:"user_id"` - ErrorCode string `json:"error_code" avro:"error_code"` - ErrorType string `json:"error_type" avro:"error_type"` - ResolveMethod string `json:"resolve_method" avro:"resolve_method"` - Message string `json:"message" avro:"message"` - UserVisible bool `json:"user_visible" avro:"user_visible"` - UserTitle string `json:"user_title,omitempty" avro:"user_title"` - UserMessage string `json:"user_message,omitempty" avro:"user_message"` - ActionRequired string `json:"action_required,omitempty" avro:"action_required"` - Timestamp int64 `json:"timestamp" avro:"timestamp"` + TransportGeneration uuid.UUID `json:"transport_generation,omitempty" avro:"transport_generation"` + Operation MailTransportOperation `json:"operation,omitempty" avro:"operation"` + TaskID string `json:"task_id" avro:"task_id"` + EmailAccountID string `json:"email_account_id" avro:"email_account_id"` + UserID string `json:"user_id" avro:"user_id"` + ErrorCode string `json:"error_code" avro:"error_code"` + ErrorType string `json:"error_type" avro:"error_type"` + ResolveMethod string `json:"resolve_method" avro:"resolve_method"` + Message string `json:"message" avro:"message"` + UserVisible bool `json:"user_visible" avro:"user_visible"` + UserTitle string `json:"user_title,omitempty" avro:"user_title"` + UserMessage string `json:"user_message,omitempty" avro:"user_message"` + ActionRequired string `json:"action_required,omitempty" avro:"action_required"` + Timestamp int64 `json:"timestamp" avro:"timestamp"` } diff --git a/internal/models/mail_transport.go b/internal/models/mail_transport.go new file mode 100644 index 000000000..6174229b4 --- /dev/null +++ b/internal/models/mail_transport.go @@ -0,0 +1,35 @@ +package models + +import "time" + +type MailTransportOperation string + +type MailTransportObservationSource string + +const ( + MailTransportClockUnknown MailTransportObservationSource = "unknown" + MailTransportClockWorker MailTransportObservationSource = "worker" + MailTransportClockReceipt MailTransportObservationSource = "receipt" +) + +const ( + MailTransportUnknown MailTransportOperation = "unknown" + MailTransportSync MailTransportOperation = "sync" + MailTransportSend MailTransportOperation = "send" +) + +func (o MailTransportOperation) Known() bool { + return o == MailTransportSync || o == MailTransportSend +} + +func (o MailTransportOperation) Conservative() MailTransportOperation { + if o.Known() { + return o + } + return MailTransportUnknown +} + +// Recovery needs an actual operation clock, not a synthesized receipt clock. +func ValidMailTransportTime(at, received time.Time) bool { + return !at.IsZero() && at.Unix() > 0 && !at.After(received) +} diff --git a/internal/models/mail_transport_test.go b/internal/models/mail_transport_test.go new file mode 100644 index 000000000..d6f833f3b --- /dev/null +++ b/internal/models/mail_transport_test.go @@ -0,0 +1,92 @@ +package models + +import ( + "encoding/json" + "testing" + "time" + + "github.com/iskorotkov/avro/v2" +) + +func TestMailTransportLegacyJSONAndAvroStayUnknown(t *testing.T) { + var legacy EmailErrorEvent + if err := json.Unmarshal([]byte(`{"email_account_id":"mailbox","task_id":"task"}`), &legacy); err != nil { + t.Fatal(err) + } + if legacy.Operation.Conservative() != MailTransportUnknown { + t.Fatal("legacy operation was inferred from task presence") + } + reader := JobEvent{}.Schema() + doc, err := SchemaDocument(reader) + if err != nil { + t.Fatal(err) + } + var tree any + if err := json.Unmarshal(doc, &tree); err != nil { + t.Fatal(err) + } + var remove func(any) + remove = func(node any) { + switch v := node.(type) { + case []any: + for _, item := range v { + remove(item) + } + case map[string]any: + if fields, ok := v["fields"].([]any); ok { + var keep []any + for _, field := range fields { + f := field.(map[string]any) + if f["name"] != "operation" { + keep = append(keep, field) + remove(f["type"]) + } + } + v["fields"] = keep + } else { + remove(v["type"]) + } + } + } + remove(tree) + legacyDoc, err := json.Marshal(tree) + if err != nil { + t.Fatal(err) + } + writer, err := avro.Parse(string(legacyDoc)) + if err != nil { + t.Fatal(err) + } + resolved, err := avro.NewSchemaCompatibility().Resolve(reader, writer) + if err != nil { + t.Fatal(err) + } + var got JobEvent + if err := EventAvro.Unmarshal(resolved, encode(t, writer, JobEvent{Type: JobEventTypeEmailServerError, Body: legacy}), &got); err != nil { + t.Fatal(err) + } + assertBody(t, got.Body, legacy) + if _, err := avro.NewSchemaCompatibility().Resolve(writer, reader); err != nil { + t.Fatalf("old reader cannot ignore operation: %v", err) + } + typed := EmailErrorEvent{Operation: MailTransportSend, TaskID: "task", EmailAccountID: "mailbox"} + if err := EventAvro.Unmarshal(reader, encode(t, reader, JobEvent{Type: JobEventTypeEmailServerError, Body: typed}), &got); err != nil { + t.Fatal(err) + } + assertBody(t, got.Body, typed) +} + +func TestMailTransportRecoveryNeedsValidOperationClock(t *testing.T) { + now := time.Now() + for _, at := range []time.Time{{}, time.Unix(-1, 0), now.Add(time.Hour)} { + if ValidMailTransportTime(at, now) { + t.Fatalf("invalid clock accepted: %v", at) + } + } + if !ValidMailTransportTime(now.Add(-time.Hour), now) { + t.Fatal("real past observation rejected") + } + if MailTransportOperation("new_operation").Conservative() != MailTransportUnknown { + t.Fatal("future operation guessed") + } +} diff --git a/internal/models/sync.go b/internal/models/sync.go index 41c54e356..0a9f27440 100644 --- a/internal/models/sync.go +++ b/internal/models/sync.go @@ -91,8 +91,9 @@ type GoogleHistoryRecovery struct { // owns the live copy and relays every change as SYNC_STATE; the consumer // persists it and the loader hands it back on the next (re)assignment. type SyncState struct { - BackfillStatus SyncBackfillStatus `json:"backfill_status" avro:"backfill_status"` - BackfillCursor SyncCursor `json:"backfill_cursor" avro:"backfill_cursor"` + TransportGeneration uuid.UUID `json:"transport_generation,omitempty" avro:"transport_generation"` + BackfillStatus SyncBackfillStatus `json:"backfill_status" avro:"backfill_status"` + BackfillCursor SyncCursor `json:"backfill_cursor" avro:"backfill_cursor"` // BackfillSynced counts messages the import has stored so far. BackfillSynced int `json:"backfill_synced" avro:"backfill_synced"` // BackfillSince is the cutoff the running import uses; fixed at start so diff --git a/internal/models/worker.go b/internal/models/worker.go index d4057f432..866ab64c7 100644 --- a/internal/models/worker.go +++ b/internal/models/worker.go @@ -120,10 +120,11 @@ type EmailSendError struct { // SendEmailResult is the result from worker after sending email type SendEmailResult struct { - TaskID uuid.UUID `json:"task_id" avro:"task_id"` - Success bool `json:"success" avro:"success"` - MessageID string `json:"message_id,omitempty" avro:"message_id"` - ProviderMsgID string `json:"provider_msg_id,omitempty" avro:"provider_msg_id"` + TransportGeneration uuid.UUID `json:"transport_generation,omitempty" avro:"transport_generation"` + TaskID uuid.UUID `json:"task_id" avro:"task_id"` + Success bool `json:"success" avro:"success"` + MessageID string `json:"message_id,omitempty" avro:"message_id"` + ProviderMsgID string `json:"provider_msg_id,omitempty" avro:"provider_msg_id"` // ThreadID is the provider-side conversation the message landed in, for // the providers that have one (Gmail). The control plane records it // against the task so the next step of the sequence can be appended to @@ -157,8 +158,9 @@ type AddWorkerEmailGraphData struct { } type AddWorkerEmail struct { - ID uuid.UUID `json:"id" avro:"id"` - UserID uuid.UUID `json:"user_id" avro:"user_id"` + TransportGeneration uuid.UUID `json:"transport_generation,omitempty" avro:"transport_generation"` + ID uuid.UUID `json:"id" avro:"id"` + UserID uuid.UUID `json:"user_id" avro:"user_id"` // OrganizationID scopes the organization-wide sync budget. Nil for a // legacy personal mailbox, which then only has per-mailbox budgets. OrganizationID *uuid.UUID `json:"organization_id" avro:"organization_id"` diff --git a/internal/repository/email_error_dedupe_live_test.go b/internal/repository/email_error_dedupe_live_test.go index 217e00e7b..c0e4354c1 100644 --- a/internal/repository/email_error_dedupe_live_test.go +++ b/internal/repository/email_error_dedupe_live_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" ) // Issue #405: a self-hosted instance whose IMAP server refused the same @@ -45,6 +46,7 @@ func TestLiveEmailErrorDedupeKeepsOneUnresolvedRowPerCode(t *testing.T) { arg any }{ {`DELETE FROM email_account_errors WHERE email_account_id = $1`, account}, + {`DELETE FROM tasks WHERE email_account_id = $1`, account}, {`DELETE FROM email_accounts WHERE organization_id = $1`, org}, {`DELETE FROM organizations WHERE id = $1`, org}, {`DELETE FROM users WHERE id = $1`, user}, @@ -55,15 +57,20 @@ func TestLiveEmailErrorDedupeKeepsOneUnresolvedRowPerCode(t *testing.T) { } }) + var generation uuid.UUID + if err := pool.QueryRow(ctx, `SELECT transport_generation FROM email_accounts WHERE id=$1`, account).Scan(&generation); err != nil { + t.Fatal(err) + } row := func(code string) *CreateEmailAccountError { return &CreateEmailAccountError{ - EmailAccountID: account, - UserID: user, - ErrorCode: code, - Severity: "WARNING", - ResolveMethod: "RETRY", - Title: "Email Error", - Message: "Something went wrong: NO System Error", + TransportGeneration: generation, + EmailAccountID: account, + UserID: user, + ErrorCode: code, + Severity: "WARNING", + ResolveMethod: "RETRY", + Title: "Email Error", + Message: "Something went wrong: NO System Error", } } @@ -106,7 +113,7 @@ func TestLiveEmailErrorDedupeKeepsOneUnresolvedRowPerCode(t *testing.T) { if xerr != nil || len(rows) != 1 || rows[0].CreatedAt.Before(before) { t.Fatalf("ongoing failure observation expired: %+v %v", rows, xerr) } - if xerr := repo.ResolveByCodesBefore(ctx, account, []string{"IMAP_UNKNOWN"}, before, "stale success"); xerr != nil { + if xerr := repo.RecordTransportSuccess(ctx, account, models.MailTransportSync, time.Now().Add(-3*time.Hour), nil, generation); xerr != nil { t.Fatal(xerr) } if n := unresolved("IMAP_UNKNOWN"); n != 1 { @@ -121,6 +128,53 @@ func TestLiveEmailErrorDedupeKeepsOneUnresolvedRowPerCode(t *testing.T) { if other == nil { t.Fatal("a different error code must still be recorded") } + workerTime := time.Now().Add(-2 * time.Hour).UTC().Truncate(time.Second) + late := row("RESOURCE_NOT_FOUND") + late.Operation = models.MailTransportSync + late.ObservedAt = &workerTime + firstLate, xerr := repo.CreateOnce(ctx, late) + if xerr != nil || firstLate == nil || !firstLate.CreatedAt.Equal(workerTime) { + t.Fatalf("delayed event looked fresh instead of retaining worker time: %v %v", firstLate, xerr) + } + newerTime := time.Now().Add(-5 * time.Minute).UTC().Truncate(time.Second) + late.ObservedAt = &newerTime + if _, xerr := repo.CreateOnce(ctx, late); xerr != nil { + t.Fatal(xerr) + } + late.ObservedAt = &workerTime + if _, xerr := repo.CreateOnce(ctx, late); xerr != nil { + t.Fatal(xerr) + } + var stored time.Time + if err := pool.QueryRow(ctx, `SELECT created_at FROM email_account_errors WHERE id=$1`, firstLate.ID).Scan(&stored); err != nil || !stored.Equal(newerTime) { + t.Fatalf("redelivery moved the latest worker observation backwards: %v %v", stored, err) + } + + taskID := uuid.New() + exec(`INSERT INTO tasks(id,task_type,email_account_id,status,message_id,scheduled_at) + VALUES($1,'warmup',$2,'pending','',NOW())`, taskID, account) + send := row("CONNECTION_LOST") + send.Operation = models.MailTransportSend + send.TaskID = &taskID + if _, xerr := repo.CreateOnce(ctx, send); xerr != nil { + t.Fatal(xerr) + } + if xerr := repo.RecordTransportSuccess(ctx, account, models.MailTransportSync, time.Now().Add(-time.Second), nil, generation); xerr != nil { + t.Fatal(xerr) + } + if unresolved("RESOURCE_NOT_FOUND") != 0 || unresolved("CONNECTION_LOST") != 1 { + t.Fatal("inbound sync cleared a task-scoped send failure or kept an older sync error") + } + send.ErrorCode = "SERVER_UNREACHABLE" + if _, xerr := repo.CreateOnce(ctx, send); xerr != nil { + t.Fatal(xerr) + } + if xerr := repo.RecordTransportSuccess(ctx, account, models.MailTransportSync, time.Now().Add(-time.Second), nil, generation); xerr != nil { + t.Fatal(xerr) + } + if unresolved("SERVER_UNREACHABLE") != 2 { + t.Fatal("deduping a send error into a mailbox report lost its send scope") + } // Once the mailbox recovers and the row is resolved, the next occurrence // is news again, otherwise a mailbox that breaks twice reports once. @@ -179,6 +233,8 @@ func TestLiveEmailErrorDedupeSurvivesConcurrentRelays(t *testing.T) { }) const racers = 8 + firstObserved := time.Now().Add(-2 * time.Hour).UTC().Truncate(time.Second) + sendTask := uuid.New() var ( wg sync.WaitGroup mu sync.Mutex @@ -190,6 +246,11 @@ func TestLiveEmailErrorDedupeSurvivesConcurrentRelays(t *testing.T) { go func() { defer wg.Done() <-start + observed := firstObserved.Add(time.Duration(i) * time.Minute) + var taskID *uuid.UUID + if i == racers-1 { + taskID = &sendTask + } row, xerr := repo.CreateOnce(ctx, &CreateEmailAccountError{ EmailAccountID: account, UserID: user, @@ -198,6 +259,8 @@ func TestLiveEmailErrorDedupeSurvivesConcurrentRelays(t *testing.T) { ResolveMethod: "RETRY", Title: "Email Error", Message: "Something went wrong: NO System Error", + ObservedAt: &observed, + TaskID: taskID, }) mu.Lock() defer mu.Unlock() @@ -225,4 +288,12 @@ func TestLiveEmailErrorDedupeSurvivesConcurrentRelays(t *testing.T) { if n != 1 { t.Errorf("unresolved rows = %d after %d concurrent relays, want 1", n, racers) } + var observed time.Time + var taskID *uuid.UUID + if err := pool.QueryRow(ctx, `SELECT created_at,task_id FROM email_account_errors WHERE email_account_id=$1 AND resolved_at IS NULL`, account).Scan(&observed, &taskID); err != nil { + t.Fatal(err) + } + if !observed.Equal(firstObserved.Add((racers-1)*time.Minute)) || taskID == nil || *taskID != sendTask { + t.Fatalf("concurrent initial reports lost the latest observation or send context: %v %v", observed, taskID) + } } diff --git a/internal/repository/mail_transport_history_live_test.go b/internal/repository/mail_transport_history_live_test.go new file mode 100644 index 000000000..caddc681d --- /dev/null +++ b/internal/repository/mail_transport_history_live_test.go @@ -0,0 +1,48 @@ +package repository + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" +) + +func TestLiveMailTransportNativeDomainExcludesHistory(t *testing.T) { + f := newTransportFixture(t) + at := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + for _, op := range []models.MailTransportOperation{models.MailTransportUnknown, models.MailTransportSync, models.MailTransportSend} { + ids := []uuid.UUID{uuid.New(), uuid.New()} + for _, id := range ids { + if _, err := f.pool.Exec(t.Context(), `INSERT INTO email_account_errors(id,email_account_id,user_id,error_code,severity,resolve_method,title,message,created_at,operation,observation_source,task_id) + VALUES($1,$2,$3,'SERVER_UNREACHABLE','WARNING','RETRY','History','Historical report',$4,$5,'unknown',$6)`, id, f.account, f.user, at, op, f.task); err != nil { + t.Fatal(err) + } + } + created, err := f.repo.CreateOnce(t.Context(), f.report(op, &at)) + if err != nil || created == nil { + t.Fatal("history occupied native dedupe", op, err) + } + if repeated, err := f.repo.CreateOnce(t.Context(), f.report(op, &at)); err != nil || repeated != nil { + t.Fatal("native dedupe changed", op, err) + } + var source *uuid.UUID + if op == models.MailTransportSend { + source = &f.task + } + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, op, at.Add(time.Minute), source, f.generation); err != nil { + t.Fatal(err) + } + var preserved int + if err := f.pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM email_account_errors WHERE id=ANY($1::uuid[]) AND created_at=$2 AND resolved_at IS NULL AND observation_source='unknown' AND task_id=$3`, ids, at, f.task).Scan(&preserved); err != nil || preserved != 2 { + t.Fatal("recovery altered historical report/task evidence", op, preserved, err) + } + var nativeResolved bool + if err := f.pool.QueryRow(t.Context(), `SELECT resolved_at IS NOT NULL FROM email_account_errors WHERE id=$1`, created.ID).Scan(&nativeResolved); err != nil || nativeResolved != op.Known() { + t.Fatal("native recovery changed", op, nativeResolved, err) + } + } + if reports := f.errors(t, false); len(reports) != 9 { + t.Fatal("historical IDs or native rows disappeared", len(reports)) + } +} diff --git a/internal/repository/mail_transport_import_concurrency_live_test.go b/internal/repository/mail_transport_import_concurrency_live_test.go new file mode 100644 index 000000000..75b71c4b2 --- /dev/null +++ b/internal/repository/mail_transport_import_concurrency_live_test.go @@ -0,0 +1,100 @@ +package repository + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/warmbly/warmbly/internal/models" +) + +func TestLiveMailTransportImportSerializesWithConfirmedResult(t *testing.T) { + f := newTransportFixture(t) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + if _, err := f.pool.Exec(ctx, `UPDATE tasks SET send_result_state=NULL WHERE id=$1`, f.task); err != nil { + t.Fatal(err) + } + recovery := NewTaskRepository(f.pool).(SendResultRecovery) + ready, proceed := make(chan struct{}, 1), make(chan struct{}) + resultDone, importDone := make(chan error, 1), make(chan error, 1) + at := time.Now().UTC().Add(-time.Hour) + go func() { + resultDone <- recovery.ApplySendResult(ctx, models.SendEmailResult{TaskID: f.task, Success: true, SentAt: at}, func(ctx context.Context) error { + ready <- struct{}{} + select { + case <-proceed: + case <-ctx.Done(): + return ctx.Err() + } + if err := f.repo.RecordTransportSuccess(ctx, f.account, models.MailTransportSend, at, &f.task, f.generation); err != nil { + return err + } + return nil + }) + }() + select { + case <-ready: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + go func() { + tx, err := f.pool.Begin(ctx) + if err != nil { + importDone <- err + return + } + defer tx.Rollback(context.Background()) + var row json.RawMessage + if err := f.pool.QueryRow(ctx, `SELECT to_jsonb(ea) || '{"name":"Restored"}'::jsonb FROM email_accounts ea WHERE id=$1`, f.account).Scan(&row); err != nil { + importDone <- err + return + } + written, err := NewOrgTransferRepository(f.db).InsertBatch(ctx, tx, "email_accounts", []string{"id", "user_id", "organization_id", "email", "name", "signature_plain", "signature_html", "provider"}, []json.RawMessage{row}, models.OrgImportConflictOverwrite, []string{"id"}, "id=$1", f.account, nil, + []ImportInvalidation{{Table: "email_account_transport_recovery", Column: "email_account_id", Key: "id", LockNamespaces: []int64{SendResultMailboxLockNamespace, MailTransportLockNamespace}}}) + if err == nil && written != 1 { + err = errors.New("wrong actual import count") + } + if err == nil { + err = tx.Commit(ctx) + } + importDone <- err + }() + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + var waiting bool + if err := f.pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event='advisory' AND query LIKE '%hashtextextended(keys.key%')`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + break + } + select { + case <-ticker.C: + case <-ctx.Done(): + t.Fatal("import did not wait for the native result's mailbox lock", ctx.Err()) + } + } + close(proceed) + for _, done := range []<-chan error{resultDone, importDone} { + select { + case err := <-done: + if err != nil { + t.Fatalf("result/import error: %T %#v (%v)", err, err, err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } + var confirmed bool + var clocks int + if err := f.pool.QueryRow(ctx, `SELECT send_result_state='sent' AND send_result_applied_at IS NOT NULL FROM tasks WHERE id=$1`, f.task).Scan(&confirmed); err != nil || !confirmed { + t.Fatal("import erased confirmed task audit", err) + } + if err := f.pool.QueryRow(ctx, `SELECT COUNT(*) FROM email_account_transport_recovery WHERE email_account_id=$1`, f.account).Scan(&clocks); err != nil || clocks != 0 { + t.Fatal("result/import retained prior derived clock", clocks, err) + } +} diff --git a/internal/repository/mail_transport_live_test.go b/internal/repository/mail_transport_live_test.go new file mode 100644 index 000000000..4e142e114 --- /dev/null +++ b/internal/repository/mail_transport_live_test.go @@ -0,0 +1,310 @@ +package repository + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" +) + +type transportFixture struct { + db *db.DB + pool *pgxpool.Pool + repo EmailAccountErrorRepository + account, user, task uuid.UUID + generation uuid.UUID +} + +func newTransportFixture(t *testing.T) transportFixture { + t.Helper() + d, pool := liveContactDB(t) + f := transportFixture{db: d, pool: pool, repo: NewEmailAccountErrorRepository(d), account: uuid.New(), user: uuid.New(), task: uuid.New()} + org := uuid.New() + for _, q := range []struct { + sql string + args []any + }{ + {`INSERT INTO users(id,first_name,last_name,email,password_hash) VALUES($1,'Transport','Test',$2,'x')`, []any{f.user, org.String() + "@test.local"}}, + {`INSERT INTO organizations(id,name,slug,owner_user_id) VALUES($1,'Transport',$2,$3)`, []any{org, org.String(), f.user}}, + {`INSERT INTO email_accounts(id,user_id,organization_id,email,name,signature_plain,signature_html,provider) VALUES($1,$2,$3,$4,'Transport','','','smtp_imap')`, []any{f.account, f.user, org, f.account.String() + "@test.local"}}, + {`INSERT INTO tasks(id,task_type,email_account_id,status,message_id,scheduled_at,send_result_state) VALUES($1,'warmup',$2,'completed','',NOW(),'sent')`, []any{f.task, f.account}}, + } { + if _, err := pool.Exec(t.Context(), q.sql, q.args...); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { + for _, q := range []struct { + sql string + id uuid.UUID + }{ + {`DELETE FROM email_accounts WHERE id=$1`, f.account}, + {`DELETE FROM organizations WHERE id=$1`, org}, + {`DELETE FROM users WHERE id=$1`, f.user}, + } { + if _, err := pool.Exec(context.Background(), q.sql, q.id); err != nil { + t.Error(err) + } + } + }) + if err := pool.QueryRow(t.Context(), `SELECT transport_generation FROM email_accounts WHERE id=$1`, f.account).Scan(&f.generation); err != nil { + t.Fatal(err) + } + return f +} + +func (f transportFixture) report(operation models.MailTransportOperation, at *time.Time) *CreateEmailAccountError { + d := &CreateEmailAccountError{TransportGeneration: f.generation, EmailAccountID: f.account, UserID: f.user, ErrorCode: "SERVER_UNREACHABLE", Severity: "WARNING", ResolveMethod: "RETRY", Title: "Transport", Message: "Transport report", Operation: operation, ObservedAt: at} + if operation == models.MailTransportSend { + d.TaskID = &f.task + } + return d +} + +func (f transportFixture) errors(t *testing.T, unresolved bool) []EmailAccountError { + t.Helper() + rows, err := f.repo.GetByAccountID(t.Context(), f.account, unresolved) + if err != nil { + t.Fatal(err) + } + return rows +} + +func TestLiveMailTransportOperationIsolationAndUnknownHold(t *testing.T) { + f := newTransportFixture(t) + old := time.Now().Add(-time.Hour).UTC().Truncate(time.Second) + for _, operation := range []models.MailTransportOperation{models.MailTransportUnknown, models.MailTransportSync, models.MailTransportSend} { + if row, err := f.repo.CreateOnce(t.Context(), f.report(operation, &old)); err != nil || row == nil { + t.Fatalf("%s: %v %v", operation, row, err) + } + } + if rows := f.errors(t, true); len(rows) != 3 { + t.Fatalf("operation classes merged: %+v", rows) + } + if _, err := f.pool.Exec(t.Context(), `UPDATE email_accounts SET send_recovery_hold=true,send_recovery_reason='unknown',send_recovery_task_id=$2 WHERE id=$1`, f.account, f.task); err != nil { + t.Fatal(err) + } + at := old.Add(time.Minute) + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSync, at, nil, f.generation); err != nil { + t.Fatal(err) + } + if rows := f.errors(t, true); len(rows) != 2 { + t.Fatalf("sync did not isolate scope: %+v", rows) + } else { + for _, row := range rows { + if row.Operation == models.MailTransportSync { + t.Fatal("sync remained unresolved") + } + } + } + // A completed task is not provider confirmation. + if _, err := f.pool.Exec(t.Context(), `UPDATE tasks SET send_result_state=NULL WHERE id=$1`, f.task); err != nil { + t.Fatal(err) + } + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSend, at, &f.task, f.generation); err != nil { + t.Fatal(err) + } + if len(f.errors(t, true)) != 2 { + t.Fatal("completion alone cleared a send error") + } + foreign := uuid.New() + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSend, at, &foreign, f.generation); err != nil { + t.Fatal(err) + } + if len(f.errors(t, true)) != 2 { + t.Fatal("foreign task cleared a send error") + } + if _, err := f.pool.Exec(t.Context(), `UPDATE tasks SET send_result_state='sent' WHERE id=$1`, f.task); err != nil { + t.Fatal(err) + } + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSend, at, &f.task, f.generation); err != nil { + t.Fatal(err) + } + rows := f.errors(t, true) + if len(rows) != 1 || rows[0].Operation != models.MailTransportUnknown { + t.Fatalf("send cleared unknown operation: %+v", rows) + } + var held bool + if err := f.pool.QueryRow(t.Context(), `SELECT send_recovery_hold FROM email_accounts WHERE id=$1`, f.account).Scan(&held); err != nil || !held { + t.Fatalf("report recovery cleared send hold: %v %v", held, err) + } + if len(f.errors(t, false)) != 3 { + t.Fatal("recovery deleted historical rows") + } + batch, xerr := f.repo.GetByAccountIDs(t.Context(), []uuid.UUID{f.account}, false) + if xerr != nil || len(batch[f.account]) != 3 { + t.Fatal("batched report model lost history", xerr) + } + byUser, xerr := f.repo.GetByUserID(t.Context(), f.user, 100) + if xerr != nil || len(byUser) != 3 { + t.Fatal("user report model lost history", xerr) + } +} + +func TestLiveMailTransportOutOfOrderFailureStaysHistorical(t *testing.T) { + f := newTransportFixture(t) + old := time.Now().Add(-time.Hour).UTC().Truncate(time.Second) + success := old.Add(time.Minute) + for _, operation := range []models.MailTransportOperation{models.MailTransportSync, models.MailTransportSend} { + var taskID *uuid.UUID + if operation == models.MailTransportSend { + taskID = &f.task + } + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, operation, success, taskID, f.generation); err != nil { + t.Fatal(err) + } + for range 2 { + if row, err := f.repo.CreateOnce(t.Context(), f.report(operation, &old)); err != nil || row != nil { + t.Fatalf("old failure notified/reopened outage: %v %v", row, err) + } + } + } + if len(f.errors(t, true)) != 0 || len(f.errors(t, false)) != 2 { + t.Fatal("historical failure was lost, duplicated or reopened") + } + newer := success.Add(time.Minute) + row, err := f.repo.CreateOnce(t.Context(), f.report(models.MailTransportSync, &newer)) + if err != nil || row == nil { + t.Fatal("fresh failure was swallowed", err) + } + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSync, old, nil, f.generation); err != nil { + t.Fatal(err) + } + if len(f.errors(t, true)) != 1 { + t.Fatal("old success cleared newer failure") + } + if row, err := f.repo.CreateOnce(t.Context(), f.report(models.MailTransportSync, &old)); err != nil || row != nil { + t.Fatal("old redelivery created a new warning", err) + } + if rows := f.errors(t, true); len(rows) != 1 || !rows[0].CreatedAt.Equal(newer) { + t.Fatalf("older error moved latest clock: %+v", rows) + } + // Worker failures have second precision; this ambiguous bucket stays held. + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSync, newer.Add(500*time.Millisecond), nil, f.generation); err != nil { + t.Fatal(err) + } + if len(f.errors(t, true)) != 1 { + t.Fatal("same-second success claimed temporal authority") + } + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSync, newer.Add(2*time.Second), nil, f.generation); err != nil { + t.Fatal(err) + } + if len(f.errors(t, true)) != 0 { + t.Fatal("newer proven success did not recover") + } + if row, err := f.repo.CreateOnce(t.Context(), f.report(models.MailTransportUnknown, &old)); err != nil || row == nil { + t.Fatal("known successes guessed legacy operation ownership", err) + } +} + +func TestLiveMailTransportClockFallbackAndConcurrentRecovery(t *testing.T) { + f := newTransportFixture(t) + base := time.Now().Add(-time.Hour).UTC().Truncate(time.Second) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range 12 { + wg.Add(1) + go func() { + defer wg.Done() + <-start + at := base.Add(time.Duration(i) * time.Minute) + if _, err := f.repo.CreateOnce(t.Context(), f.report(models.MailTransportSync, &at)); err != nil { + t.Error(err) + } + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSync, base.Add(20*time.Minute), nil, f.generation); err != nil { + t.Error(err) + } + }() + } + close(start) + wg.Wait() + if len(f.errors(t, true)) != 0 { + t.Fatal("concurrent error/recovery left a false outage") + } + var recovered time.Time + if err := f.pool.QueryRow(t.Context(), `SELECT recovered_at FROM email_account_transport_recovery WHERE email_account_id=$1 AND operation='sync'`, f.account).Scan(&recovered); err != nil || !recovered.Equal(base.Add(20*time.Minute)) { + t.Fatalf("success watermark regressed: %v %v", recovered, err) + } + zero := time.Time{} + future := time.Now().Add(time.Hour) + negative := time.Unix(-1, 0) + for _, invalid := range []*time.Time{nil, &zero, &future, &negative} { + before := time.Now() + if _, err := f.repo.CreateOnce(t.Context(), f.report(models.MailTransportSync, invalid)); err != nil { + t.Fatal(err) + } + rows := f.errors(t, true) + if len(rows) != 1 || rows[0].CreatedAt.Before(before) || rows[0].ObservationSource != "receipt" || rows[0].ReceivedAt == nil { + t.Fatalf("invalid worker time was not receipt-scoped: %+v", rows) + } + } + current := f.errors(t, true)[0] + if _, err := f.repo.CreateOnce(t.Context(), f.report(models.MailTransportSync, &base)); err != nil { + t.Fatal(err) + } + if row := f.errors(t, true)[0]; !row.CreatedAt.Equal(current.CreatedAt) || row.ObservationSource != "receipt" { + t.Fatalf("older delivery changed latest provenance: %+v", row) + } + for _, at := range []time.Time{zero, future, negative} { + if err := f.repo.RecordTransportSuccess(t.Context(), f.account, models.MailTransportSync, at, nil, f.generation); err != nil { + t.Fatal(err) + } + } + if len(f.errors(t, true)) != 1 { + t.Fatal("invalid success clock cleared receipt-scoped failure") + } +} + +func TestLiveMailTransportRecoveryJoinsResultTransaction(t *testing.T) { + f := newTransportFixture(t) + old := time.Now().Add(-time.Hour) + if _, err := f.repo.CreateOnce(t.Context(), f.report(models.MailTransportSend, &old)); err != nil { + t.Fatal(err) + } + tx, err := f.db.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(t.Context()) + ctx := context.WithValue(t.Context(), sendResultKey{}, sendResultContext{tx: tx, taskID: f.task}) + if err := f.repo.RecordTransportSuccess(ctx, f.account, models.MailTransportSend, old.Add(time.Minute), &f.task, f.generation); err != nil { + t.Fatal(err) + } + if err := tx.Rollback(t.Context()); err != nil { + t.Fatal(err) + } + if len(f.errors(t, true)) != 1 { + t.Fatal("recovery escaped failed send-result transaction") + } + var n int + if err := f.pool.QueryRow(t.Context(), `SELECT count(*) FROM email_account_transport_recovery WHERE email_account_id=$1`, f.account).Scan(&n); err != nil || n != 0 { + t.Fatal("recovery watermark survived rollback", err) + } +} + +func TestLiveMailTransportSyncTimestampIsMonotonic(t *testing.T) { + f := newTransportFixture(t) + repo := NewEmailSyncStateRepository(f.db) + newer := time.Now().Add(-time.Minute).UTC().Truncate(time.Microsecond) + older := newer.Add(-time.Hour) + future := time.Now().Add(time.Hour) + zero := time.Time{} + for _, at := range []*time.Time{&newer, &older, nil, &future, &zero} { + if err := repo.Put(t.Context(), f.user, f.account, &models.SyncState{LastSyncedAt: at}); err != nil { + t.Fatal(err) + } + } + state, err := repo.Get(t.Context(), f.account) + if err != nil || state.LastSyncedAt == nil || !state.LastSyncedAt.Equal(newer) { + t.Fatalf("older redelivery regressed monitoring clock: %+v %v", state, err) + } + var accountAt time.Time + if err := f.pool.QueryRow(t.Context(), `SELECT last_synced_at FROM email_accounts WHERE id=$1`, f.account).Scan(&accountAt); err != nil || !accountAt.Equal(newer) { + t.Fatalf("account/table clocks diverged: %v %v", accountAt, err) + } +} diff --git a/internal/repository/mail_transport_snapshot_live_test.go b/internal/repository/mail_transport_snapshot_live_test.go new file mode 100644 index 000000000..e1b512f75 --- /dev/null +++ b/internal/repository/mail_transport_snapshot_live_test.go @@ -0,0 +1,68 @@ +package repository + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/pkg/encrypt" +) + +func TestLiveMailTransportSnapshotKeepsConfigurationAndGenerationTogether(t *testing.T) { + f := newTransportFixture(t) + enc, err := encrypt.NewEncrypter(make([]byte, 32)) + if err != nil { + t.Fatal(err) + } + repo := NewEmailRepostory(f.db, enc) + if _, err := f.pool.Exec(t.Context(), `INSERT INTO email_accounts_oauth(email_account_id,access_token,refresh_token,expires_at) VALUES($1,'old-access','old-refresh',NOW()+INTERVAL '1 hour')`, f.account); err != nil { + t.Fatal(err) + } + next := uuid.New() + if err := repo.(EmailTransportSnapshot).WithTransportSnapshot(t.Context(), func(snapshot EmailRepository) error { + account, xerr := snapshot.GetByID(t.Context(), f.account) + if xerr != nil { + return xerr + } + tx, err := f.pool.Begin(t.Context()) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if _, err := tx.Exec(t.Context(), `UPDATE email_accounts SET transport_generation=$2,name='Fresh' WHERE id=$1`, f.account, next); err != nil { + return err + } + if _, err := tx.Exec(t.Context(), `UPDATE email_accounts_oauth SET access_token='fresh-access',refresh_token='fresh-refresh' WHERE email_account_id=$1`, f.account); err != nil { + return err + } + if err := tx.Commit(t.Context()); err != nil { + return err + } + credentials, xerr := snapshot.GetOAuthCredentials(t.Context(), f.account) + if xerr != nil { + return xerr + } + if account.TransportGeneration != f.generation || credentials.AccessToken != "old-access" || credentials.RefreshToken != "old-refresh" { + t.Fatal("configuration crossed a generation boundary") + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := repo.(EmailTransportSnapshot).WithTransportSnapshot(t.Context(), func(snapshot EmailRepository) error { + account, xerr := snapshot.GetByID(t.Context(), f.account) + if xerr != nil { + return xerr + } + credentials, xerr := snapshot.GetOAuthCredentials(t.Context(), f.account) + if xerr != nil { + return xerr + } + if account.TransportGeneration != next || account.Name != "Fresh" || credentials.AccessToken != "fresh-access" { + t.Fatal("fresh snapshot did not load fresh configuration") + } + return nil + }); err != nil { + t.Fatal(err) + } +} diff --git a/internal/repository/pg_email.go b/internal/repository/pg_email.go index 93515d32c..1841f1db9 100644 --- a/internal/repository/pg_email.go +++ b/internal/repository/pg_email.go @@ -223,8 +223,33 @@ type EmailRepository interface { } type emailRepository struct { - DB *db.DB - Encrypt *encrypt.Encrypter + DB *db.DB + Encrypt *encrypt.Encrypter + transportSnapshot pgx.Tx +} + +type EmailTransportSnapshot interface { + WithTransportSnapshot(context.Context, func(EmailRepository) error) error +} + +func (r *emailRepository) WithTransportSnapshot(ctx context.Context, read func(EmailRepository) error) error { + tx, err := r.DB.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return err + } + defer tx.Rollback(ctx) + snapshot := &emailRepository{DB: r.DB, Encrypt: r.Encrypt, transportSnapshot: tx} + if err := read(snapshot); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (r *emailRepository) transportQueryRow(ctx context.Context, query string, args ...any) pgx.Row { + if r.transportSnapshot != nil { + return r.transportSnapshot.QueryRow(ctx, query, args...) + } + return r.DB.QueryRow(ctx, query, args...) } // NewEmailRepostory builds the email repository. enc seals SMTP/IMAP and @@ -493,7 +518,7 @@ func (r *emailRepository) GetDelegation(ctx context.Context, accountID uuid.UUID WHERE id = $1 AND auth_method = 'delegated' AND domain_grant_id IS NOT NULL AND organization_id IS NOT NULL` var d models.DelegatedMailbox var provider string - err := r.DB.QueryRow(ctx, query, accountID).Scan(&d.AccountID, &d.OrganizationID, &provider, &d.Email, &d.GrantID, &d.Subject, &d.Status) + err := r.transportQueryRow(ctx, query, accountID).Scan(&d.AccountID, &d.OrganizationID, &provider, &d.Email, &d.GrantID, &d.Subject, &d.Status) if errors.Is(err, pgx.ErrNoRows) { return nil, nil } @@ -2253,6 +2278,7 @@ func (r *emailRepository) SetWarmupLifecycle(ctx context.Context, orgID, emailAc func (r *emailRepository) GetByID(ctx context.Context, emailAccountID uuid.UUID) (*models.Email, *errx.Error) { query := ` SELECT + ea.transport_generation, ea.id, ea.user_id, ea.organization_id, ea.worker_id, ea.email, ea.name, ea.signature_plain, ea.signature_html, ea.signature_sync, ea.signature_code, ea.send_as_email, ea.provider, ea.mail_host, ea.auth_method, ea.domain_grant_id, ea.vendor_connection_id, COALESCE((SELECT vc.vendor FROM mailbox_vendor_connections vc WHERE vc.id = ea.vendor_connection_id), ''), ea.avatar_url, ea.status, COALESCE(ea.last_synced_at, ea.created_at) AS last_synced_at, ea.last_id, ea.campaign_limit, ea.min_wait_time, ea.reply_to, ea.tracking_domain, ea.tracking_domain_verified, ea.tracking_domain_verified_at, ea.track_direct_mail, ea.warmup, ea.warmup_paused_at, ea.warmup_base, @@ -2268,7 +2294,8 @@ func (r *emailRepository) GetByID(ctx context.Context, emailAccountID uuid.UUID) ` var i models.Email - err := r.DB.QueryRow(ctx, query, emailAccountID).Scan( + err := r.transportQueryRow(ctx, query, emailAccountID).Scan( + &i.TransportGeneration, &i.ID, &i.UserID, &i.OrganizationID, &i.WorkerID, &i.Email, &i.Name, &i.SignaturePlain, &i.SignatureHTML, &i.SignatureSync, &i.SignatureCode, &i.SendAsEmail, &i.Provider, &i.MailHost, &i.AuthMethod, &i.DomainGrantID, &i.VendorConnectionID, &i.Vendor, &i.AvatarURL, &i.Status, &i.LastSyncedAt, &i.LastID, &i.CampaignLimit, &i.MinWaitTime, &i.ReplyTo, &i.TrackingDomain, &i.TrackingDomainVerified, &i.TrackingDomainVerifiedAt, &i.TrackDirectMail, &i.Warmup, &i.WarmupPausedAt, &i.WarmupBase, @@ -2589,7 +2616,7 @@ func (r *emailRepository) GetSMTPCredentials(ctx context.Context, emailAccountID var creds SMTPCredentials var smtpHost, smtpUser, smtpPassword, imapHost, imapUser, imapPassword string - err := r.DB.QueryRow(ctx, query, emailAccountID).Scan( + err := r.transportQueryRow(ctx, query, emailAccountID).Scan( &smtpHost, &creds.SMTPPort, &smtpUser, &smtpPassword, &creds.SMTPSecurity, &imapHost, &creds.IMAPPort, &imapUser, &imapPassword, &creds.IMAPSecurity, ) @@ -2652,7 +2679,7 @@ func (r *emailRepository) GetOAuthCredentials(ctx context.Context, emailAccountI var accessToken, refreshToken string var expiresAt time.Time - err := r.DB.QueryRow(ctx, query, emailAccountID).Scan( + err := r.transportQueryRow(ctx, query, emailAccountID).Scan( &accessToken, &refreshToken, &expiresAt, ) if err != nil { @@ -2674,7 +2701,7 @@ func (r *emailRepository) GetOAuthCredentials(ctx context.Context, emailAccountI errs.CaptureException(xerr) return nil, errx.InternalError() } - if accessLegacy || refreshLegacy { + if (accessLegacy || refreshLegacy) && r.transportSnapshot == nil { r.resealOAuthCredentials(ctx, emailAccountID, decryptedAccessToken, decryptedRefreshToken) } diff --git a/internal/repository/pg_email_error.go b/internal/repository/pg_email_error.go index c9a4ca2c5..5c5b7ef82 100644 --- a/internal/repository/pg_email_error.go +++ b/internal/repository/pg_email_error.go @@ -9,47 +9,50 @@ import ( "github.com/jackc/pgx/v5" "github.com/warmbly/warmbly/internal/errx" "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" ) // EmailAccountError represents an error associated with an email account type EmailAccountError struct { - ID uuid.UUID `json:"id"` - EmailAccountID uuid.UUID `json:"email_account_id"` - UserID uuid.UUID `json:"user_id"` - ErrorCode string `json:"error_code"` - Severity string `json:"severity"` - ResolveMethod string `json:"resolve_method"` - Title string `json:"title"` - Message string `json:"message"` - UserMessage *string `json:"user_message,omitempty"` - ActionRequired *string `json:"action_required,omitempty"` - TaskID *uuid.UUID `json:"task_id,omitempty"` - ResolvedAt *time.Time `json:"resolved_at,omitempty"` - ResolvedBy *string `json:"resolved_by,omitempty"` - CreatedAt time.Time `json:"created_at"` + ID uuid.UUID `json:"id"` + EmailAccountID uuid.UUID `json:"email_account_id"` + UserID uuid.UUID `json:"user_id"` + ErrorCode string `json:"error_code"` + Severity string `json:"severity"` + ResolveMethod string `json:"resolve_method"` + Title string `json:"title"` + Message string `json:"message"` + UserMessage *string `json:"user_message,omitempty"` + ActionRequired *string `json:"action_required,omitempty"` + TaskID *uuid.UUID `json:"task_id,omitempty"` + ResolvedAt *time.Time `json:"resolved_at,omitempty"` + ResolvedBy *string `json:"resolved_by,omitempty"` + CreatedAt time.Time `json:"created_at"` + Operation models.MailTransportOperation `json:"operation"` + ObservationSource models.MailTransportObservationSource `json:"observation_source"` + ReceivedAt *time.Time `json:"received_at,omitempty"` } // CreateEmailAccountError contains data for creating a new error record type CreateEmailAccountError struct { - EmailAccountID uuid.UUID - UserID uuid.UUID - ErrorCode string - Severity string - ResolveMethod string - Title string - Message string - UserMessage *string - ActionRequired *string - TaskID *uuid.UUID + TransportGeneration uuid.UUID + EmailAccountID uuid.UUID + UserID uuid.UUID + ErrorCode string + Severity string + ResolveMethod string + Title string + Message string + UserMessage *string + ActionRequired *string + TaskID *uuid.UUID + ObservedAt *time.Time + Operation models.MailTransportOperation } // EmailAccountErrorRepository defines operations for email account errors type EmailAccountErrorRepository interface { - // CreateOnce records an error unless the account already has an unresolved - // one with the same code, in which case it writes nothing and returns a - // nil record. It is the only write path: migration 000145 makes a second - // unresolved row for one code impossible, so an unconditional insert would - // only turn a repeat into a constraint violation. + // CreateOnce deduplicates unresolved reports by mailbox, code and operation. CreateOnce(ctx context.Context, err *CreateEmailAccountError) (*EmailAccountError, *errx.Error) GetByAccountID(ctx context.Context, accountID uuid.UUID, unresolvedOnly bool) ([]EmailAccountError, *errx.Error) // GetByAccountIDs is the batched form, keyed by account id, so a status list @@ -62,10 +65,7 @@ type EmailAccountErrorRepository interface { // ResolveByCodes resolves the account's unresolved errors carrying any of // the given codes, for a flow that just fixed that class of error. ResolveByCodes(ctx context.Context, accountID uuid.UUID, codes []string, resolvedBy string) *errx.Error - // ResolveByCodesBefore is ResolveByCodes bounded to errors raised before - // a given moment, so evidence of recovery cannot resolve a failure that - // happened after it. - ResolveByCodesBefore(ctx context.Context, accountID uuid.UUID, codes []string, before time.Time, resolvedBy string) *errx.Error + RecordTransportSuccess(ctx context.Context, accountID uuid.UUID, operation models.MailTransportOperation, at time.Time, taskID *uuid.UUID, generation uuid.UUID) *errx.Error ResolveAllForAccount(ctx context.Context, accountID uuid.UUID, resolvedBy string) *errx.Error } @@ -80,46 +80,76 @@ func NewEmailAccountErrorRepository(database *db.DB) EmailAccountErrorRepository } } -// Create stores a new email account error -// CreateOnce refreshes unresolved transport observations without creating or notifying twice. -// -// A mail server that refuses the same command every pass is not a new problem -// every pass. The sync loop retries about once a minute and reports what it -// got, so one IMAP_UNKNOWN that nobody can fix wrote 1440 identical rows a day -// into the mailbox's error list (issue #405). -// -// Both guards are load-bearing. WHERE NOT EXISTS answers the ordinary repeat -// without touching the index, and ON CONFLICT answers the race it cannot see: -// two workers relaying the same failure in the same instant both find no row. -// The unique index behind it is partial on the unresolved rows (migration -// 000145), so resolved history is untouched and a problem that returns after -// it was fixed is recorded again. +// CreateOnce serializes observations with operation recovery and deduplicates warnings. func (r *emailAccountErrorRepository) CreateOnce(ctx context.Context, data *CreateEmailAccountError) (*EmailAccountError, *errx.Error) { - // Unresolved transport timestamps represent the latest failure observation. + tx, err := beginResultTx(ctx, r.DB) + if err != nil { + return nil, errx.InternalError() + } + defer tx.Rollback(ctx) + if err := lockTransportMailbox(ctx, tx, data.EmailAccountID); err != nil { + return nil, errx.InternalError() + } + operation := data.Operation.Conservative() + if operation.Known() { + var matches bool + if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM email_accounts WHERE id=$1 AND transport_generation=$2)`, data.EmailAccountID, data.TransportGeneration).Scan(&matches); err != nil { + return nil, errx.InternalError() + } + if !matches { + operation = models.MailTransportUnknown + } + } + if operation == models.MailTransportSync && data.TaskID != nil || operation == models.MailTransportSend && (data.TaskID == nil || *data.TaskID == uuid.Nil) { + operation = models.MailTransportUnknown + } + recordID := uuid.New() query := ` - WITH observed AS ( - UPDATE email_account_errors SET created_at=NOW() - WHERE email_account_id=$1 AND user_id=$2 AND error_code=$3 AND resolved_at IS NULL - AND error_code IN ('SERVER_UNREACHABLE','CONNECTION_LOST','RESOURCE_NOT_FOUND','IMAP_UNKNOWN') - RETURNING id + WITH observation AS ( + SELECT CASE WHEN $11::timestamptz > to_timestamp(0) AND $11 <= NOW() + THEN $11 ELSE NOW() END AS at, + CASE WHEN $11::timestamptz > to_timestamp(0) AND $11 <= NOW() + THEN 'worker' ELSE 'receipt' END AS source + ), candidate AS ( + SELECT observation.*, recovery.recovered_at + FROM observation LEFT JOIN email_account_transport_recovery recovery + ON recovery.email_account_id=$1 AND recovery.operation=$13 + AND $3 IN ('SERVER_UNREACHABLE','CONNECTION_LOST','RESOURCE_NOT_FOUND','IMAP_UNKNOWN') + AND observation.at < recovery.recovered_at - INTERVAL '1 second' ) - INSERT INTO email_account_errors ( - email_account_id, user_id, error_code, severity, resolve_method, - title, message, user_message, action_required, task_id + INSERT INTO email_account_errors AS existing ( + id, email_account_id, user_id, error_code, severity, resolve_method, + title, message, user_message, action_required, task_id, created_at, + operation,observation_source,received_at,resolved_at,resolved_by ) - -- The casts are load-bearing: a SELECT source, unlike VALUES, gives - -- the planner no target columns to infer the parameter types from. - SELECT $1::uuid, $2::uuid, $3::varchar, $4::email_error_severity, - $5::email_error_resolve_method, $6::varchar, $7::text, $8::text, - $9::text, $10::uuid - WHERE NOT EXISTS ( - SELECT 1 FROM email_account_errors - WHERE email_account_id = $1 AND error_code = $3 AND resolved_at IS NULL - ) - ON CONFLICT DO NOTHING + SELECT $12,$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,at,$13,source,NOW(),recovered_at, + CASE WHEN recovered_at IS NOT NULL THEN $13 || ' recovered' END + FROM candidate + WHERE EXISTS(SELECT 1 FROM email_accounts WHERE id=$1 AND user_id=$2) + AND NOT (recovered_at IS NOT NULL AND EXISTS( + SELECT 1 FROM email_account_errors historical + WHERE historical.email_account_id=$1 AND historical.user_id=$2 + AND historical.error_code=$3 AND historical.operation=$13 + AND historical.created_at=candidate.at + AND historical.task_id IS NOT DISTINCT FROM $10::uuid + AND historical.resolved_by=$13 || ' recovered')) + ON CONFLICT (email_account_id,error_code,operation) + WHERE resolved_at IS NULL AND observation_source IN ('worker','receipt') DO UPDATE + SET created_at=CASE WHEN existing.user_id=EXCLUDED.user_id + AND existing.error_code IN ('SERVER_UNREACHABLE','CONNECTION_LOST','RESOURCE_NOT_FOUND','IMAP_UNKNOWN') + THEN GREATEST(existing.created_at,EXCLUDED.created_at) ELSE existing.created_at END, + task_id=CASE WHEN existing.user_id=EXCLUDED.user_id + AND existing.error_code IN ('SERVER_UNREACHABLE','CONNECTION_LOST','RESOURCE_NOT_FOUND','IMAP_UNKNOWN') + THEN COALESCE(existing.task_id,EXCLUDED.task_id) ELSE existing.task_id END, + observation_source=CASE WHEN existing.user_id=EXCLUDED.user_id + AND EXCLUDED.created_at >= existing.created_at + AND existing.error_code IN ('SERVER_UNREACHABLE','CONNECTION_LOST','RESOURCE_NOT_FOUND','IMAP_UNKNOWN') + THEN EXCLUDED.observation_source ELSE existing.observation_source END, + received_at=CASE WHEN existing.user_id=EXCLUDED.user_id + THEN GREATEST(existing.received_at,EXCLUDED.received_at) ELSE existing.received_at END RETURNING id, email_account_id, user_id, error_code, severity, resolve_method, title, message, user_message, action_required, task_id, - resolved_at, resolved_by, created_at + resolved_at, resolved_by, created_at,operation,observation_source,received_at ` params := []any{ @@ -133,13 +163,16 @@ func (r *emailAccountErrorRepository) CreateOnce(ctx context.Context, data *Crea data.UserMessage, data.ActionRequired, data.TaskID, + data.ObservedAt, + recordID, + operation, } var e EmailAccountError - err := r.DB.QueryRow(ctx, query, params...).Scan( + err = tx.QueryRow(ctx, query, params...).Scan( &e.ID, &e.EmailAccountID, &e.UserID, &e.ErrorCode, &e.Severity, &e.ResolveMethod, &e.Title, &e.Message, &e.UserMessage, &e.ActionRequired, &e.TaskID, - &e.ResolvedAt, &e.ResolvedBy, &e.CreatedAt, + &e.ResolvedAt, &e.ResolvedBy, &e.CreatedAt, &e.Operation, &e.ObservationSource, &e.ReceivedAt, ) if errors.Is(err, pgx.ErrNoRows) { // The row the caller wanted is already on screen, unresolved. @@ -158,6 +191,12 @@ func (r *emailAccountErrorRepository) CreateOnce(ctx context.Context, data *Crea db.CaptureError(err, query, params, "queryrow") return nil, errx.InternalError() } + if err := tx.Commit(ctx); err != nil { + return nil, errx.InternalError() + } + if e.ID != recordID || e.ResolvedAt != nil { + return nil, nil + } return &e, nil } @@ -167,7 +206,7 @@ func (r *emailAccountErrorRepository) GetByAccountID(ctx context.Context, accoun query := ` SELECT id, email_account_id, user_id, error_code, severity, resolve_method, title, message, user_message, action_required, task_id, - resolved_at, resolved_by, created_at + resolved_at, resolved_by, created_at, operation, observation_source, received_at FROM email_account_errors WHERE email_account_id = $1 ` @@ -191,7 +230,7 @@ func (r *emailAccountErrorRepository) GetByAccountID(ctx context.Context, accoun err := rows.Scan( &e.ID, &e.EmailAccountID, &e.UserID, &e.ErrorCode, &e.Severity, &e.ResolveMethod, &e.Title, &e.Message, &e.UserMessage, &e.ActionRequired, &e.TaskID, - &e.ResolvedAt, &e.ResolvedBy, &e.CreatedAt, + &e.ResolvedAt, &e.ResolvedBy, &e.CreatedAt, &e.Operation, &e.ObservationSource, &e.ReceivedAt, ) if err != nil { db.CaptureError(err, "", nil, "scan") @@ -213,7 +252,7 @@ func (r *emailAccountErrorRepository) GetByAccountIDs(ctx context.Context, accou query := ` SELECT id, email_account_id, user_id, error_code, severity, resolve_method, title, message, user_message, action_required, task_id, - resolved_at, resolved_by, created_at + resolved_at, resolved_by, created_at, operation, observation_source, received_at FROM email_account_errors WHERE email_account_id = ANY($1::uuid[]) ` @@ -236,7 +275,7 @@ func (r *emailAccountErrorRepository) GetByAccountIDs(ctx context.Context, accou err := rows.Scan( &e.ID, &e.EmailAccountID, &e.UserID, &e.ErrorCode, &e.Severity, &e.ResolveMethod, &e.Title, &e.Message, &e.UserMessage, &e.ActionRequired, &e.TaskID, - &e.ResolvedAt, &e.ResolvedBy, &e.CreatedAt, + &e.ResolvedAt, &e.ResolvedBy, &e.CreatedAt, &e.Operation, &e.ObservationSource, &e.ReceivedAt, ) if err != nil { db.CaptureError(err, "", nil, "scan") @@ -257,7 +296,7 @@ func (r *emailAccountErrorRepository) GetByUserID(ctx context.Context, userID uu query := ` SELECT id, email_account_id, user_id, error_code, severity, resolve_method, title, message, user_message, action_required, task_id, - resolved_at, resolved_by, created_at + resolved_at, resolved_by, created_at, operation, observation_source, received_at FROM email_account_errors WHERE user_id = $1 ORDER BY created_at DESC @@ -277,7 +316,7 @@ func (r *emailAccountErrorRepository) GetByUserID(ctx context.Context, userID uu err := rows.Scan( &e.ID, &e.EmailAccountID, &e.UserID, &e.ErrorCode, &e.Severity, &e.ResolveMethod, &e.Title, &e.Message, &e.UserMessage, &e.ActionRequired, &e.TaskID, - &e.ResolvedAt, &e.ResolvedBy, &e.CreatedAt, + &e.ResolvedAt, &e.ResolvedBy, &e.CreatedAt, &e.Operation, &e.ObservationSource, &e.ReceivedAt, ) if err != nil { db.CaptureError(err, "", nil, "scan") @@ -352,37 +391,6 @@ func (r *emailAccountErrorRepository) ResolveByCodes(ctx context.Context, accoun return nil } -// ResolveByCodesBefore resolves the account's unresolved errors carrying any -// of the codes that were raised before the given moment. -// -// The bound matters because the bus can redeliver: JetStream is configured -// with MaxDeliver and no MaxAckPending, so an older event can arrive after a -// newer one. Without it, a stale "the sync succeeded" could clear a failure -// that happened afterwards, and the mailbox would look healthy while it was -// not, until the next distinct outage raised a fresh row. -func (r *emailAccountErrorRepository) ResolveByCodesBefore(ctx context.Context, accountID uuid.UUID, codes []string, before time.Time, resolvedBy string) *errx.Error { - if len(codes) == 0 { - return nil - } - - query := ` - UPDATE email_account_errors - SET resolved_at = NOW(), resolved_by = $1 - WHERE email_account_id = $2 - AND error_code = ANY($3::text[]) - AND created_at < $4 - AND resolved_at IS NULL - ` - - _, err := r.DB.Exec(ctx, query, resolvedBy, accountID, codes, before) - if err != nil { - db.CaptureError(err, query, []any{resolvedBy, accountID, codes, before}, "exec") - return errx.InternalError() - } - - return nil -} - // ResolveAllForAccount resolves all unresolved errors for an email account func (r *emailAccountErrorRepository) ResolveAllForAccount(ctx context.Context, accountID uuid.UUID, resolvedBy string) *errx.Error { query := ` diff --git a/internal/repository/pg_email_sync_state.go b/internal/repository/pg_email_sync_state.go index 309f235db..27c9c0ab7 100644 --- a/internal/repository/pg_email_sync_state.go +++ b/internal/repository/pg_email_sync_state.go @@ -82,7 +82,8 @@ func (r *pgEmailSyncStateRepository) Put(ctx context.Context, userID, emailID uu email_id, user_id, backfill_status, backfill_cursor, backfill_synced, backfill_since, backfill_started_at, backfill_completed_at, throttled_until, throttle_reason, deferred, last_synced_at, updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now()) + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, + CASE WHEN $12::timestamptz > to_timestamp(0) AND $12 <= now() THEN $12 END, now()) ON CONFLICT (email_id) DO UPDATE SET user_id = EXCLUDED.user_id, backfill_status = EXCLUDED.backfill_status, @@ -94,7 +95,7 @@ func (r *pgEmailSyncStateRepository) Put(ctx context.Context, userID, emailID uu throttled_until = EXCLUDED.throttled_until, throttle_reason = EXCLUDED.throttle_reason, deferred = EXCLUDED.deferred, - last_synced_at = COALESCE(EXCLUDED.last_synced_at, email_sync_state.last_synced_at), + last_synced_at = GREATEST(EXCLUDED.last_synced_at, email_sync_state.last_synced_at), updated_at = now() ` if _, err := r.db.Exec(ctx, q, @@ -106,7 +107,8 @@ func (r *pgEmailSyncStateRepository) Put(ctx context.Context, userID, emailID uu return fmt.Errorf("email_sync_state: put: %w", err) } if state.LastSyncedAt != nil { - const touch = `UPDATE email_accounts SET last_synced_at = $2 WHERE id = $1 AND (last_synced_at IS NULL OR last_synced_at < $2)` + const touch = `UPDATE email_accounts SET last_synced_at = $2 WHERE id = $1 AND (last_synced_at IS NULL OR last_synced_at < $2) + AND $2::timestamptz > to_timestamp(0) AND $2 <= now()` if _, err := r.db.Exec(ctx, touch, emailID, *state.LastSyncedAt); err != nil { return fmt.Errorf("email_sync_state: touch last_synced_at: %w", err) } diff --git a/internal/repository/pg_mail_transport_recovery.go b/internal/repository/pg_mail_transport_recovery.go new file mode 100644 index 000000000..3f7ab3451 --- /dev/null +++ b/internal/repository/pg_mail_transport_recovery.go @@ -0,0 +1,67 @@ +package repository + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" +) + +const MailTransportLockNamespace int64 = 285 + +func lockTransportMailbox(ctx context.Context, tx pgx.Tx, accountID uuid.UUID) error { + _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,$2))`, accountID.String(), MailTransportLockNamespace) + return err +} + +func (r *emailAccountErrorRepository) RecordTransportSuccess(ctx context.Context, accountID uuid.UUID, operation models.MailTransportOperation, at time.Time, taskID *uuid.UUID, generation uuid.UUID) *errx.Error { + if generation == uuid.Nil || !operation.Known() || !models.ValidMailTransportTime(at, time.Now()) { + return nil + } + if operation == models.MailTransportSend && taskID == nil || operation == models.MailTransportSync && taskID != nil { + return nil + } + tx, err := beginResultTx(ctx, r.DB) + if err != nil { + return errx.InternalError() + } + defer tx.Rollback(ctx) + if err := lockTransportMailbox(ctx, tx, accountID); err != nil { + return errx.InternalError() + } + var current uuid.UUID + if err := tx.QueryRow(ctx, `SELECT transport_generation FROM email_accounts WHERE id=$1 AND transport_generation=$2 FOR UPDATE`, accountID, generation).Scan(¤t); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + return errx.InternalError() + } + _, err = tx.Exec(ctx, `INSERT INTO email_account_transport_recovery AS existing(email_account_id,operation,recovered_at,source_task_id) + SELECT id,$2,$3,$4 FROM email_accounts WHERE id=$1 AND transport_generation=$5 AND $3::timestamptz <= NOW() + AND ($2='sync' OR EXISTS(SELECT 1 FROM tasks WHERE id=$4 AND email_account_id=$1 AND send_result_state='sent')) + ON CONFLICT(email_account_id,operation) DO UPDATE + SET recovered_at=GREATEST(existing.recovered_at,EXCLUDED.recovered_at), + source_task_id=CASE WHEN EXCLUDED.recovered_at >= existing.recovered_at THEN EXCLUDED.source_task_id ELSE existing.source_task_id END`, accountID, operation, at.UTC(), taskID, generation) + if err != nil { + return errx.InternalError() + } + _, err = tx.Exec(ctx, `UPDATE email_account_errors report SET resolved_at=recovery.recovered_at,resolved_by=$2 || ' recovered' + FROM email_account_transport_recovery recovery, email_accounts account + WHERE report.email_account_id=$1 AND report.operation=$2 AND report.resolved_at IS NULL + AND report.observation_source IN ('worker','receipt') + AND account.id=report.email_account_id AND account.transport_generation=$3 + AND recovery.email_account_id=report.email_account_id AND recovery.operation=report.operation + AND report.created_at < recovery.recovered_at - INTERVAL '1 second' + AND report.error_code IN ('SERVER_UNREACHABLE','CONNECTION_LOST','RESOURCE_NOT_FOUND','IMAP_UNKNOWN')`, accountID, operation, generation) + if err != nil { + return errx.InternalError() + } + if err := tx.Commit(ctx); err != nil { + return errx.InternalError() + } + return nil +} diff --git a/internal/repository/pg_monitoring.go b/internal/repository/pg_monitoring.go index ff419ff6e..d832717c4 100644 --- a/internal/repository/pg_monitoring.go +++ b/internal/repository/pg_monitoring.go @@ -226,7 +226,7 @@ var monitoringSections = map[string]monitoringSection{ {id: "credential_errors", title: "Unresolved mailbox credential reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS')`}, {id: "sync_errors", title: "Unresolved sync transport reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('SYNC_ERROR','SYNC_AUTH_ERROR','SYNC_CONNECTION_ERROR')`}, {id: "sync_safety_reports", title: "Unresolved sync flood/fair-use reports", unit: "reports", condition: "safety_hold", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code IN ('SYNC_FLOOD','SYNC_FAIR_USE')`}, - {id: "other_mailbox_errors", title: "Other unresolved mailbox reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated first stored report, not latest failing attempt or recurrence. Includes inactive accounts.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code NOT IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS','SYNC_ERROR','SYNC_AUTH_ERROR','SYNC_CONNECTION_ERROR','SYNC_FLOOD','SYNC_FAIR_USE')`}, + {id: "other_mailbox_errors", title: "Other unresolved mailbox reports", unit: "reports", condition: "recovery_unverified", severity: "warning", note: "Deduplicated reports include inactive accounts. Transport times reflect the latest worker observation where available; older rows or missing worker clocks use backend receipt time. Neither proves an ongoing failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r LEFT JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND r.error_code NOT IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS','SYNC_ERROR','SYNC_AUTH_ERROR','SYNC_CONNECTION_ERROR','SYNC_FLOOD','SYNC_FAIR_USE')`}, {id: "inactive_credential_errors", title: "Inactive mailboxes with unresolved credential reports", unit: "reports", condition: "safety_hold", severity: "warning", note: "Inactive without matching error is not an authentication failure.", window: 0 * time.Second, query: `SELECT COUNT(*), COUNT(DISTINCT r.email_account_id), COUNT(DISTINCT e.organization_id), COUNT(*) FILTER(WHERE e.organization_id IS NULL), MIN(r.created_at::timestamptz), MAX(r.created_at::timestamptz), MIN(NULL::timestamptz) FROM email_account_errors r JOIN email_accounts e ON e.id=r.email_account_id WHERE r.resolved_at IS NULL AND e.status<>'active' AND r.error_code IN ('GOOGLE_AUTHENTICATION_FAILED','AUTHENTICATION_FAILED','AUTHORIZATION_FAILED','INVALID_CREDENTIALS')`}, }}, "jobs": {permission: models.AdminPermViewAnalytics, investigate: "/jobs", queries: []monitoringQuery{ diff --git a/internal/repository/pg_orgtransfer.go b/internal/repository/pg_orgtransfer.go index eb8c22c15..daf86708d 100644 --- a/internal/repository/pg_orgtransfer.go +++ b/internal/repository/pg_orgtransfer.go @@ -29,6 +29,15 @@ import ( // Identifier safety: table names come from the compiled registry, and column // names are always intersected against the destination catalog before they // reach a query. Nothing from an uploaded archive is ever interpolated. +// ImportInvalidation clears derived rows keyed by records actually imported. +type ImportInvalidation struct { + ResetColumns []string + Table string + Column string + Key string + LockNamespaces []int64 +} + type OrgTransferRepository interface { // ---- schema introspection ---- @@ -66,7 +75,7 @@ type OrgTransferRepository interface { CountForeignRows(ctx context.Context, tx pgx.Tx, table, owner string, pk []string, refs []TenantReference, orgID uuid.UUID, rows []json.RawMessage) (int64, error) // InsertBatch writes rows into table, honouring the conflict strategy. An // overwrite only updates an existing row that owner selects for orgID. - InsertBatch(ctx context.Context, tx pgx.Tx, table string, cols []string, rows []json.RawMessage, conflict models.OrgImportConflict, pk []string, owner string, orgID uuid.UUID, resetOnOverwrite []string) (int64, error) + InsertBatch(ctx context.Context, tx pgx.Tx, table string, cols []string, rows []json.RawMessage, conflict models.OrgImportConflict, pk []string, owner string, orgID uuid.UUID, resetOnOverwrite []string, invalidations []ImportInvalidation) (int64, error) // DeveloperBlocked reports an operator's block on building apps for this // workspace or person. DeveloperBlocked(ctx context.Context, tx pgx.Tx, orgID, userID uuid.UUID) (bool, error) @@ -306,7 +315,7 @@ func (r *orgTransferRepository) StreamScoped(ctx context.Context, table, scope s return fmt.Errorf("lift statement timeout: %w", err) } - q := `SELECT to_jsonb(t) FROM public.` + quoteIdent(table) + ` t WHERE ` + scope + q := `SELECT to_jsonb(` + quoteIdent(table) + `) FROM public.` + quoteIdent(table) + ` WHERE ` + scope rows, err := tx.Query(ctx, q, orgID) if err != nil { return fmt.Errorf("read %s: %w", table, err) @@ -476,6 +485,7 @@ func (r *orgTransferRepository) InsertBatch( owner string, orgID uuid.UUID, resetOnOverwrite []string, + invalidations []ImportInvalidation, ) (int64, error) { if len(rows) == 0 || len(cols) == 0 { return 0, nil @@ -526,6 +536,45 @@ func (r *orgTransferRepository) InsertBatch( q += ` ON CONFLICT DO NOTHING` } + if len(invalidations) > 0 { + for _, relation := range invalidations { + for _, namespace := range relation.LockNamespaces { + lockQuery := `SELECT pg_advisory_xact_lock(hashtextextended(keys.key::text,$2)) + FROM (SELECT DISTINCT ` + quoteIdent(relation.Key) + ` AS key + FROM jsonb_populate_recordset(NULL::public.` + quoteIdent(table) + `,$1::jsonb)) keys + ORDER BY keys.key::text` + if _, err := tx.Exec(ctx, lockQuery, payload, namespace); err != nil { + return 0, fmt.Errorf("lock import checkpoints for %s: %w", table, err) + } + } + } + keys := make([]string, 0, len(invalidations)) + seen := map[string]bool{} + for _, relation := range invalidations { + if !seen[relation.Key] { + keys = append(keys, relation.Key) + seen[relation.Key] = true + } + } + q = `WITH imported AS (` + q + ` RETURNING ` + quoteIdents(keys) + `)` + for i, relation := range invalidations { + if len(relation.ResetColumns) > 0 { + sets := make([]string, 0, len(relation.ResetColumns)) + for _, c := range relation.ResetColumns { + sets = append(sets, quoteIdent(c)+" = DEFAULT") + } + q += fmt.Sprintf(", invalidated_%d AS (UPDATE public.%s SET %s WHERE %s IN (SELECT %s FROM imported))", i, quoteIdent(relation.Table), strings.Join(sets, ", "), quoteIdent(relation.Column), quoteIdent(relation.Key)) + } else { + q += fmt.Sprintf(", invalidated_%d AS (DELETE FROM public.%s WHERE %s IN (SELECT %s FROM imported))", i, quoteIdent(relation.Table), quoteIdent(relation.Column), quoteIdent(relation.Key)) + } + } + q += ` SELECT COUNT(*) FROM imported` + var written int64 + if err := tx.QueryRow(ctx, q, args...).Scan(&written); err != nil { + return 0, fmt.Errorf("insert into %s and invalidate checkpoints: %w", table, err) + } + return written, nil + } tag, err := tx.Exec(ctx, q, args...) if err != nil { return 0, fmt.Errorf("insert into %s: %w", table, err) diff --git a/internal/repository/pg_send_recovery.go b/internal/repository/pg_send_recovery.go index 26264d3d2..353439266 100644 --- a/internal/repository/pg_send_recovery.go +++ b/internal/repository/pg_send_recovery.go @@ -154,6 +154,8 @@ func resultState(result models.SendEmailResult) string { return "unknown" } +const SendResultMailboxLockNamespace int64 = 269 + // ApplySendResult commits reconciliation and its terminal marker together. func (r *taskRepository) ApplySendResult(ctx context.Context, result models.SendEmailResult, apply func(context.Context) error) error { if result.TaskID == uuid.Nil { @@ -179,7 +181,7 @@ func (r *taskRepository) ApplySendResult(ctx context.Context, result models.Send if err != nil { return err } - if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 269))`, mailboxID.String()); err != nil { + if _, err = tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, $2))`, mailboxID.String(), SendResultMailboxLockNamespace); err != nil { return err } state := resultState(result) diff --git a/internal/repository/unknown_send_resolution.go b/internal/repository/unknown_send_resolution.go index 85356b7b3..83654e4f0 100644 --- a/internal/repository/unknown_send_resolution.go +++ b/internal/repository/unknown_send_resolution.go @@ -106,7 +106,7 @@ func (r *taskRepository) ListUnappliedSendResults(ctx context.Context, limit int WHERE r.previous_reason='unknown' AND r.conflict_detected_at IS NULL AND r.confirmed_result IS NOT NULL AND r.recorded_locally_at IS NOT NULL AND w.dispatch_result IS NOT NULL AND t.send_result_applied_at IS NOT NULL AND t.status<>'cancelled' UNION ALL - SELECT t.id,t.created_at,t.send_result_state,t.send_result_applied_at,r.confirmed_result,2 FROM send_recovery_resolutions r JOIN tasks t ON t.id=r.recovery_task_id + SELECT t.id,t.created_at,t.send_result_state,t.send_result_applied_at,r.confirmed_result-'transport_generation',2 FROM send_recovery_resolutions r JOIN tasks t ON t.id=r.recovery_task_id WHERE r.confirmed_result IS NOT NULL AND r.recorded_locally_at IS NOT NULL AND (t.send_result_applied_at IS NULL OR (`+repairableSendDeadLetterPredicate+`)) AND t.status<>'cancelled' ),