feat: fence identified mailbox sync reports against current worker assignments and verify password-only reauthentication recovery

This commit is contained in:
Matthew Meszaros
2026-11-03 08:17:16 +01:00
parent 46106d13f8
commit 2da3e7d299
13 changed files with 250 additions and 26 deletions
+1 -1
View File
@@ -322,7 +322,7 @@ If Gmail expires its history checkpoint after a mailbox has been offline, Warmbl
Nested folders are followed as well, so mail in a subfolder of the inbox or under a label group arrives like anything else. Up to `100` folders per mailbox are synced. Past that the inbox, sent, drafts, spam, trash and archive are always kept and the rest are taken in the order the server lists them, with a note in the mailbox drawer's **Sync** card saying how many were left out. The note goes away by itself once the mailbox is back under the limit.
The last reported counts of folders omitted by the cap or a folder identity conflict are saved across backend and worker restarts. Updated workers timestamp folder-coverage changes separately, so delayed or repeated reports do not replace a newer observation. Coverage ordering keeps the original worker timestamp and accepts up to five seconds ahead of the database clock. It never replaces that timestamp with the time of receipt, including on replay. Timestamps farther ahead or at or before the Unix epoch are ignored along with their counters; a later heartbeat may accept the original timestamp once it enters the allowed window. Keep worker clocks synchronized. This allowance applies only to folder coverage, not to the last successful sync. During an upgrade or rollback with v0.6.72 workers, reports without that timestamp use the last successful sync as a fallback; an unstamped progress report cannot replace a timestamped observation. A legacy completion must also be at least as recent as the stored successful sync. A partial pass does not advance **Last synced**. The cap applies to the provider's folder listing before your exclusions, so excluding a folder does not remove that limit. Provider-confirmed sends alone do not establish complete sync coverage.
The last reported counts of folders omitted by the cap or a folder identity conflict are saved across backend and worker restarts. Updated workers timestamp folder-coverage changes separately, so delayed or repeated reports do not replace a newer observation. Their sync reports also identify the reporting worker. The database checks the current mailbox assignment while saving a report, and a report from a superseded worker does not change the stored sync state. During rollout, older workers omit this identity and retain timestamp-based fallback; deploy updated consumers before workers. Coverage ordering keeps the original worker timestamp and accepts up to five seconds ahead of the database clock. It never replaces that timestamp with the time of receipt, including on replay. Timestamps farther ahead or at or before the Unix epoch are ignored along with their counters; a later heartbeat may accept the original timestamp once it enters the allowed window. Keep worker clocks synchronized. This allowance applies only to folder coverage, not to the last successful sync. During an upgrade or rollback with v0.6.72 workers, reports without that timestamp use the last successful sync as a fallback; an unstamped progress report cannot replace a timestamped observation. A legacy completion must also be at least as recent as the stored successful sync. A partial pass does not advance **Last synced**. The cap applies to the provider's folder listing before your exclusions, so excluding a folder does not remove that limit. Provider-confirmed sends alone do not establish complete sync coverage.
On IMAP, folders are identified by the standard attributes a server publishes, and by name when it publishes none. The common names are recognized in a dozen languages, so a mailbox whose Sent folder is called "Gesendete Elemente" or "Éléments envoyés" still files sent mail as sent rather than as inbox.
@@ -248,3 +248,40 @@ func TestReauthHTTPPreservesPasswordAvailabilityFailure(t *testing.T) {
})
}
}
type reauthSequencedHashAuth struct {
reauthHTTPAuth
calls int
}
func (a *reauthSequencedHashAuth) PasswordHashFor(context.Context, uuid.UUID) (string, *errx.Error) {
a.calls++
if a.calls == 1 {
return "", errx.InternalError()
}
return a.hash, nil
}
func TestReauthHTTPStampsSessionOnPasswordAfterFailedLookup(t *testing.T) {
hash, err := argon2.Hash("correct-password")
if err != nil {
t.Fatal(err)
}
a := &reauthSequencedHashAuth{reauthHTTPAuth: reauthHTTPAuth{hash: hash}}
tokens := &reauthHTTPToken{}
h := &Handler{AuthService: a, TokenService: tokens, TwoFAService: reauthFactorStub{disabled: true}}
r := gin.New()
uid, sid := uuid.New(), uuid.New()
r.Use(func(c *gin.Context) {
c.Set(middleware.UserIDKey, uid.String())
c.Set(middleware.SessionKey, &models.Session{ID: sid, UserID: uid})
})
r.POST("/v1/auth/reauth", h.Reauth)
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/v1/auth/reauth", strings.NewReader(`{"password":"correct-password"}`))
req.Header.Set("Content-Type", "application/json")
r.ServeHTTP(w, req)
if w.Code != http.StatusOK || !tokens.stamped || tokens.stampedFor != sid || !a.cleared || a.clearedFor != uid {
t.Fatalf("status=%d stamped=%t stampedFor=%s cleared=%t clearedFor=%s", w.Code, tokens.stamped, tokens.stampedFor, a.cleared, a.clearedFor)
}
}
+10 -1
View File
@@ -34,13 +34,22 @@ func (s *JobsService) HandleSyncState(ctx context.Context, e *models.JobEventSyn
}
}
if err := s.EmailSyncStateRepository.Put(ctx, e.UserID, e.EmailID, &e.State); err != nil {
accepted := true
if e.WorkerID != nil {
accepted, err = s.EmailSyncStateRepository.PutFromWorker(ctx, e.UserID, e.EmailID, *e.WorkerID, &e.State)
} else {
err = s.EmailSyncStateRepository.Put(ctx, e.UserID, e.EmailID, &e.State)
}
if err != nil {
if s.dropForDeletedMailbox(ctx, e.UserID, e.EmailID, err) {
return nil
}
CaptureError(e.UserID, e.EmailID, err)
return err
}
if !accepted {
return nil
}
if s.StreamingPublisher == nil {
return nil
+28 -3
View File
@@ -17,9 +17,11 @@ var syncTestGeneration = uuid.New()
// stubSyncStateRepo accepts what the worker relays.
type stubSyncStateRepo struct {
repository.EmailSyncStateRepository
put *models.SyncState
saved *models.SyncState
getErr error
put *models.SyncState
saved *models.SyncState
getErr error
reportWorker uuid.UUID
reportAccepted bool
}
func (s *stubSyncStateRepo) Get(context.Context, uuid.UUID) (*models.SyncState, error) {
@@ -31,6 +33,29 @@ func (s *stubSyncStateRepo) Put(_ context.Context, _, _ uuid.UUID, st *models.Sy
return nil
}
func (s *stubSyncStateRepo) PutFromWorker(_ context.Context, _, _, workerID uuid.UUID, st *models.SyncState) (bool, error) {
s.reportWorker = workerID
if s.reportAccepted {
s.put = st
}
return s.reportAccepted, nil
}
func TestSyncStateChecksReportingWorkerBeforeSaving(t *testing.T) {
for _, accepted := range []bool{false, true} {
repo := &stubSyncStateRepo{reportAccepted: accepted}
s := &JobsService{EmailSyncStateRepository: repo}
workerID := uuid.New()
event := &models.JobEventSyncState{WorkerID: &workerID, EmailID: uuid.New(), State: models.SyncState{FoldersSkippedCap: 4}}
if err := s.HandleSyncState(t.Context(), event); err != nil {
t.Fatal(err)
}
if repo.reportWorker != workerID || (repo.put != nil) != accepted {
t.Fatalf("worker=%s accepted=%t saved=%+v", repo.reportWorker, accepted, repo.put)
}
}
}
// stubErrorRepo records which codes were resolved and how far back.
type stubErrorRepo struct {
repository.EmailAccountErrorRepository
@@ -692,6 +692,14 @@
},
{
"fields": [
{
"default": null,
"name": "worker_id",
"type": [
"null",
"string"
]
},
{
"default": "",
"name": "user_id",
+12
View File
@@ -2,12 +2,24 @@ package worker
import (
"context"
"fmt"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/infrastructure/kafka"
"github.com/warmbly/warmbly/internal/models"
)
func (s *WorkerService) Produce(jobEventType models.JobEventType, key string, body any) error {
if state, ok := body.(*models.JobEventSyncState); ok {
workerID, err := uuid.Parse(s.ID)
if err != nil || workerID == uuid.Nil {
return fmt.Errorf("sync state requires a valid reporting worker")
}
copy := *state
copy.WorkerID = &workerID
body = &copy
}
ctx := context.Background()
payload, err := s.Codec.Serialize(ctx, kafka.TopicWorkerEvents, &models.JobEvent{
Type: jobEventType,
@@ -0,0 +1,46 @@
package worker
import (
"context"
"encoding/json"
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/infrastructure/codec"
"github.com/warmbly/warmbly/internal/infrastructure/eventbus"
"github.com/warmbly/warmbly/internal/models"
)
type syncReportBus struct {
eventbus.EventBus
payload []byte
}
func (b *syncReportBus) Publish(_ context.Context, _, _ string, payload []byte) error {
b.payload = payload
return nil
}
func TestSyncStateReportsPublishingWorker(t *testing.T) {
wid, other := uuid.New(), uuid.New()
bus := &syncReportBus{}
w := &WorkerService{ID: wid.String(), Bus: bus, Codec: codec.NewJSON()}
report := &models.JobEventSyncState{WorkerID: &other, EmailID: uuid.New()}
if err := w.Produce(models.JobEventTypeSyncState, report.EmailID.String(), report); err != nil {
t.Fatal(err)
}
var got struct {
Body models.JobEventSyncState `json:"body"`
}
if err := json.Unmarshal(bus.payload, &got); err != nil {
t.Fatal(err)
}
if got.Body.WorkerID == nil || *got.Body.WorkerID != wid || got.Body.EmailID != report.EmailID || *report.WorkerID != other {
t.Fatalf("publishing identity or source report changed: %+v", got.Body)
}
bus.payload = nil
w.ID = ""
if err := w.Produce(models.JobEventTypeSyncState, report.EmailID.String(), report); err == nil || bus.payload != nil {
t.Fatal("report published without a worker identity")
}
}
@@ -66,7 +66,7 @@ func TestMailTransportCodecJSONCompatibility(t *testing.T) {
body = models.AddWorkerEmail{ID: mailbox, TransportGeneration: generation}
eventType = models.JobEventType(models.WorkerEventTypeAddEmail)
case "sync":
body = &models.JobEventSyncState{EmailID: mailbox, State: models.SyncState{LastSyncedAt: &at, FolderCoverageObservedAt: &at, TransportGeneration: generation}}
body = &models.JobEventSyncState{WorkerID: &generation, EmailID: mailbox, State: models.SyncState{LastSyncedAt: &at, FolderCoverageObservedAt: &at, TransportGeneration: generation}}
eventType = models.JobEventTypeSyncState
case "error":
body = models.EmailErrorEvent{EmailAccountID: mailbox.String(), TransportGeneration: generation, Operation: models.MailTransportSync}
@@ -100,6 +100,7 @@ func TestMailTransportCodecJSONCompatibility(t *testing.T) {
oldBody := old["body"].(map[string]any)
delete(oldBody, "transport_generation")
delete(oldBody, "operation")
delete(oldBody, "worker_id")
if state, ok := oldBody["state"].(map[string]any); ok {
delete(state, "transport_generation")
delete(state, "folder_coverage_observed_at")
@@ -122,7 +123,7 @@ func TestMailTransportCodecJSONCompatibility(t *testing.T) {
if err := c.Deserialize(t.Context(), "transport", payload, &current); err != nil {
t.Fatal(err)
}
if current.Body.State.TransportGeneration != uuid.Nil || current.Body.State.FolderCoverageObservedAt != nil || current.Body.State.LastSyncedAt == nil || !current.Body.State.LastSyncedAt.Equal(at) {
if current.Body.WorkerID != nil || current.Body.State.TransportGeneration != uuid.Nil || current.Body.State.FolderCoverageObservedAt != nil || current.Body.State.LastSyncedAt == nil || !current.Body.State.LastSyncedAt.Equal(at) {
t.Fatal("old sync changed checkpoint/proof")
}
case "error":
@@ -51,7 +51,7 @@ func priorTransportSchema(t *testing.T, schema avro.Schema) avro.Schema {
kept := fields[:0]
for _, f := range fields {
record := f.(map[string]any)
if record["name"] == "transport_generation" || record["name"] == "folder_coverage_observed_at" || x["name"] == "EmailErrorEvent" && record["name"] == "operation" {
if record["name"] == "transport_generation" || record["name"] == "folder_coverage_observed_at" || x["name"] == "EmailErrorEvent" && record["name"] == "operation" || x["name"] == "JobEventSyncState" && record["name"] == "worker_id" {
continue
}
strip(f)
+1 -1
View File
@@ -130,7 +130,7 @@ func preTransportSchema(t *testing.T, schema avro.Schema) avro.Schema {
kept := fields[:0]
for _, f := range fields {
field := f.(map[string]any)
if slices.Contains(fieldsAfterFixtures, field["name"]) || x["name"] == "EmailErrorEvent" && field["name"] == "operation" {
if slices.Contains(fieldsAfterFixtures, field["name"]) || x["name"] == "EmailErrorEvent" && field["name"] == "operation" || x["name"] == "JobEventSyncState" && field["name"] == "worker_id" {
continue
}
strip(f)
+4 -3
View File
@@ -144,7 +144,8 @@ type AddWorkerEmailSyncData struct {
// JobEventSyncState is the worker's SYNC_STATE relay: the full current state,
// not a delta, so a lost event is repaired by the next one.
type JobEventSyncState struct {
UserID uuid.UUID `json:"user_id" avro:"user_id"`
EmailID uuid.UUID `json:"email_id" avro:"email_id"`
State SyncState `json:"state" avro:"state"`
WorkerID *uuid.UUID `json:"worker_id,omitempty" avro:"worker_id"`
UserID uuid.UUID `json:"user_id" avro:"user_id"`
EmailID uuid.UUID `json:"email_id" avro:"email_id"`
State SyncState `json:"state" avro:"state"`
}
@@ -823,3 +823,69 @@ func TestLiveMailTransportSyncFolderCoverageAllowsBoundedClockSkew(t *testing.T)
t.Fatalf("replay ordering or successful-sync clock changed: %+v", got)
}
}
func TestLiveMailTransportSyncFolderCoverageRequiresCurrentWorker(t *testing.T) {
f := newTransportFixture(t)
repo := NewEmailSyncStateRepository(f.db)
oldWorker, currentWorker := uuid.New(), uuid.New()
if _, err := f.pool.Exec(t.Context(), `INSERT INTO fleet_nodes(id,role) VALUES($1,'worker'),($2,'worker')`, oldWorker, currentWorker); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if _, err := f.pool.Exec(context.Background(), `DELETE FROM fleet_nodes WHERE id IN ($1,$2)`, oldWorker, currentWorker); err != nil {
t.Error(err)
}
})
if _, err := f.pool.Exec(t.Context(), `INSERT INTO workers(id) VALUES($1),($2)`, oldWorker, currentWorker); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if _, err := f.pool.Exec(context.Background(), `DELETE FROM workers WHERE id IN ($1,$2)`, oldWorker, currentWorker); err != nil {
t.Error(err)
}
})
assign := func(worker *uuid.UUID) {
t.Helper()
if _, err := f.pool.Exec(t.Context(), `UPDATE email_accounts SET worker_id=$2 WHERE id=$1`, f.account, worker); err != nil {
t.Fatal(err)
}
}
completed := time.Now().UTC().Add(-time.Hour).Truncate(time.Millisecond)
first := completed.Add(time.Minute)
latestOld := first.Add(2 * time.Minute)
oldReport := models.SyncState{LastSyncedAt: &completed, FolderCoverageObservedAt: &first, FoldersSkippedCap: 4}
put := func(worker uuid.UUID, state *models.SyncState, want bool) {
t.Helper()
accepted, err := repo.PutFromWorker(t.Context(), f.user, f.account, worker, state)
if err != nil || accepted != want {
t.Fatalf("accepted=%t want=%t err=%v", accepted, want, err)
}
}
put(oldWorker, &oldReport, false)
assign(&oldWorker)
put(oldWorker, &oldReport, true)
seed, err := repo.Get(t.Context(), f.account)
if err != nil {
t.Fatal(err)
}
assign(&currentWorker)
oldReport.FolderCoverageObservedAt = &latestOld
oldReport.FoldersSkippedCap = 17
oldReport.BackfillSynced = 99
put(oldWorker, &oldReport, false)
put(uuid.Nil, &oldReport, false)
currentObservation := first.Add(time.Minute)
seed.FolderCoverageObservedAt = &currentObservation
seed.FoldersSkippedCap = 0
put(currentWorker, seed, true)
put(oldWorker, &oldReport, false)
got, err := repo.Get(t.Context(), f.account)
if err != nil {
t.Fatal(err)
}
if got.FoldersSkippedCap != 0 || got.BackfillSynced != 0 || got.FolderCoverageObservedAt == nil || !got.FolderCoverageObservedAt.Equal(currentObservation) {
t.Fatalf("current owner's report: %+v", got)
}
assign(nil)
put(currentWorker, &oldReport, false)
}
+33 -14
View File
@@ -21,6 +21,8 @@ type EmailSyncStateRepository interface {
// Put stores the full state and stamps email_accounts.last_synced_at, the
// column the admin and dashboard "Last synced" surfaces read.
Put(ctx context.Context, userID, emailID uuid.UUID, state *models.SyncState) error
// PutFromWorker stores a report only while the reporting worker owns the mailbox.
PutFromWorker(ctx context.Context, userID, emailID, workerID uuid.UUID, state *models.SyncState) (bool, error)
// Get returns nil, nil when the mailbox has never reported.
Get(ctx context.Context, emailID uuid.UUID) (*models.SyncState, error)
// IsOwnConversation reports whether any of the RFC message ids or the
@@ -66,12 +68,24 @@ func NewEmailSyncStateRepository(d *db.DB) EmailSyncStateRepository {
}
func (r *pgEmailSyncStateRepository) Put(ctx context.Context, userID, emailID uuid.UUID, state *models.SyncState) error {
_, err := r.put(ctx, userID, emailID, nil, state)
return err
}
func (r *pgEmailSyncStateRepository) PutFromWorker(ctx context.Context, userID, emailID, workerID uuid.UUID, state *models.SyncState) (bool, error) {
if workerID == uuid.Nil {
return false, nil
}
return r.put(ctx, userID, emailID, &workerID, state)
}
func (r *pgEmailSyncStateRepository) put(ctx context.Context, userID, emailID uuid.UUID, workerID *uuid.UUID, state *models.SyncState) (bool, error) {
if state == nil {
return nil
return false, nil
}
cursor, err := json.Marshal(state.BackfillCursor)
if err != nil {
return fmt.Errorf("email_sync_state: cursor: %w", err)
return false, fmt.Errorf("email_sync_state: cursor: %w", err)
}
status := state.BackfillStatus
if status == "" {
@@ -79,7 +93,9 @@ func (r *pgEmailSyncStateRepository) Put(ctx context.Context, userID, emailID uu
}
const q = `
-- Retain original coverage clocks for replay ordering, allowing at most five seconds of lead.
WITH coverage AS (
WITH owner AS (
SELECT id FROM email_accounts WHERE id = $1 AND ($16::uuid IS NULL OR worker_id = $16) FOR UPDATE
), coverage AS (
SELECT $15::timestamptz IS NULL OR ($15 > to_timestamp(0) AND $15 <= now() + interval '5 seconds') AS valid
)
INSERT INTO email_sync_state (
@@ -91,8 +107,8 @@ func (r *pgEmailSyncStateRepository) Put(ctx context.Context, userID, emailID uu
CASE WHEN $12::timestamptz > to_timestamp(0) AND $12 <= now() THEN $12 END,
CASE WHEN valid THEN $13 ELSE 0 END, CASE WHEN valid THEN $14 ELSE 0 END,
CASE WHEN valid THEN $15::timestamptz END, now()
FROM coverage
WHERE true
FROM coverage LEFT JOIN owner ON true
WHERE $16::uuid IS NULL OR owner.id IS NOT NULL
ON CONFLICT (email_id) DO UPDATE SET
user_id = EXCLUDED.user_id,
backfill_status = EXCLUDED.backfill_status,
@@ -123,23 +139,26 @@ func (r *pgEmailSyncStateRepository) Put(ctx context.Context, userID, emailID uu
last_synced_at = GREATEST(EXCLUDED.last_synced_at, email_sync_state.last_synced_at),
updated_at = now()
`
if _, err := r.db.Exec(ctx, q,
command, err := r.db.Exec(ctx, q,
emailID, userID, string(status), cursor, state.BackfillSynced,
state.BackfillSince, state.BackfillStartedAt, state.BackfillCompletedAt,
state.ThrottledUntil, state.ThrottleReason, state.Deferred, state.LastSyncedAt,
state.FoldersSkippedCap, state.FoldersSkippedConflict, state.FolderCoverageObservedAt,
); err != nil {
// A deleted mailbox refuses this as a foreign-key violation; the consumer evicts it.
return fmt.Errorf("email_sync_state: put: %w", err)
state.FoldersSkippedCap, state.FoldersSkippedConflict, state.FolderCoverageObservedAt, workerID,
)
if err != nil {
return false, fmt.Errorf("email_sync_state: put: %w", err)
}
if command.RowsAffected() == 0 {
return false, nil
}
if state.LastSyncedAt != nil {
const touch = `UPDATE email_accounts SET last_synced_at = $2 WHERE id = $1 AND (last_synced_at IS NULL OR last_synced_at < $2)
AND $2::timestamptz > to_timestamp(0) AND $2 <= now()`
if _, err := r.db.Exec(ctx, touch, emailID, *state.LastSyncedAt); err != nil {
return fmt.Errorf("email_sync_state: touch last_synced_at: %w", err)
AND $2::timestamptz > to_timestamp(0) AND $2 <= now() AND ($3::uuid IS NULL OR worker_id = $3)`
if _, err := r.db.Exec(ctx, touch, emailID, *state.LastSyncedAt, workerID); err != nil {
return false, fmt.Errorf("email_sync_state: touch last_synced_at: %w", err)
}
}
return nil
return true, nil
}
func (r *pgEmailSyncStateRepository) Get(ctx context.Context, emailID uuid.UUID) (*models.SyncState, error) {