diff --git a/internal/app/consumer/warmup_inbox_cleanup.go b/internal/app/consumer/warmup_inbox_cleanup.go index 7edde7756..b62aa98a2 100644 --- a/internal/app/consumer/warmup_inbox_cleanup.go +++ b/internal/app/consumer/warmup_inbox_cleanup.go @@ -13,6 +13,7 @@ import ( "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/observability/arrivaldiag" + "github.com/warmbly/warmbly/internal/repository" ) // StartWarmupInboxCleanup repairs old leaks in bounded passes, retrying verification failures per mailbox. @@ -291,14 +292,23 @@ func (s *JobsService) retryPendingWarmupVerification(ctx context.Context) error if err != nil { return err } - var failures []error + var failures, waiting []error for _, e := range events { - if err := s.UniboxRepository.ProcessPendingWarmupVerification(ctx, e.Message.ID, func(current *models.JobEventNewEmail) error { + err := s.UniboxRepository.ProcessPendingWarmupVerification(ctx, e.Message.ID, func(current *models.JobEventNewEmail) error { return s.ingestNewEmail(ctx, current) - }); err != nil { + }) + switch { + case err == nil: + case errors.Is(err, repository.ErrWarmupDeliveryPending): + // The hold this loop exists for: the send's provider id is not recorded yet, so the lease brings it back. + waiting = append(waiting, err) + default: failures = append(failures, err) } } + if len(failures) == 0 { + return jobrun.Quiet(errors.Join(waiting...)) + } return errors.Join(failures...) } diff --git a/internal/app/consumer/warmup_inbox_test.go b/internal/app/consumer/warmup_inbox_test.go index f7c8bbaa7..3ff000fb6 100644 --- a/internal/app/consumer/warmup_inbox_test.go +++ b/internal/app/consumer/warmup_inbox_test.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/warmbly/warmbly/internal/config" "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/repository" ) @@ -203,3 +204,59 @@ func TestWarmupInboxCleanupRetriesCloudFailureWithoutDeletingMail(t *testing.T) t.Fatalf("cleanup retry failed: %v %v %v", sweep.afterID, done, err) } } + +type pendingVerificationInbox struct { + repository.UniboxRepository + held []models.JobEventNewEmail + processed int +} + +func (r *pendingVerificationInbox) ClaimPendingWarmupVerification(context.Context, int) ([]models.JobEventNewEmail, error) { + return r.held, nil +} + +func (r *pendingVerificationInbox) ProcessPendingWarmupVerification(_ context.Context, id uuid.UUID, process func(*models.JobEventNewEmail) error) error { + for i := range r.held { + if r.held[i].Message.ID == id { + r.processed++ + return process(&r.held[i]) + } + } + return nil +} + +type pendingDeliveryRepo struct { + repository.WarmupRepository + deliveryErr map[string]error +} + +func (pendingDeliveryRepo) FindDeliveredWarmupToken(context.Context, uuid.UUID, string, string, string) (*models.WarmupToken, error) { + return nil, nil +} + +func (r pendingDeliveryRepo) IsWarmupDelivery(_ context.Context, _ uuid.UUID, _, messageID, _ string) (bool, error) { + return false, r.deliveryErr[messageID] +} + +// A send whose provider id is not recorded yet is the wait this loop exists for, not a failure to report. +func TestPendingWarmupVerificationWaitsQuietlyForTheSendConfirmation(t *testing.T) { + arrival := func(messageID string) models.JobEventNewEmail { + return models.JobEventNewEmail{UserID: uuid.New(), Message: &models.EmailMessageStoreData{ + ID: uuid.New(), EmailID: uuid.New(), MessageID: messageID, Subject: "quick question", FromAddr: []string{"[EMAIL]"}, + }} + } + inbox := &pendingVerificationInbox{held: []models.JobEventNewEmail{arrival(""), arrival("")}} + repo := pendingDeliveryRepo{deliveryErr: map[string]error{"": repository.ErrWarmupDeliveryPending, "": repository.ErrWarmupDeliveryPending}} + s := &JobsService{UniboxRepository: inbox, WarmupRepo: repo} + + err := s.retryPendingWarmupVerification(context.Background()) + if inbox.processed != 2 || !errors.Is(err, repository.ErrWarmupDeliveryPending) || !jobrun.IsQuiet(err) { + t.Fatalf("awaited confirmations were reported or dropped: processed=%d err=%v", inbox.processed, err) + } + + repo.deliveryErr[""] = errors.New("connection reset") + err = s.retryPendingWarmupVerification(context.Background()) + if err == nil || jobrun.IsQuiet(err) || errors.Is(err, repository.ErrWarmupDeliveryPending) { + t.Fatalf("a lookup failure must be reported on its own: %v", err) + } +} diff --git a/internal/app/consumer/warmup_verification_live_test.go b/internal/app/consumer/warmup_verification_live_test.go index 2a292dea1..e472b87c6 100644 --- a/internal/app/consumer/warmup_verification_live_test.go +++ b/internal/app/consumer/warmup_verification_live_test.go @@ -268,6 +268,36 @@ func TestLiveWarmupLineageConcurrentInitialRelayArrivalsEngageOnce(t *testing.T) } } +// A send that ended failed or cancelled is never confirmed, so its arrival is filed now instead of held until the token expires. +func TestLiveWarmupLineageArrivalOfASettledSendIsNotHeld(t *testing.T) { + for _, status := range []string{"failed", "cancelled"} { + t.Run(status, func(t *testing.T) { + s, handle := liveWarmupService(t) + ctx := t.Context() + f := newWarmupFixture(t, handle) + token := f.mintToken(t, s.WarmupRepo) + if _, err := handle.Exec(ctx, `INSERT INTO warmup_tasks(task_id,lineage_version,subject,max_turns)VALUES($1,1,$2,1)`, f.task, f.subject); err != nil { + t.Fatal(err) + } + if _, err := handle.Exec(ctx, `UPDATE tasks SET status=$2::task_status WHERE id=$1`, f.task, status); err != nil { + t.Fatal(err) + } + e := f.arrival("<[EMAIL]>", []string{config.WarmupVerifyHeader + ":" + token.String()}) + if err := s.HandleNewEmail(ctx, e); err != nil { + t.Fatal(err) + } + var pending, visible, receipts int + if err := handle.QueryRow(ctx, `SELECT (SELECT COUNT(*) FROM unibox_pending_emails WHERE id=$1), (SELECT COUNT(*) FROM unibox_emails WHERE id=$1), + (SELECT COUNT(*) FROM warmup_received WHERE sender_account_id=$2)`, e.Message.ID, f.sender).Scan(&pending, &visible, &receipts); err != nil { + t.Fatal(err) + } + if pending != 0 || visible != 0 || receipts != 0 { + t.Fatalf("settled send: pending=%d visible=%d receipts=%d", pending, visible, receipts) + } + }) + } +} + func TestLiveWarmupLineageConsumerReceiptGuards(t *testing.T) { for _, name := range []string{"exact headerless", "headerless rewrite", "forwarded marker", "foreign mailbox", "expired", "retired"} { t.Run(name, func(t *testing.T) { diff --git a/internal/app/warmupcontent/controls_test.go b/internal/app/warmupcontent/controls_test.go index bc1577136..64496747e 100644 --- a/internal/app/warmupcontent/controls_test.go +++ b/internal/app/warmupcontent/controls_test.go @@ -395,3 +395,20 @@ func TestCanonicalTransformKeepsEmptyTurnsAndFacts(t *testing.T) { t.Fatal("facts or alternating positions changed") } } + +func TestScheduledGenerationWaitsOnAnInFlightBatchWithoutFailing(t *testing.T) { + s, r, p := newControlFixture() + r.settings.RefreshEnabled = false + r.reserveErr = repository.ErrScheduledGenerationInFlight + if err := s.RunScheduled(t.Context()); err != nil { + t.Fatalf("a segment already being topped up failed the run: %v", err) + } + if len(p.requests) != 0 || len(r.jobs) != 0 { + t.Fatal("a second batch was submitted while one was in flight") + } + r.reserveErr = errors.New("connection reset") + err := s.RunScheduled(t.Context()) + if err == nil || !strings.Contains(err.Error(), "shared library") || !strings.Contains(err.Error(), "connection reset") { + t.Fatalf("a real submission failure lost its segment or cause: %v", err) + } +} diff --git a/internal/app/warmupcontent/service.go b/internal/app/warmupcontent/service.go index 8719c321f..0a690e137 100644 --- a/internal/app/warmupcontent/service.go +++ b/internal/app/warmupcontent/service.go @@ -229,9 +229,13 @@ func (s *service) RunScheduled(ctx context.Context) error { Model: settings.Model, Count: count, }) + if errors.Is(err, repository.ErrScheduledGenerationInFlight) { + // Its output is not in the bank until the poller ingests it, so the deficit stands meanwhile. + continue + } if err != nil { log.Warn().Err(err).Str("segment", segment).Msg("warmup generation: scheduled batch not submitted") - scheduleErr = errors.Join(scheduleErr, fmt.Errorf("warmup batch submission failed for segment %q", segment)) + scheduleErr = errors.Join(scheduleErr, fmt.Errorf("warmup batch submission failed for %s: %w", segmentLabel(segment), err)) continue } if settings.DailyGenerationCap > 0 { @@ -245,6 +249,14 @@ func (s *service) RunScheduled(ctx context.Context) error { return scheduleErr } +// segmentLabel names a library segment; "" is the shared library every mailbox draws from. +func segmentLabel(segment string) string { + if segment == "" { + return "the shared library" + } + return fmt.Sprintf("segment %q", segment) +} + func cleanMessages(in []string) []string { out := make([]string, 0, len(in)) for _, m := range in { diff --git a/internal/jobrun/jobrun.go b/internal/jobrun/jobrun.go index ba2cd1d20..8c83e049a 100644 --- a/internal/jobrun/jobrun.go +++ b/internal/jobrun/jobrun.go @@ -272,8 +272,7 @@ func Run(ctx context.Context, name string, fn func(ctx context.Context) error) { } } runErr := safeRun(ctx, name, fn) - var quiet quietError - if runErr != nil && !errors.As(runErr, &quiet) { + if runErr != nil && !IsQuiet(runErr) { errs.CaptureException(runErr) log.Warn().Err(runErr).Str("job", name).Msg("scheduled job failed") } @@ -284,7 +283,7 @@ func Run(ctx context.Context, name string, fn func(ctx context.Context) error) { } } -// Quiet marks a standing condition a job already reported: the panel records it, but it is not logged or captured again. +// Quiet marks an expected or already reported condition: the panel records it, but it is not logged or captured. func Quiet(err error) error { if err == nil { return nil @@ -292,6 +291,12 @@ func Quiet(err error) error { return quietError{err} } +// IsQuiet reports whether err was marked with Quiet. +func IsQuiet(err error) bool { + var quiet quietError + return errors.As(err, &quiet) +} + type quietError struct{ error } func (q quietError) Unwrap() error { return q.error } diff --git a/internal/jobs/warmup_generation.go b/internal/jobs/warmup_generation.go index 63848f75c..8b616e994 100644 --- a/internal/jobs/warmup_generation.go +++ b/internal/jobs/warmup_generation.go @@ -5,8 +5,6 @@ import ( "sync" "time" - "github.com/warmbly/warmbly/internal/observability/errs" - "github.com/warmbly/warmbly/internal/app/warmupcontent" "github.com/warmbly/warmbly/internal/jobrun" "github.com/warmbly/warmbly/internal/repository" @@ -36,9 +34,9 @@ func (j *WarmupGenerationJob) Run(ctx context.Context) error { if j.svc == nil || j.repo == nil || !j.svc.Enabled() { return nil } + // jobrun reports the returned error; capturing it here as well would file it twice. settings, err := j.repo.GetGenerationSettings(ctx) if err != nil { - errs.CaptureException(err) return err } if settings == nil || !settings.ScheduleEnabled { @@ -53,7 +51,6 @@ func (j *WarmupGenerationJob) Run(ctx context.Context) error { } if err := j.svc.RunScheduled(ctx); err != nil { - errs.CaptureException(err) return err } j.lastRun = time.Now() diff --git a/internal/repository/content_opening_live_test.go b/internal/repository/content_opening_live_test.go index 8863c6cf8..d4c37b9aa 100644 --- a/internal/repository/content_opening_live_test.go +++ b/internal/repository/content_opening_live_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/google/uuid" + "github.com/warmbly/warmbly/internal/models" ) func TestLiveContentPoolRejectsDuplicateBatchContentUnderConcurrency(t *testing.T) { @@ -57,6 +58,26 @@ func TestLiveContentPoolRejectsDuplicateBatchContentUnderConcurrency(t *testing. } } +func TestLiveScheduledGenerationReservesOneInFlightBatchPerSegment(t *testing.T) { + f, r, segment, _ := contentPoolFixture(t) + t.Cleanup(func() { + _, _ = f.pool.Exec(t.Context(), `DELETE FROM warmup_generation_jobs WHERE segment=$1`, segment) + }) + job := func(trigger string) *models.WarmupGenerationJob { + return &models.WarmupGenerationJob{ID: uuid.New(), Trigger: trigger, Mode: models.WarmupGenerationModeBatch, + PoolType: "premium", Segment: segment, Model: "fixture-model", RequestedCount: 1, Status: "pending"} + } + if err := r.CreateGenerationJob(t.Context(), job("schedule")); err != nil { + t.Fatal(err) + } + if err := r.CreateGenerationJob(t.Context(), job("schedule")); !errors.Is(err, ErrScheduledGenerationInFlight) { + t.Fatalf("second scheduled batch for the segment: %v", err) + } + if err := r.CreateGenerationJob(t.Context(), job("manual")); err != nil { + t.Fatalf("a manual batch is not limited to one in flight: %v", err) + } +} + func contentPoolFixture(t *testing.T) (*poolLinkFixture, *warmupContentRepository, string, uuid.UUID) { t.Helper() f, _ := lineageFixture(t) diff --git a/internal/repository/pg_warmup.go b/internal/repository/pg_warmup.go index df9c0dace..d0ae99fb8 100644 --- a/internal/repository/pg_warmup.go +++ b/internal/repository/pg_warmup.go @@ -1701,18 +1701,23 @@ func (r *warmupRepository) acceptReceipt(ctx context.Context, q interface { if token.ConsumedAt != nil || !token.ExpiresAt.After(time.Now()) { return true, nil } - var lineage, awaitingSend, retired bool + var lineage, awaitingSend, retired, settled bool if err = q.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM warmup_tasks WHERE task_id=$1 AND lineage_version=1), EXISTS(SELECT 1 FROM warmup_tasks w JOIN tasks sent ON sent.id=w.task_id WHERE w.task_id=$1 AND w.lineage_version=1 AND (sent.status<>'completed' OR $2='')), EXISTS(SELECT 1 FROM warmup_tokens WHERE token=$3 AND sent_retired_at IS NOT NULL) - OR EXISTS(SELECT 1 FROM tasks WHERE id=$1 AND warmup_send_retired_at IS NOT NULL)`, token.TaskID, token.SentMessageID, token.Token).Scan(&lineage, &awaitingSend, &retired); err != nil { + OR EXISTS(SELECT 1 FROM tasks WHERE id=$1 AND warmup_send_retired_at IS NOT NULL), + EXISTS(SELECT 1 FROM tasks WHERE id=$1 AND status IN ('failed','cancelled'))`, token.TaskID, token.SentMessageID, token.Token).Scan(&lineage, &awaitingSend, &retired, &settled); err != nil { return false, err } if retired { return true, nil } if awaitingSend { + if settled { + // A failed or cancelled send is never confirmed later, so it gets now what expiry would give it. + return true, nil + } return false, ErrWarmupDeliveryPending } if lineage && (strings.Trim(proof.MessageID, "<> \t") == "" || diff --git a/internal/repository/pg_warmup_content.go b/internal/repository/pg_warmup_content.go index 83aa3c121..ed084d10d 100644 --- a/internal/repository/pg_warmup_content.go +++ b/internal/repository/pg_warmup_content.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" "github.com/warmbly/warmbly/internal/models" "github.com/warmbly/warmbly/internal/pkg/generation" @@ -106,6 +107,12 @@ type warmupContentRepository struct { var ErrWarmupConversationDuplicate = errors.New("warmup subject or opening is already in the conversation library") +// ErrScheduledGenerationInFlight: the library segment already has a scheduled batch pending or running. +var ErrScheduledGenerationInFlight = errors.New("a scheduled warmup batch for this library segment is already in flight") + +// scheduledGenerationInFlightIndex is the unique index from migration 000077. +const scheduledGenerationInFlightIndex = "idx_warmup_generation_jobs_one_scheduled_inflight" + // NewWarmupContentRepository creates a new warmup content repository. func NewWarmupContentRepository(db *pgxpool.Pool) WarmupContentRepository { return &warmupContentRepository{db: db} @@ -507,6 +514,10 @@ func (r *warmupContentRepository) CreateGenerationJob(ctx context.Context, j *mo j.ID, j.RequestedBy, j.Trigger, j.Mode, j.PoolType, j.Segment, j.Theme, j.Model, j.RequestedCount, j.Status, j.BatchID, j.BatchInputFileID, j.BatchOutputFileID, j.BatchStatus, j.CompletionWindow, j.ContentVersion, j.MaxMessagesPerThread, ) + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == scheduledGenerationInFlightIndex { + return ErrScheduledGenerationInFlight + } if err != nil { return err }