mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 16:19:13 +00:00
Merge pull request #1178 from warmbly/fix/warmup-posthog-errors
feat: wait quietly for a warmup send's Message-ID, settle holds whose send failed or was cancelled, and treat an in-flight scheduled generation batch as already topping up its segment
This commit is contained in:
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/warmbly/warmbly/internal/jobrun"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/observability/arrivaldiag"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
|
||||
// StartWarmupInboxCleanup repairs old leaks in bounded passes, retrying verification failures per mailbox.
|
||||
@@ -291,14 +292,23 @@ func (s *JobsService) retryPendingWarmupVerification(ctx context.Context) error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var failures []error
|
||||
var failures, waiting []error
|
||||
for _, e := range events {
|
||||
if err := s.UniboxRepository.ProcessPendingWarmupVerification(ctx, e.Message.ID, func(current *models.JobEventNewEmail) error {
|
||||
err := s.UniboxRepository.ProcessPendingWarmupVerification(ctx, e.Message.ID, func(current *models.JobEventNewEmail) error {
|
||||
return s.ingestNewEmail(ctx, current)
|
||||
}); err != nil {
|
||||
})
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.Is(err, repository.ErrWarmupDeliveryPending):
|
||||
// The hold this loop exists for: the send's provider id is not recorded yet, so the lease brings it back.
|
||||
waiting = append(waiting, err)
|
||||
default:
|
||||
failures = append(failures, err)
|
||||
}
|
||||
}
|
||||
if len(failures) == 0 {
|
||||
return jobrun.Quiet(errors.Join(waiting...))
|
||||
}
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/config"
|
||||
"github.com/warmbly/warmbly/internal/errx"
|
||||
"github.com/warmbly/warmbly/internal/jobrun"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
)
|
||||
@@ -203,3 +204,59 @@ func TestWarmupInboxCleanupRetriesCloudFailureWithoutDeletingMail(t *testing.T)
|
||||
t.Fatalf("cleanup retry failed: %v %v %v", sweep.afterID, done, err)
|
||||
}
|
||||
}
|
||||
|
||||
type pendingVerificationInbox struct {
|
||||
repository.UniboxRepository
|
||||
held []models.JobEventNewEmail
|
||||
processed int
|
||||
}
|
||||
|
||||
func (r *pendingVerificationInbox) ClaimPendingWarmupVerification(context.Context, int) ([]models.JobEventNewEmail, error) {
|
||||
return r.held, nil
|
||||
}
|
||||
|
||||
func (r *pendingVerificationInbox) ProcessPendingWarmupVerification(_ context.Context, id uuid.UUID, process func(*models.JobEventNewEmail) error) error {
|
||||
for i := range r.held {
|
||||
if r.held[i].Message.ID == id {
|
||||
r.processed++
|
||||
return process(&r.held[i])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type pendingDeliveryRepo struct {
|
||||
repository.WarmupRepository
|
||||
deliveryErr map[string]error
|
||||
}
|
||||
|
||||
func (pendingDeliveryRepo) FindDeliveredWarmupToken(context.Context, uuid.UUID, string, string, string) (*models.WarmupToken, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r pendingDeliveryRepo) IsWarmupDelivery(_ context.Context, _ uuid.UUID, _, messageID, _ string) (bool, error) {
|
||||
return false, r.deliveryErr[messageID]
|
||||
}
|
||||
|
||||
// A send whose provider id is not recorded yet is the wait this loop exists for, not a failure to report.
|
||||
func TestPendingWarmupVerificationWaitsQuietlyForTheSendConfirmation(t *testing.T) {
|
||||
arrival := func(messageID string) models.JobEventNewEmail {
|
||||
return models.JobEventNewEmail{UserID: uuid.New(), Message: &models.EmailMessageStoreData{
|
||||
ID: uuid.New(), EmailID: uuid.New(), MessageID: messageID, Subject: "quick question", FromAddr: []string{"[EMAIL]"},
|
||||
}}
|
||||
}
|
||||
inbox := &pendingVerificationInbox{held: []models.JobEventNewEmail{arrival("<a@x>"), arrival("<b@x>")}}
|
||||
repo := pendingDeliveryRepo{deliveryErr: map[string]error{"<a@x>": repository.ErrWarmupDeliveryPending, "<b@x>": repository.ErrWarmupDeliveryPending}}
|
||||
s := &JobsService{UniboxRepository: inbox, WarmupRepo: repo}
|
||||
|
||||
err := s.retryPendingWarmupVerification(context.Background())
|
||||
if inbox.processed != 2 || !errors.Is(err, repository.ErrWarmupDeliveryPending) || !jobrun.IsQuiet(err) {
|
||||
t.Fatalf("awaited confirmations were reported or dropped: processed=%d err=%v", inbox.processed, err)
|
||||
}
|
||||
|
||||
repo.deliveryErr["<b@x>"] = errors.New("connection reset")
|
||||
err = s.retryPendingWarmupVerification(context.Background())
|
||||
if err == nil || jobrun.IsQuiet(err) || errors.Is(err, repository.ErrWarmupDeliveryPending) {
|
||||
t.Fatalf("a lookup failure must be reported on its own: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,6 +268,36 @@ func TestLiveWarmupLineageConcurrentInitialRelayArrivalsEngageOnce(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
// A send that ended failed or cancelled is never confirmed, so its arrival is filed now instead of held until the token expires.
|
||||
func TestLiveWarmupLineageArrivalOfASettledSendIsNotHeld(t *testing.T) {
|
||||
for _, status := range []string{"failed", "cancelled"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
s, handle := liveWarmupService(t)
|
||||
ctx := t.Context()
|
||||
f := newWarmupFixture(t, handle)
|
||||
token := f.mintToken(t, s.WarmupRepo)
|
||||
if _, err := handle.Exec(ctx, `INSERT INTO warmup_tasks(task_id,lineage_version,subject,max_turns)VALUES($1,1,$2,1)`, f.task, f.subject); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := handle.Exec(ctx, `UPDATE tasks SET status=$2::task_status WHERE id=$1`, f.task, status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := f.arrival("<[EMAIL]>", []string{config.WarmupVerifyHeader + ":" + token.String()})
|
||||
if err := s.HandleNewEmail(ctx, e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var pending, visible, receipts int
|
||||
if err := handle.QueryRow(ctx, `SELECT (SELECT COUNT(*) FROM unibox_pending_emails WHERE id=$1), (SELECT COUNT(*) FROM unibox_emails WHERE id=$1),
|
||||
(SELECT COUNT(*) FROM warmup_received WHERE sender_account_id=$2)`, e.Message.ID, f.sender).Scan(&pending, &visible, &receipts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pending != 0 || visible != 0 || receipts != 0 {
|
||||
t.Fatalf("settled send: pending=%d visible=%d receipts=%d", pending, visible, receipts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveWarmupLineageConsumerReceiptGuards(t *testing.T) {
|
||||
for _, name := range []string{"exact headerless", "headerless rewrite", "forwarded marker", "foreign mailbox", "expired", "retired"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
|
||||
@@ -395,3 +395,20 @@ func TestCanonicalTransformKeepsEmptyTurnsAndFacts(t *testing.T) {
|
||||
t.Fatal("facts or alternating positions changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledGenerationWaitsOnAnInFlightBatchWithoutFailing(t *testing.T) {
|
||||
s, r, p := newControlFixture()
|
||||
r.settings.RefreshEnabled = false
|
||||
r.reserveErr = repository.ErrScheduledGenerationInFlight
|
||||
if err := s.RunScheduled(t.Context()); err != nil {
|
||||
t.Fatalf("a segment already being topped up failed the run: %v", err)
|
||||
}
|
||||
if len(p.requests) != 0 || len(r.jobs) != 0 {
|
||||
t.Fatal("a second batch was submitted while one was in flight")
|
||||
}
|
||||
r.reserveErr = errors.New("connection reset")
|
||||
err := s.RunScheduled(t.Context())
|
||||
if err == nil || !strings.Contains(err.Error(), "shared library") || !strings.Contains(err.Error(), "connection reset") {
|
||||
t.Fatalf("a real submission failure lost its segment or cause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,9 +229,13 @@ func (s *service) RunScheduled(ctx context.Context) error {
|
||||
Model: settings.Model,
|
||||
Count: count,
|
||||
})
|
||||
if errors.Is(err, repository.ErrScheduledGenerationInFlight) {
|
||||
// Its output is not in the bank until the poller ingests it, so the deficit stands meanwhile.
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("segment", segment).Msg("warmup generation: scheduled batch not submitted")
|
||||
scheduleErr = errors.Join(scheduleErr, fmt.Errorf("warmup batch submission failed for segment %q", segment))
|
||||
scheduleErr = errors.Join(scheduleErr, fmt.Errorf("warmup batch submission failed for %s: %w", segmentLabel(segment), err))
|
||||
continue
|
||||
}
|
||||
if settings.DailyGenerationCap > 0 {
|
||||
@@ -245,6 +249,14 @@ func (s *service) RunScheduled(ctx context.Context) error {
|
||||
return scheduleErr
|
||||
}
|
||||
|
||||
// segmentLabel names a library segment; "" is the shared library every mailbox draws from.
|
||||
func segmentLabel(segment string) string {
|
||||
if segment == "" {
|
||||
return "the shared library"
|
||||
}
|
||||
return fmt.Sprintf("segment %q", segment)
|
||||
}
|
||||
|
||||
func cleanMessages(in []string) []string {
|
||||
out := make([]string, 0, len(in))
|
||||
for _, m := range in {
|
||||
|
||||
@@ -272,8 +272,7 @@ func Run(ctx context.Context, name string, fn func(ctx context.Context) error) {
|
||||
}
|
||||
}
|
||||
runErr := safeRun(ctx, name, fn)
|
||||
var quiet quietError
|
||||
if runErr != nil && !errors.As(runErr, &quiet) {
|
||||
if runErr != nil && !IsQuiet(runErr) {
|
||||
errs.CaptureException(runErr)
|
||||
log.Warn().Err(runErr).Str("job", name).Msg("scheduled job failed")
|
||||
}
|
||||
@@ -284,7 +283,7 @@ func Run(ctx context.Context, name string, fn func(ctx context.Context) error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Quiet marks a standing condition a job already reported: the panel records it, but it is not logged or captured again.
|
||||
// Quiet marks an expected or already reported condition: the panel records it, but it is not logged or captured.
|
||||
func Quiet(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
@@ -292,6 +291,12 @@ func Quiet(err error) error {
|
||||
return quietError{err}
|
||||
}
|
||||
|
||||
// IsQuiet reports whether err was marked with Quiet.
|
||||
func IsQuiet(err error) bool {
|
||||
var quiet quietError
|
||||
return errors.As(err, &quiet)
|
||||
}
|
||||
|
||||
type quietError struct{ error }
|
||||
|
||||
func (q quietError) Unwrap() error { return q.error }
|
||||
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/observability/errs"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/app/warmupcontent"
|
||||
"github.com/warmbly/warmbly/internal/jobrun"
|
||||
"github.com/warmbly/warmbly/internal/repository"
|
||||
@@ -36,9 +34,9 @@ func (j *WarmupGenerationJob) Run(ctx context.Context) error {
|
||||
if j.svc == nil || j.repo == nil || !j.svc.Enabled() {
|
||||
return nil
|
||||
}
|
||||
// jobrun reports the returned error; capturing it here as well would file it twice.
|
||||
settings, err := j.repo.GetGenerationSettings(ctx)
|
||||
if err != nil {
|
||||
errs.CaptureException(err)
|
||||
return err
|
||||
}
|
||||
if settings == nil || !settings.ScheduleEnabled {
|
||||
@@ -53,7 +51,6 @@ func (j *WarmupGenerationJob) Run(ctx context.Context) error {
|
||||
}
|
||||
|
||||
if err := j.svc.RunScheduled(ctx); err != nil {
|
||||
errs.CaptureException(err)
|
||||
return err
|
||||
}
|
||||
j.lastRun = time.Now()
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
func TestLiveContentPoolRejectsDuplicateBatchContentUnderConcurrency(t *testing.T) {
|
||||
@@ -57,6 +58,26 @@ func TestLiveContentPoolRejectsDuplicateBatchContentUnderConcurrency(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveScheduledGenerationReservesOneInFlightBatchPerSegment(t *testing.T) {
|
||||
f, r, segment, _ := contentPoolFixture(t)
|
||||
t.Cleanup(func() {
|
||||
_, _ = f.pool.Exec(t.Context(), `DELETE FROM warmup_generation_jobs WHERE segment=$1`, segment)
|
||||
})
|
||||
job := func(trigger string) *models.WarmupGenerationJob {
|
||||
return &models.WarmupGenerationJob{ID: uuid.New(), Trigger: trigger, Mode: models.WarmupGenerationModeBatch,
|
||||
PoolType: "premium", Segment: segment, Model: "fixture-model", RequestedCount: 1, Status: "pending"}
|
||||
}
|
||||
if err := r.CreateGenerationJob(t.Context(), job("schedule")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := r.CreateGenerationJob(t.Context(), job("schedule")); !errors.Is(err, ErrScheduledGenerationInFlight) {
|
||||
t.Fatalf("second scheduled batch for the segment: %v", err)
|
||||
}
|
||||
if err := r.CreateGenerationJob(t.Context(), job("manual")); err != nil {
|
||||
t.Fatalf("a manual batch is not limited to one in flight: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func contentPoolFixture(t *testing.T) (*poolLinkFixture, *warmupContentRepository, string, uuid.UUID) {
|
||||
t.Helper()
|
||||
f, _ := lineageFixture(t)
|
||||
|
||||
@@ -1701,18 +1701,23 @@ func (r *warmupRepository) acceptReceipt(ctx context.Context, q interface {
|
||||
if token.ConsumedAt != nil || !token.ExpiresAt.After(time.Now()) {
|
||||
return true, nil
|
||||
}
|
||||
var lineage, awaitingSend, retired bool
|
||||
var lineage, awaitingSend, retired, settled bool
|
||||
if err = q.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM warmup_tasks WHERE task_id=$1 AND lineage_version=1),
|
||||
EXISTS(SELECT 1 FROM warmup_tasks w JOIN tasks sent ON sent.id=w.task_id
|
||||
WHERE w.task_id=$1 AND w.lineage_version=1 AND (sent.status<>'completed' OR $2='')),
|
||||
EXISTS(SELECT 1 FROM warmup_tokens WHERE token=$3 AND sent_retired_at IS NOT NULL)
|
||||
OR EXISTS(SELECT 1 FROM tasks WHERE id=$1 AND warmup_send_retired_at IS NOT NULL)`, token.TaskID, token.SentMessageID, token.Token).Scan(&lineage, &awaitingSend, &retired); err != nil {
|
||||
OR EXISTS(SELECT 1 FROM tasks WHERE id=$1 AND warmup_send_retired_at IS NOT NULL),
|
||||
EXISTS(SELECT 1 FROM tasks WHERE id=$1 AND status IN ('failed','cancelled'))`, token.TaskID, token.SentMessageID, token.Token).Scan(&lineage, &awaitingSend, &retired, &settled); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if retired {
|
||||
return true, nil
|
||||
}
|
||||
if awaitingSend {
|
||||
if settled {
|
||||
// A failed or cancelled send is never confirmed later, so it gets now what expiry would give it.
|
||||
return true, nil
|
||||
}
|
||||
return false, ErrWarmupDeliveryPending
|
||||
}
|
||||
if lineage && (strings.Trim(proof.MessageID, "<> \t") == "" ||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
"github.com/warmbly/warmbly/internal/pkg/generation"
|
||||
@@ -106,6 +107,12 @@ type warmupContentRepository struct {
|
||||
|
||||
var ErrWarmupConversationDuplicate = errors.New("warmup subject or opening is already in the conversation library")
|
||||
|
||||
// ErrScheduledGenerationInFlight: the library segment already has a scheduled batch pending or running.
|
||||
var ErrScheduledGenerationInFlight = errors.New("a scheduled warmup batch for this library segment is already in flight")
|
||||
|
||||
// scheduledGenerationInFlightIndex is the unique index from migration 000077.
|
||||
const scheduledGenerationInFlightIndex = "idx_warmup_generation_jobs_one_scheduled_inflight"
|
||||
|
||||
// NewWarmupContentRepository creates a new warmup content repository.
|
||||
func NewWarmupContentRepository(db *pgxpool.Pool) WarmupContentRepository {
|
||||
return &warmupContentRepository{db: db}
|
||||
@@ -507,6 +514,10 @@ func (r *warmupContentRepository) CreateGenerationJob(ctx context.Context, j *mo
|
||||
j.ID, j.RequestedBy, j.Trigger, j.Mode, j.PoolType, j.Segment, j.Theme, j.Model, j.RequestedCount, j.Status,
|
||||
j.BatchID, j.BatchInputFileID, j.BatchOutputFileID, j.BatchStatus, j.CompletionWindow, j.ContentVersion, j.MaxMessagesPerThread,
|
||||
)
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == scheduledGenerationInFlightIndex {
|
||||
return ErrScheduledGenerationInFlight
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user