From 06f4a270cc1bf3c2ca369eebcd4d693a335455dc Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 24 Sep 2026 17:46:46 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20keep=20the=20mailbox=20imports=20menu?= =?UTF-8?q?=20live=20and=20self-explaining=20(deferred=20rather=20than=20d?= =?UTF-8?q?ropped=20progress=20events,=20a=20five-second=20refresh=20while?= =?UTF-8?q?=20an=20import=20runs,=20each=20import's=20state=20and=20what?= =?UTF-8?q?=20it=20waits=20on=20in=20words,=20a=20badge=20for=20imports=20?= =?UTF-8?q?that=20need=20you,=20and=20an=20=C3=97=20that=20hides=20an=20im?= =?UTF-8?q?port=20for=20the=20workspace=20through=20POST=20/emails/imports?= =?UTF-8?q?/:id/dismiss=20with=20migration=20000211,=20stopping=20a=20runn?= =?UTF-8?q?ing=20one=20first),=20show=20the=20vendor's=20own=20status=20an?= =?UTF-8?q?d=20the=20Microsoft=20and=20Google=20time=20expectations=20on?= =?UTF-8?q?=20rows=20being=20authorized,=20record=20a=20vendor=20row's=20m?= =?UTF-8?q?ail=20host=20so=20it=20gets=20its=20provider=20icon=20and=20a?= =?UTF-8?q?=20working=20Sign=20in,=20drop=20the=20Fix=20button=20from=20ro?= =?UTF-8?q?ws=20the=20vendor=20is=20authorizing,=20and=20count=20warming?= =?UTF-8?q?=20mailboxes=20rather=20than=20connected=20ones=20in=20the=20po?= =?UTF-8?q?ol=20banner,=20refreshed=20when=20mailboxes=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/docs/api/endpoints.mdx | 1 + docs/content/docs/guides/mailbox-import.mdx | 6 +- internal/api/handler/mailbox_import.go | 20 ++ internal/api/routes.go | 1 + internal/app/mailboximport/retry_test.go | 14 ++ internal/app/mailboximport/runner.go | 54 ++++- internal/app/mailboximport/service.go | 24 +++ internal/app/vendorconn/authorize.go | 23 +- .../000211_mailbox_import_dismissed.down.sql | 1 + .../000211_mailbox_import_dismissed.up.sql | 2 + internal/pkg/mailvendor/inboxkit.go | 2 +- internal/pkg/mailvendor/mailvendor.go | 2 + internal/repository/pg_mailbox_import.go | 38 +++- web/src/app/app/emails/page.tsx | 2 +- .../components/app/emails/CloudPathsPanel.tsx | 23 +- .../app/emails/import/MailboxImportsMenu.tsx | 198 ++++++++++++++---- .../components/app/emails/import/RunStep.tsx | 17 +- web/src/hooks/useRealtimeEvents.ts | 4 +- .../emails/imports/dismissMailboxImport.ts | 10 + .../app/emails/useMailboxImportActions.ts | 9 + .../api/hooks/app/emails/useMailboxImports.ts | 4 +- 21 files changed, 371 insertions(+), 84 deletions(-) create mode 100644 internal/infrastructure/db/migrations/000211_mailbox_import_dismissed.down.sql create mode 100644 internal/infrastructure/db/migrations/000211_mailbox_import_dismissed.up.sql create mode 100644 web/src/lib/api/client/app/emails/imports/dismissMailboxImport.ts diff --git a/docs/content/docs/api/endpoints.mdx b/docs/content/docs/api/endpoints.mdx index 56d3d79ac..597f685a8 100644 --- a/docs/content/docs/api/endpoints.mdx +++ b/docs/content/docs/api/endpoints.mdx @@ -453,6 +453,7 @@ Alongside them, `api_url` is this API's own public base (for a copyable example - `PATCH /emails/imports/:id/rows/:line`: correct one failed row (`password`, `app_password`, `username`, `smtp`, `imap`) and queue it again - `POST /emails/imports/:id/retry`: requeue failed rows, all of them or those with one `cause` or the listed `lines`, optionally with one new `password` for all - `POST /emails/imports/:id/cancel`: stop the rows not yet started; rows already connecting finish + - `POST /emails/imports/:id/dismiss`: hide the import from `GET /emails/imports` for the workspace, stopping it first when it is still running. Its rows and history stay, and `GET /emails/imports/:id` still reads it. Answers `204`; a repeat changes nothing - `GET /emails/imports/:id/failed.csv`: every row that did not connect, as uploaded minus passwords, with `status`, `problem` and `how_to_fix` Preview and create take `multipart/form-data`: `file` (CSV, TSV or XLSX up to 10 MB) or `text` (a pasted list), `mapping` (a JSON object of column index to field, such as `{"0":"email","1":"password"}`; omit it to use the automatic mapping) and `options` (JSON: `has_header`, `shared_password`, `on_existing` of `update` or `skip`, `save_mapping`, and `settings` applied to every mailbox). One import takes up to 5,000 rows. The two lists answer `data` plus `pagination` with `next_cursor` and `has_more`; `limit` is at most `100` for imports and `200` for rows, and a cursor that is not one this API issued returns `400` with `invalid_cursor`. None of these takes an `Idempotency-Key`. Repeating a create makes a second import, whose rows find the first one's mailboxes already connected and update (or skip) them, so nothing is connected twice. A retry only requeues rows that failed, and a fix is refused for a row that is not failed, so repeating either connects nothing twice. Cancelling an import twice is a no-op. Progress is published as the realtime event `MAILBOX_IMPORT_PROGRESS`; see [realtime](/api/realtime/). Error codes are under [mailbox import refusals](/api/error-codes/#mailbox-import-refusals) diff --git a/docs/content/docs/guides/mailbox-import.mdx b/docs/content/docs/guides/mailbox-import.mdx index da9c6edbe..36726f827 100644 --- a/docs/content/docs/guides/mailbox-import.mdx +++ b/docs/content/docs/guides/mailbox-import.mdx @@ -124,7 +124,7 @@ A key that reaches several workspaces (or, at ScaledMail, organizations) lists t One vendor account lists up to 10,000 mailboxes. - **Credentials are read when each row is worked.** Picking mailboxes stores only which ones you picked. The password, app password and servers of each are fetched from the vendor at the moment its row connects, and the row is then judged exactly like a file row: Google mailboxes need the app password the vendor holds (or [an admin grant](#connect-a-whole-google-workspace-domain) over the domain), and Microsoft mailboxes connect through [a grant](#connect-a-whole-microsoft-365-organization) over their domain. -- **The vendor can authorize Warmbly for you.** When a row would otherwise wait for a sign-in and the vendor can approve apps through the admin mailbox it holds on the domain (InboxKit can), Warmbly asks it to: tenant-wide consent for Warmbly's Microsoft app, or domain-wide delegation for Warmbly's Google client. The rows show as **Authorizing** in the meantime, which usually takes a few minutes, and the import stays open until they connect on their own, with nobody signing in. One request covers every mailbox on the domain. The resulting grant covers only domains this vendor account holds, even when the vendor's Microsoft organization has others. A Google domain needs an admin mailbox listed at the vendor. When the vendor cannot authorize the domain, or has not finished within two hours, its rows fall back to **Sign in** with the reason on the row. Cancelling the import does the same for rows still waiting. +- **The vendor can authorize Warmbly for you.** When a row would otherwise wait for a sign-in and the vendor can approve apps through the admin mailbox it holds on the domain (InboxKit can), Warmbly asks it to: tenant-wide consent for Warmbly's Microsoft app, or domain-wide delegation for Warmbly's Google client. The rows show as **Authorizing** in the meantime, each with the vendor's own status for its request (for example queued or processing). Microsoft usually takes a few minutes and Google can take up to an hour. The import stays open until they connect on their own, with nobody signing in, and **Sign in** on a row connects that mailbox straight away if you would rather not wait. One request covers every mailbox on the domain. The resulting grant covers only domains this vendor account holds, even when the vendor's Microsoft organization has others. A Google domain needs an admin mailbox listed at the vendor. When the vendor cannot authorize the domain, or has not finished within two hours, its rows fall back to **Sign in** with the reason on the row. Cancelling the import does the same for rows still waiting. - **The key is sealed and never shown again.** The API key is sealed with the workspace's own encryption key. The list of connections shows each one's vendor, label, status and mailbox count, never the key. **Update** replaces the key after checking the new one with the vendor. - **Passwords the vendor rotates are picked up.** An SMTP and IMAP mailbox that came from a vendor and has stopped with an unresolved connection error is looked at every 15 minutes. When the vendor now holds a different password for it, Warmbly verifies that password against the server and reconnects the mailbox with it. Each mailbox is tried at most once every 6 hours, so a password that still fails is not retried on every pass. - **A refused key marks the connection.** A key the vendor accepts but that reaches no workspace is refused with `mailbox_vendor_no_workspace`, since there is nothing to list. When the vendor stops accepting the key, the connection shows as invalid with the vendor's reason, and rows that needed it fail as `vendor_unauthorized` until the key is updated. @@ -246,7 +246,9 @@ A setting that cannot be applied does not fail the row: the mailbox connects and ## While it runs -The import runs on the server. Closing the tab does not stop it, and everyone in the workspace who can manage mailboxes sees its progress live, with the mailbox list filling in behind it. A server restart does not lose rows either: a row being connected at the time is picked up again, and a mailbox it had already created is finished with the import's settings rather than skipped as one that was already there. **Recent imports** in the dialog reopens one later. +The import runs on the server. Closing the tab does not stop it, and everyone in the workspace who can manage mailboxes sees its progress live, with the mailbox list filling in behind it. A server restart does not lose rows either: a row being connected at the time is picked up again, and a mailbox it had already created is finished with the import's settings rather than skipped as one that was already there. **Recent imports** in the dialog, and the **Imports** button on the Accounts page, reopen one later. + +The **Imports** menu says what each import is doing: how many mailboxes it has checked so far, how many the inbox vendor is still authorizing, how many need a sign-in or failed. A badge on the button counts the imports that need you. It stays current while it is open. The **×** on an import hides it from the list for everyone in the workspace; a running import is stopped first, after a confirmation, and mailboxes it already connected stay connected. A hidden import's history is kept. Rows connect 8 at a time, and at most 3 at once against any one mail host, since providers throttle bursts of sign-ins from one address. Each credential is verified against the live server before it is saved, exactly like a single connect, so wrong credentials fail here rather than at the first send. The check runs on the worker the mailbox is going to be placed on, so the provider sees the sign-in come from the address the mailbox will keep using. When a worker does not answer, the check moves to another one, and a row whose check no worker answered is tried again a few times before it is reported as **No worker available**. diff --git a/internal/api/handler/mailbox_import.go b/internal/api/handler/mailbox_import.go index 7b9189954..08c509444 100644 --- a/internal/api/handler/mailbox_import.go +++ b/internal/api/handler/mailbox_import.go @@ -280,6 +280,26 @@ func (h *Handler) CancelMailboxImport(c *gin.Context) { c.JSON(http.StatusOK, imp) } +// DismissMailboxImport is POST /emails/imports/:id/dismiss: hides the import +// from the recent list, stopping it first when it is still running. A repeat +// changes nothing, so it needs no Idempotency-Key. +func (h *Handler) DismissMailboxImport(c *gin.Context) { + orgID, id, ok := importID(c) + if !ok { + return + } + userID, err := middleware.GetUserUUID(c) + if err != nil { + errx.Handle(c, errx.ErrUser) + return + } + if xerr := h.MailboxImportService.Dismiss(c.Request.Context(), orgID, userID, id); xerr != nil { + errx.Handle(c, xerr) + return + } + c.Status(http.StatusNoContent) +} + // DownloadMailboxImportFailures is GET /emails/imports/:id/failed.csv. func (h *Handler) DownloadMailboxImportFailures(c *gin.Context) { orgID, id, ok := importID(c) diff --git a/internal/api/routes.go b/internal/api/routes.go index 392568684..137955287 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -562,6 +562,7 @@ func Run( mailboxImports.PATCH("/:id/rows/:line", h.FixMailboxImportRow) mailboxImports.POST("/:id/retry", h.RetryMailboxImport) mailboxImports.POST("/:id/cancel", h.CancelMailboxImport) + mailboxImports.POST("/:id/dismiss", h.DismissMailboxImport) mailboxImports.GET("/:id/failed.csv", h.DownloadMailboxImportFailures) } diff --git a/internal/app/mailboximport/retry_test.go b/internal/app/mailboximport/retry_test.go index 89c9a3ec9..4f29a84d9 100644 --- a/internal/app/mailboximport/retry_test.go +++ b/internal/app/mailboximport/retry_test.go @@ -2,6 +2,7 @@ package mailboximport import ( "context" + "strings" "testing" "time" @@ -46,3 +47,16 @@ func TestRetryUnansweredRetriesOnlyASilentFleet(t *testing.T) { t.Fatalf("retried past the lease: %d calls", calls) } } + +func TestAuthorizingMessageSaysWhoWhatAndHowLong(t *testing.T) { + ms := authorizingMessage(VendorAuthorization{Pending: true, Vendor: "InboxKit", Stage: "processing"}, causeMicrosoftSignin, "acme.io") + for _, want := range []string{"InboxKit is authorizing Warmbly on acme.io", "InboxKit status: processing", "Microsoft usually takes a few minutes", "within 2 hours"} { + if !strings.Contains(ms, want) { + t.Fatalf("microsoft message %q lacks %q", ms, want) + } + } + g := authorizingMessage(VendorAuthorization{Pending: true}, causeGoogleSignin, "acme.io") + if !strings.HasPrefix(g, "Your inbox vendor is authorizing") || !strings.Contains(g, "Google can take up to an hour") || strings.Contains(g, "status:") { + t.Fatalf("google message %q", g) + } +} diff --git a/internal/app/mailboximport/runner.go b/internal/app/mailboximport/runner.go index fc9ceb6f1..9ceef1fbe 100644 --- a/internal/app/mailboximport/runner.go +++ b/internal/app/mailboximport/runner.go @@ -190,6 +190,10 @@ func (s *Service) process(ctx context.Context, w repository.ImportWorkRow) { // A vendor row learns its credentials only now, and is judged like a file row. if p.VendorConnectionID != nil && p.SMTP == nil && !p.Signin && p.GrantID == nil { resolved, cause, problem := s.resolveVendorRow(rowCtx, w, p) + // The row learns its host only now; recorded so it shows the provider and offers Sign in. + if resolved.MailHost != "" && resolved.MailHost != w.MailHost { + _ = s.repo.SetRowMailHost(ctx, w.ImportID, w.Line, resolved.MailHost) + } if cause == causeMicrosoftSignin || cause == causeGoogleSignin { auth := s.authorizeVendorDomain(rowCtx, w, *p.VendorConnectionID, cause) switch { @@ -198,8 +202,7 @@ func (s *Service) process(ctx context.Context, w repository.ImportWorkRow) { resolved.SMTP, resolved.IMAP = nil, nil cause = "" case auth.Pending: - s.finish(ctx, w, models.ImportRowNeedsSignin, cause, causeVendorAuthorizing, - "Your inbox vendor is authorizing Warmbly on "+domainOf(w.Email)+". This mailbox connects on its own when it finishes.", nil, true) + s.finish(ctx, w, models.ImportRowNeedsSignin, cause, causeVendorAuthorizing, authorizingMessage(auth, cause, domainOf(w.Email)), nil, true) return case auth.Message != "": problem = auth.Message @@ -391,6 +394,24 @@ func (s *Service) authorizeVendorDomain(ctx context.Context, w repository.Import return az.AuthorizeDomain(ctx, w.OrgID, *w.CreatedBy, connectionID, w.Email, provider) } +// authorizingMessage tells the person watching a parked row who is doing what, and how long it can take. +func authorizingMessage(auth VendorAuthorization, signinCause, domain string) string { + vendor := auth.Vendor + if vendor == "" { + vendor = "Your inbox vendor" + } + msg := vendor + " is authorizing Warmbly on " + domain + if auth.Stage != "" { + msg += " (" + vendor + " status: " + auth.Stage + ")" + } + if signinCause == causeGoogleSignin { + msg += ". Google can take up to an hour to apply it." + } else { + msg += ". Microsoft usually takes a few minutes." + } + return msg + " The mailbox connects on its own, and switches to Sign in if this is not done within 2 hours." +} + // resumeVendorAuthorizations requeues rows parked on a vendor authorization once // it has an answer, asking the vendor once per domain. func (s *Service) resumeVendorAuthorizations(ctx context.Context) { @@ -405,7 +426,7 @@ func (s *Service) resumeVendorAuthorizations(ctx context.Context) { org, conn uuid.UUID signin, domain string } - settled := map[domainKey]bool{} + settled := map[domainKey]VendorAuthorization{} resumed := false var waiting []repository.ImportWorkRow for _, w := range rows { @@ -418,13 +439,16 @@ func (s *Service) resumeVendorAuthorizations(ctx context.Context) { continue } k := domainKey{w.OrgID, *p.VendorConnectionID, w.Code, domainOf(w.Email)} - done, seen := settled[k] + auth, seen := settled[k] if !seen { - done = !s.authorizeVendorDomain(ctx, w, *p.VendorConnectionID, w.Code).Pending - settled[k] = done + auth = s.authorizeVendorDomain(ctx, w, *p.VendorConnectionID, w.Code) + settled[k] = auth } - if !done { + if auth.Pending { waiting = append(waiting, w) + if changed, err := s.repo.SetParkedMessage(ctx, w.ImportID, w.Line, causeVendorAuthorizing, authorizingMessage(auth, w.Code, domainOf(w.Email))); err == nil && changed { + s.publish(ctx, w.OrgID, w.ImportID, models.ImportRunning, false) + } continue } if err := s.repo.ResumeParked(ctx, w.ImportID, w.Line, causeVendorAuthorizing); err == nil { @@ -722,15 +746,25 @@ func (s *Service) classifyHosts(ctx context.Context) { } } -// publish tells the workspace an import moved, at most once a second per import unless final. +// publish tells the workspace an import moved, at most once a second per import unless final; +// an update inside the second is sent at its end. func (s *Service) publish(ctx context.Context, orgID, importID uuid.UUID, status string, force bool) { if s.publisher == nil { return } now := time.Now() if !force { - if last, ok := s.progress.Load(importID); ok && now.Sub(last.(time.Time)) < time.Second { - return + if last, ok := s.progress.Load(importID); ok { + if wait := time.Second - now.Sub(last.(time.Time)); wait > 0 { + // Deferred, not dropped: the last update of a burst is the one a watcher needs. + if _, pending := s.trailing.LoadOrStore(importID, true); !pending { + time.AfterFunc(wait, func() { + s.trailing.Delete(importID) + s.publish(context.WithoutCancel(ctx), orgID, importID, status, true) + }) + } + return + } } } s.progress.Store(importID, now) diff --git a/internal/app/mailboximport/service.go b/internal/app/mailboximport/service.go index e12d1076b..9749c2cb1 100644 --- a/internal/app/mailboximport/service.go +++ b/internal/app/mailboximport/service.go @@ -92,6 +92,9 @@ type VendorAuthorization struct { Pending bool // Message says why the vendor could not authorize the domain. Message string + // Vendor is the vendor's name and Stage its own word for where a pending request is. + Vendor string + Stage string } // VendorAuthorizer is a VendorSource that can have the vendor authorize this @@ -142,6 +145,7 @@ type Service struct { kick chan struct{} progress sync.Map // import id -> time.Time of the last progress event + trailing sync.Map // import id -> a progress event deferred to the end of its window causes sync.Map // scrubbed server reply -> refined cause key } @@ -849,6 +853,26 @@ func (s *Service) Cancel(ctx context.Context, orgID, userID, id uuid.UUID) (*mod return s.Get(ctx, orgID, id) } +// Dismiss hides an import from the recent list for the whole workspace; a running one is stopped first. +func (s *Service) Dismiss(ctx context.Context, orgID, userID, id uuid.UUID) *errx.Error { + imp, xerr := s.Get(ctx, orgID, id) + if xerr != nil { + return xerr + } + if imp.Status == models.ImportRunning { + if err := s.repo.Cancel(ctx, orgID, id); err != nil { + return errx.InternalError() + } + s.audit(ctx, orgID, userID, models.AuditActionUpdate, id, map[string]string{"status": models.ImportCancelled}) + } + if err := s.repo.Dismiss(ctx, orgID, id); err != nil { + return errx.InternalError() + } + s.audit(ctx, orgID, userID, models.AuditActionUpdate, id, map[string]string{"dismissed": "true"}) + s.publish(ctx, orgID, id, models.ImportCancelled, true) + return nil +} + // FailedCSV is every row that did not connect, as uploaded minus secrets, with the reason and the fix. func (s *Service) FailedCSV(ctx context.Context, orgID, id uuid.UUID) ([]byte, string, *errx.Error) { imp, xerr := s.Get(ctx, orgID, id) diff --git a/internal/app/vendorconn/authorize.go b/internal/app/vendorconn/authorize.go index 8b3c992a2..71373ac7c 100644 --- a/internal/app/vendorconn/authorize.go +++ b/internal/app/vendorconn/authorize.go @@ -23,6 +23,7 @@ type authState struct { RequestID string `json:"request_id,omitempty"` Failed string `json:"failed,omitempty"` Started time.Time `json:"started,omitempty"` + Stage string `json:"stage,omitempty"` Completed time.Time `json:"completed,omitempty"` } @@ -83,12 +84,12 @@ func (s *Service) AuthorizeDomain(ctx context.Context, orgID, userID, connection case st.Failed != "": return DomainAuthorization{Message: st.Failed} case st.RequestID == "": - return DomainAuthorization{Pending: true} + return DomainAuthorization{Pending: true, Vendor: label} } status, err := az.AuthorizationStatus(ctx, st.RequestID) switch { case err != nil && transient(err): - status = mailvendor.AuthorizationStatus{State: mailvendor.AuthorizationPending} + status = mailvendor.AuthorizationStatus{State: mailvendor.AuthorizationPending, Stage: st.Stage} case errors.Is(err, mailvendor.ErrUnauthorized): return DomainAuthorization{Message: s.failed(ctx, c, err).Message} case err != nil: @@ -99,7 +100,11 @@ func (s *Service) AuthorizeDomain(ctx context.Context, orgID, userID, connection } switch status.State { case mailvendor.AuthorizationPending: - return DomainAuthorization{Pending: true} + if status.Stage != st.Stage { + st.Stage = status.Stage + _ = s.cache.SetJSON(ctx, key, st, time.Until(st.Started.Add(authPendingTTL))) + } + return DomainAuthorization{Pending: true, Vendor: label, Stage: status.Stage} case mailvendor.AuthorizationFailed: msg := label + " could not authorize Warmbly on " + domain + reasonSuffix(status.Reason) + ". Sign in on each mailbox instead, or retry these rows once it is fixed." _ = s.cache.SetJSON(ctx, key, authState{Failed: msg}, authFailedTTL) @@ -117,14 +122,14 @@ func (s *Service) AuthorizeDomain(ctx context.Context, orgID, userID, connection return DomainAuthorization{} } if !started { - return DomainAuthorization{Pending: true} + return DomainAuthorization{Pending: true, Vendor: label} } list, err := s.listed(ctx, c, client) if err != nil { _ = s.cache.Del(ctx, key) switch { case transient(err): - return DomainAuthorization{Pending: true} + return DomainAuthorization{Pending: true, Vendor: label} case errors.Is(err, mailvendor.ErrUnauthorized): return DomainAuthorization{Message: s.failed(ctx, c, err).Message} } @@ -152,7 +157,7 @@ func (s *Service) AuthorizeDomain(ctx context.Context, orgID, userID, connection if err != nil { if transient(err) { _ = s.cache.Del(ctx, key) - return DomainAuthorization{Pending: true} + return DomainAuthorization{Pending: true, Vendor: label} } if errors.Is(err, mailvendor.ErrUnauthorized) { _ = s.failed(ctx, c, err) @@ -163,7 +168,7 @@ func (s *Service) AuthorizeDomain(ctx context.Context, orgID, userID, connection return DomainAuthorization{Message: msg} } _ = s.cache.SetJSON(ctx, key, authState{RequestID: reqID, Started: time.Now()}, authPendingTTL) - return DomainAuthorization{Pending: true} + return DomainAuthorization{Pending: true, Vendor: label, Stage: "queued"} } // recordGrant turns a completed vendor authorization into the workspace's grant, @@ -172,7 +177,7 @@ func (s *Service) AuthorizeDomain(ctx context.Context, orgID, userID, connection func (s *Service) recordGrant(ctx context.Context, c *models.VendorConnection, client mailvendor.Client, key string, orgID, userID uuid.UUID, provider, domain string, settling bool) DomainAuthorization { list, err := s.listed(ctx, c, client) if err != nil { - return DomainAuthorization{Pending: true} + return DomainAuthorization{Pending: true, Vendor: labelOf(c.Vendor)} } owned := map[string]bool{} admin := "" @@ -195,7 +200,7 @@ func (s *Service) recordGrant(ctx context.Context, c *models.VendorConnection, c g, xerr := s.grants.GrantFromVendor(ctx, orgID, userID, provider, domain, admin, domains) if xerr != nil { if settling || xerr.Code == errx.Internal || xerr.ResponseCode() == "mailbox_grant_unavailable" { - return DomainAuthorization{Pending: true} + return DomainAuthorization{Pending: true, Vendor: labelOf(c.Vendor)} } msg := labelOf(c.Vendor) + " authorized Warmbly on " + domain + ", but the grant did not verify: " + xerr.Message _ = s.cache.SetJSON(ctx, key, authState{Failed: msg}, authFailedTTL) diff --git a/internal/infrastructure/db/migrations/000211_mailbox_import_dismissed.down.sql b/internal/infrastructure/db/migrations/000211_mailbox_import_dismissed.down.sql new file mode 100644 index 000000000..d26a604db --- /dev/null +++ b/internal/infrastructure/db/migrations/000211_mailbox_import_dismissed.down.sql @@ -0,0 +1 @@ +ALTER TABLE mailbox_imports DROP COLUMN IF EXISTS dismissed_at; diff --git a/internal/infrastructure/db/migrations/000211_mailbox_import_dismissed.up.sql b/internal/infrastructure/db/migrations/000211_mailbox_import_dismissed.up.sql new file mode 100644 index 000000000..edeb34ca7 --- /dev/null +++ b/internal/infrastructure/db/migrations/000211_mailbox_import_dismissed.up.sql @@ -0,0 +1,2 @@ +-- A dismissed import is hidden from the workspace's recent list; its rows and history stay. +ALTER TABLE mailbox_imports ADD COLUMN dismissed_at timestamp with time zone; diff --git a/internal/pkg/mailvendor/inboxkit.go b/internal/pkg/mailvendor/inboxkit.go index f630ba802..e096ac179 100644 --- a/internal/pkg/mailvendor/inboxkit.go +++ b/internal/pkg/mailvendor/inboxkit.go @@ -247,7 +247,7 @@ func (c *inboxKit) AuthorizationStatus(ctx context.Context, requestID string) (A } return AuthorizationStatus{State: AuthorizationFailed, Reason: reason}, nil } - return AuthorizationStatus{State: AuthorizationPending}, nil + return AuthorizationStatus{State: AuthorizationPending, Stage: strings.ToLower(strings.TrimSpace(res.Data.Status))}, nil } const inboxKitDomainPageSize = 100 diff --git a/internal/pkg/mailvendor/mailvendor.go b/internal/pkg/mailvendor/mailvendor.go index cbf222cae..d024860c5 100644 --- a/internal/pkg/mailvendor/mailvendor.go +++ b/internal/pkg/mailvendor/mailvendor.go @@ -144,6 +144,8 @@ const ( type AuthorizationStatus struct { State string Reason string + // Stage is the vendor's own word for where the request is, for showing to a person. + Stage string } // AppAuthorizer is a vendor that can authorize an app on a domain it administers. diff --git a/internal/repository/pg_mailbox_import.go b/internal/repository/pg_mailbox_import.go index c9f6e31ff..c0187bbde 100644 --- a/internal/repository/pg_mailbox_import.go +++ b/internal/repository/pg_mailbox_import.go @@ -103,6 +103,12 @@ type MailboxImportRepository interface { PendingSignins(ctx context.Context, limit int) ([]PendingSignin, error) // ParkedRows lists rows waiting on sign-in under one cause that still hold credentials. ParkedRows(ctx context.Context, cause string, limit int) ([]ImportWorkRow, error) + // SetRowMailHost records the host a vendor row resolved to. + SetRowMailHost(ctx context.Context, id uuid.UUID, line int, mailHost string) error + // SetParkedMessage rewrites a parked row's message; false when it already said that. + SetParkedMessage(ctx context.Context, id uuid.UUID, line int, cause, message string) (bool, error) + // Dismiss hides an import from List. + Dismiss(ctx context.Context, orgID, id uuid.UUID) error // TouchParked moves rows still waiting to the back of ParkedRows, so no import holds the others back. TouchParked(ctx context.Context, cause string, rows []ImportWorkRow) error // ResumeParked queues a parked row again and reopens its import when it had completed. @@ -249,7 +255,7 @@ func (r *mailboxImportRepository) fillCounts(ctx context.Context, imp *models.Ma func (r *mailboxImportRepository) List(ctx context.Context, orgID uuid.UUID, before *time.Time, beforeID *uuid.UUID, limit int) ([]models.MailboxImport, error) { query := `SELECT ` + importColumns + ` FROM mailbox_imports - WHERE organization_id = $1 AND ($2::timestamptz IS NULL OR (created_at, id) < ($2, $3)) + WHERE organization_id = $1 AND dismissed_at IS NULL AND ($2::timestamptz IS NULL OR (created_at, id) < ($2, $3)) ORDER BY created_at DESC, id DESC LIMIT $4` rows, err := r.DB.Query(ctx, query, orgID, before, beforeID, limit) @@ -663,6 +669,36 @@ func (r *mailboxImportRepository) ParkedRows(ctx context.Context, cause string, return out, rows.Err() } +func (r *mailboxImportRepository) SetRowMailHost(ctx context.Context, id uuid.UUID, line int, mailHost string) error { + query := `UPDATE mailbox_import_rows SET mail_host = $3 WHERE import_id = $1 AND line = $2` + if _, err := r.DB.Exec(ctx, query, id, line, mailHost); err != nil { + db.CaptureError(err, query, nil, "exec") + return err + } + return nil +} + +func (r *mailboxImportRepository) SetParkedMessage(ctx context.Context, id uuid.UUID, line int, cause, message string) (bool, error) { + query := `UPDATE mailbox_import_rows SET message = $4, updated_at = now() + WHERE import_id = $1 AND line = $2 AND status = 'needs_signin' AND cause = $3 AND message <> $4` + tag, err := r.DB.Exec(ctx, query, id, line, cause, message) + if err != nil { + db.CaptureError(err, query, nil, "exec") + return false, err + } + return tag.RowsAffected() > 0, nil +} + +func (r *mailboxImportRepository) Dismiss(ctx context.Context, orgID, id uuid.UUID) error { + query := `UPDATE mailbox_imports SET dismissed_at = now(), updated_at = now() + WHERE organization_id = $1 AND id = $2 AND dismissed_at IS NULL` + if _, err := r.DB.Exec(ctx, query, orgID, id); err != nil { + db.CaptureError(err, query, nil, "exec") + return err + } + return nil +} + func (r *mailboxImportRepository) TouchParked(ctx context.Context, cause string, rows []ImportWorkRow) error { if len(rows) == 0 { return nil diff --git a/web/src/app/app/emails/page.tsx b/web/src/app/app/emails/page.tsx index cc3ccb907..1b0b50d2d 100644 --- a/web/src/app/app/emails/page.tsx +++ b/web/src/app/app/emails/page.tsx @@ -368,7 +368,7 @@ export default function AddressesPage() { openMigration()} /> setCloudDialog(true)} mailboxCount={stats.total} /> - {!emailsData.isLoading && p?.setAddEmail(true)} />} + {!emailsData.isLoading && p?.setAddEmail(true)} />} {/* Hosted, the pool is thousands of mailboxes: the pool-size advice is self-host only. */} {cloud.selfHosted && ( void }) { +export default function CloudPathsPanel({ + mailboxCount, + warmingCount, + onAdd, +}: { + mailboxCount: number; + /** Mailboxes with warmup on and not paused; connected is not the same as warming. */ + warmingCount: number; + onAdd: () => void; +}) { const authConfig = useAuthConfig(); const access = useFeatureAccess(); const hosted = authConfig.data?.self_hosted === false; @@ -89,12 +98,14 @@ export default function CloudPathsPanel({ mailboxCount, onAdd }: { mailboxCount: - {free - ? `${used} of ${allowance} free mailboxes used.` - : onWarmupPlan - ? `${used} mailboxes warming in the premium pool.` - : `${used} mailboxes warming in the pool.`} + {free ? `${used} of ${allowance} free mailboxes used. ` : ""} + {warmingCount === 0 + ? "No mailbox is warming yet." + : `${warmingCount} of ${mailboxCount} mailbox${mailboxCount === 1 ? "" : "es"} warming in the ${onWarmupPlan ? "premium " : ""}pool.`} + {warmingCount === 0 && mailboxCount > 0 && ( + Turn on warmup from a mailbox's Warmup tab, or select several and start it for all. + )} {(linkedLabel || (free && canBuy)) && ( {linkedLabel} diff --git a/web/src/components/app/emails/import/MailboxImportsMenu.tsx b/web/src/components/app/emails/import/MailboxImportsMenu.tsx index 21de56147..d1d72d224 100644 --- a/web/src/components/app/emails/import/MailboxImportsMenu.tsx +++ b/web/src/components/app/emails/import/MailboxImportsMenu.tsx @@ -1,41 +1,97 @@ // MailboxImportsMenu: the mailboxes page's way back into a running or recent -// import. Hidden until the workspace has one. +// import. Hidden until the workspace has one. Each entry says what the import +// is doing or waiting on, and can be hidden from the list. import React from "react"; -import { FileSpreadsheetIcon, Loader2Icon } from "lucide-react"; +import toast from "react-hot-toast"; +import { FileSpreadsheetIcon, Loader2Icon, XIcon } from "lucide-react"; import { PopoverMenu, PopoverMenuContent, - PopoverMenuItem, PopoverMenuLabel, PopoverMenuTrigger, } from "@/components/ui/popover-menu"; import useMailboxImports from "@/lib/api/hooks/app/emails/useMailboxImports"; -import { importDone, type MailboxImport } from "@/lib/api/models/app/emails/MailboxImport"; +import { useDismissMailboxImport } from "@/lib/api/hooks/app/emails/useMailboxImportActions"; +import { type MailboxImport } from "@/lib/api/models/app/emails/MailboxImport"; +import { vendorLabel } from "@/lib/api/models/app/emails/MailboxSources"; +import { useConfirm } from "@/hooks/context/confirm"; +import type { AppError } from "@/lib/api/client/normalizeError"; +import buildError from "@/lib/helper/buildError"; import timeAgo from "@/lib/helper/timeAgo"; import { cn } from "@/lib/utils"; -import { importSourceName } from "./importFields"; +import { VENDOR_AUTHORIZING, importSourceName, plural } from "./importFields"; import MailboxImportDialog from "./MailboxImportDialog"; import ProviderLogo from "@/components/app/emails/ProviderLogo"; -function jobState(job: MailboxImport): { text: string; cls: string } { - const c = job.counts; - if (job.status === "running") return { text: `${importDone(c).toLocaleString()}/${job.total.toLocaleString()}`, cls: "text-sky-600" }; - if (job.status === "cancelled") return { text: "Stopped", cls: "text-slate-400" }; - if (c.failed > 0) return { text: `${c.failed.toLocaleString()} failed`, cls: "text-red-600" }; - if (c.needs_signin > 0) return { text: `${c.needs_signin.toLocaleString()} to sign in`, cls: "text-sky-600" }; - return { text: "Done", cls: "text-emerald-600" }; +type Tone = "working" | "waiting" | "action" | "error" | "done" | "muted"; + +interface JobState { + text: string; + hint?: string; + tone: Tone; } +// jobState says, in words, what an import is doing now and what it waits on. +function jobState(job: MailboxImport): JobState { + const c = job.counts; + const authorizing = job.causes?.find((x) => x.cause === VENDOR_AUTHORIZING)?.count ?? 0; + const signin = Math.max(0, c.needs_signin - authorizing); + const inFlight = c.queued + c.running; + const settled = job.total - inFlight; + + if (job.status === "cancelled") return { text: "Stopped", tone: "muted" }; + if (job.status === "running" && inFlight > 0) { + return { + text: `Connecting ${settled.toLocaleString()} of ${job.total.toLocaleString()}`, + hint: "Each mailbox is checked against its mail server, a few seconds each.", + tone: "working", + }; + } + if (authorizing > 0) { + return { + text: `Authorizing ${plural(authorizing, "mailbox", "mailboxes")}`, + hint: `${vendorLabel(job.vendor) || "The inbox vendor"} is approving Warmbly, usually within a few minutes. They connect on their own.`, + tone: "waiting", + }; + } + if (c.failed > 0) { + return { text: `${c.failed.toLocaleString()} failed`, hint: "Open to see why and retry.", tone: "error" }; + } + if (signin > 0) { + return { + text: `${plural(signin, "mailbox needs", "mailboxes need")} sign-in`, + hint: "Open to sign in to each one.", + tone: "action", + }; + } + const ok = c.connected + c.updated; + return { text: ok > 0 ? `${ok.toLocaleString()} connected` : "Done", tone: "done" }; +} + +const TONE: Record = { + working: "text-sky-700", + waiting: "text-sky-700", + action: "text-amber-700", + error: "text-red-600", + done: "text-emerald-700", + muted: "text-slate-500", +}; + export default function MailboxImportsMenu() { const imports = useMailboxImports(); const [openId, setOpenId] = React.useState(null); + const [menuOpen, setMenuOpen] = React.useState(false); const jobs = imports.data?.data ?? []; const running = jobs.filter((j) => j.status === "running").length; + const needsYou = jobs.filter((j) => { + const t = jobState(j).tone; + return t === "action" || t === "error"; + }).length; return ( <> {jobs.length > 0 && ( - + - + Recent imports - {jobs.map((job) => { - const st = jobState(job); - return ( - + {jobs.map((job) => ( + setOpenId(job.id)} - icon={ - - } - trailing={ - - {st.text} - {timeAgo(job.created_at)} - - } - > - - {job.vendor && } - - {importSourceName(job)} · {job.total.toLocaleString()} - - - - ); - })} + job={job} + onOpen={() => { + setMenuOpen(false); + setOpenId(job.id); + }} + /> + ))} + )} @@ -89,3 +135,69 @@ export default function MailboxImportsMenu() { ); } + +function ImportEntry({ job, onOpen }: { job: MailboxImport; onOpen: () => void }) { + const confirm = useConfirm(); + const dismiss = useDismissMailboxImport(); + const st = jobState(job); + const live = job.status === "running"; + + const hide = (e: React.MouseEvent) => { + e.stopPropagation(); + const run = async () => { + try { + await dismiss.mutateAsync(job.id); + toast.success(live ? "Import stopped and hidden" : "Import hidden"); + } catch (err) { + toast.error(buildError(err as AppError)); + } + }; + if (live) { + confirm.show("Stop this import and hide it? Mailboxes it already connected stay connected.", run); + return; + } + void run(); + }; + + return ( +
+ + +
+ ); +} diff --git a/web/src/components/app/emails/import/RunStep.tsx b/web/src/components/app/emails/import/RunStep.tsx index 08feb08a4..19407352f 100644 --- a/web/src/components/app/emails/import/RunStep.tsx +++ b/web/src/components/app/emails/import/RunStep.tsx @@ -301,7 +301,7 @@ export default function RunStep({ onClick={() => applyFilter({ status: filter.status === "connected" ? "" : "connected", cause: "" })} /> 0 ? "Authorizing" : "Needs sign-in"} value={c.needs_signin} accent={c.needs_signin > 0 ? "sky" : "slate"} active={filter.status === "needs_signin"} @@ -362,8 +362,10 @@ export default function RunStep({ {plural(authorizing, "mailbox", "mailboxes")}

- It approves Warmbly through the admin mailbox it holds on each domain, so nobody has to sign in. This - usually takes a few minutes, and the rows connect on their own. You can close this window. + It approves Warmbly through the admin mailbox it holds on each domain, so nobody has to sign in. Microsoft + usually takes a few minutes and Google can take up to an hour; each row shows where its request is. The rows + connect on their own, and switch to Sign in if it is not done within 2 hours. You can close this window, or + use Sign in on a row now instead of waiting.

@@ -495,14 +497,13 @@ export default function RunStep({
No rows here.
) : ( page.data.map((row) => { - const st = - row.status === "needs_signin" && row.cause === VENDOR_AUTHORIZING - ? AUTHORIZING_STATUS - : (ROW_STATUS[row.status] ?? ROW_STATUS.failed); + // A row the vendor is authorizing connects on its own; nothing on it to fix yet. + const vendorAuthorizing = row.status === "needs_signin" && row.cause === VENDOR_AUTHORIZING; + const st = vendorAuthorizing ? AUTHORIZING_STATUS : (ROW_STATUS[row.status] ?? ROW_STATUS.failed); const provider = signInProvider(row); const wantsSignIn = row.status === "needs_signin" || (row.status === "failed" && isSigninCause(row.cause)); const canSignIn = wantsSignIn && !!provider; - const canFix = row.status === "failed" || (row.status === "needs_signin" && !provider); + const canFix = row.status === "failed" || (row.status === "needs_signin" && !provider && !vendorAuthorizing); return (
diff --git a/web/src/hooks/useRealtimeEvents.ts b/web/src/hooks/useRealtimeEvents.ts index c48d502d7..6d123c935 100644 --- a/web/src/hooks/useRealtimeEvents.ts +++ b/web/src/hooks/useRealtimeEvents.ts @@ -91,7 +91,7 @@ export function useRealtimeEvents() { if (event === 'MAILBOX_IMPORT_PROGRESS') { const importId = getString('import_id') // A finished row can move a mailbox off Google sign-in. - invalidate([['emails', 'imports'], ['emails', 'list'], ['sending-domains'], ['mailbox-grants', 'migration']]) + invalidate([['emails', 'imports'], ['emails', 'list'], ['sending-domains'], ['mailbox-grants', 'migration'], ['pool-link']]) if (importId) invalidate([['emails', 'imports', importId]]) return } @@ -418,7 +418,7 @@ export function useRealtimeEvents() { // rolled plan), so a teammate retuning a mailbox's sending behaviour // refreshes everyone's open drawer instead of only the list row. // A mailbox write can move it off Google sign-in, so the migration list follows. - email_account: [['emails'], ['analytics', 'accounts'], ['sending-domains'], ['mailbox-grants', 'migration']], + email_account: [['emails'], ['analytics', 'accounts'], ['sending-domains'], ['mailbox-grants', 'migration'], ['pool-link']], // A mailbox import created, retried or cancelled by a teammate; it may move mailboxes onto a grant. mailbox_import: [['emails', 'imports'], ['mailbox-grants', 'migration']], // An admin grant added, re-checked or removed, and an inbox vendor diff --git a/web/src/lib/api/client/app/emails/imports/dismissMailboxImport.ts b/web/src/lib/api/client/app/emails/imports/dismissMailboxImport.ts new file mode 100644 index 000000000..2fe1fd2ba --- /dev/null +++ b/web/src/lib/api/client/app/emails/imports/dismissMailboxImport.ts @@ -0,0 +1,10 @@ +import Request from "../../../Request"; + +// Hides an import from the recent list for the workspace, stopping it first when it is running. +export default async function dismissMailboxImport(id: string): Promise { + await Request({ + method: "POST", + url: `/emails/imports/${id}/dismiss`, + authorization: true, + }); +} diff --git a/web/src/lib/api/hooks/app/emails/useMailboxImportActions.ts b/web/src/lib/api/hooks/app/emails/useMailboxImportActions.ts index 523503693..7fdd2a8dd 100644 --- a/web/src/lib/api/hooks/app/emails/useMailboxImportActions.ts +++ b/web/src/lib/api/hooks/app/emails/useMailboxImportActions.ts @@ -4,6 +4,7 @@ import createMailboxImport from "@/lib/api/client/app/emails/imports/createMailb import fixMailboxImportRow from "@/lib/api/client/app/emails/imports/fixMailboxImportRow"; import retryMailboxImport from "@/lib/api/client/app/emails/imports/retryMailboxImport"; import cancelMailboxImport from "@/lib/api/client/app/emails/imports/cancelMailboxImport"; +import dismissMailboxImport from "@/lib/api/client/app/emails/imports/dismissMailboxImport"; import type { MailboxImport, MailboxImportInput, @@ -50,6 +51,14 @@ export function useCancelMailboxImport(id: string | null) { return useJobMutation(id, (i) => cancelMailboxImport(i)); } +export function useDismissMailboxImport() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (id: string) => dismissMailboxImport(id), + onSuccess: () => qc.invalidateQueries({ queryKey: ["emails", "imports"] }), + }); +} + export function useFixMailboxImportRow(id: string | null) { const qc = useQueryClient(); return useMutation({ diff --git a/web/src/lib/api/hooks/app/emails/useMailboxImports.ts b/web/src/lib/api/hooks/app/emails/useMailboxImports.ts index 7b25cd17f..bc64fae90 100644 --- a/web/src/lib/api/hooks/app/emails/useMailboxImports.ts +++ b/web/src/lib/api/hooks/app/emails/useMailboxImports.ts @@ -3,12 +3,14 @@ import listMailboxImports from "@/lib/api/client/app/emails/imports/listMailboxI export const MAILBOX_IMPORTS_KEY = ["emails", "imports"] as const; -// Recent imports for the mailboxes page; MAILBOX_IMPORT_PROGRESS keeps it live. +// Recent imports for the mailboxes page; MAILBOX_IMPORT_PROGRESS keeps it live, +// and a running import is also re-read every few seconds in case the socket is down. export default function useMailboxImports(enabled = true, limit = 10) { return useQuery({ queryKey: [...MAILBOX_IMPORTS_KEY, "list", limit], queryFn: () => listMailboxImports({ limit }), enabled, staleTime: 15_000, + refetchInterval: (q) => (q.state.data?.data?.some((j) => j.status === "running") ? 5_000 : false), }); }