mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-21 00:06:38 +00:00
470 lines
14 KiB
Go
470 lines
14 KiB
Go
package toolmigrate
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/rs/zerolog/log"
|
|
|
|
"github.com/warmbly/warmbly/internal/errx"
|
|
"github.com/warmbly/warmbly/internal/jobrun"
|
|
"github.com/warmbly/warmbly/internal/models"
|
|
"github.com/warmbly/warmbly/internal/observability/errs"
|
|
source "github.com/warmbly/warmbly/internal/pkg/outreachsource"
|
|
"github.com/warmbly/warmbly/internal/repository"
|
|
)
|
|
|
|
// Runner pacing. A lease outlives one provider page with its retries.
|
|
const (
|
|
leaseDuration = 3 * time.Minute
|
|
maxAttempts = 5
|
|
idleWindow = 7 * 24 * time.Hour
|
|
maxNotes = 200
|
|
)
|
|
|
|
// Kick wakes the runner now instead of at its next tick.
|
|
func (s *Service) Kick() {
|
|
select {
|
|
case s.kick <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Run scans and applies migrations until ctx ends, plus the upkeep sweep. Work
|
|
// interrupted by a shutdown resumes from its last checkpoint on the next pass.
|
|
func (s *Service) Run(ctx context.Context) {
|
|
go jobrun.Loop(ctx, "tool_migration_upkeep", time.Hour, true, func(ctx context.Context) error {
|
|
_, err := s.d.Repo.Upkeep(ctx, idleWindow, maxAttempts)
|
|
return err
|
|
})
|
|
ticker := time.NewTicker(5 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
s.pass(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
case <-s.kick:
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Service) pass(ctx context.Context) {
|
|
for ctx.Err() == nil {
|
|
job, err := s.d.Repo.ClaimScan(ctx, leaseDuration)
|
|
if err != nil || job == nil {
|
|
break
|
|
}
|
|
held, release := s.hold(ctx, func(c context.Context) (bool, error) {
|
|
return s.d.Repo.RenewScan(c, job.ID, job.Token, leaseDuration)
|
|
})
|
|
s.scan(held, job)
|
|
release()
|
|
}
|
|
for ctx.Err() == nil {
|
|
job, err := s.d.Repo.ClaimTarget(ctx, leaseDuration)
|
|
if err != nil || job == nil {
|
|
return
|
|
}
|
|
held, release := s.hold(ctx, func(c context.Context) (bool, error) {
|
|
return s.d.Repo.RenewTarget(c, job.ID, job.Token, leaseDuration)
|
|
})
|
|
s.runTarget(held, job)
|
|
release()
|
|
}
|
|
}
|
|
|
|
// hold renews a lease while its work runs. The returned context ends when the
|
|
// lease is lost, so provider calls stop; writes are refused by the token anyway.
|
|
func (s *Service) hold(parent context.Context, renew func(context.Context) (bool, error)) (context.Context, func()) {
|
|
ctx, cancel := context.WithCancel(parent)
|
|
done := make(chan struct{})
|
|
go func() {
|
|
t := time.NewTicker(s.renewEvery)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
// A failed renewal is retried; only a refusal means another holder has it.
|
|
if held, err := renew(context.WithoutCancel(ctx)); err == nil && !held {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
return ctx, func() {
|
|
close(done)
|
|
cancel()
|
|
}
|
|
}
|
|
|
|
// openCredentials decrypts a migration's credential sets.
|
|
func (s *Service) openCredentials(sealed string) ([]source.Credentials, error) {
|
|
raw, err := s.d.Sealer.Decrypt(sealed)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var creds []source.Credentials
|
|
if err := json.Unmarshal([]byte(raw), &creds); err != nil {
|
|
return nil, err
|
|
}
|
|
return creds, nil
|
|
}
|
|
|
|
// scan reads every source workspace each key can see and what it holds.
|
|
func (s *Service) scan(ctx context.Context, job *repository.ToolMigrationScanJob) {
|
|
work := context.WithoutCancel(ctx)
|
|
failScan := func(msg string) {
|
|
_, _ = s.d.Repo.FinishScan(work, job.ID, job.Token, nil, models.ToolMigrationFailed, msg)
|
|
}
|
|
if job.Attempts > maxAttempts {
|
|
failScan("Reading the other tool stopped unexpectedly several times. Start a new migration.")
|
|
return
|
|
}
|
|
p, ok := s.d.Registry.Get(source.ProviderID(job.Provider))
|
|
if !ok {
|
|
failScan("Warmbly no longer migrates from this tool.")
|
|
return
|
|
}
|
|
creds, err := s.openCredentials(job.Sealed)
|
|
if err != nil {
|
|
errs.CaptureException(err)
|
|
failScan("The keys for this migration could not be read. Start a new migration.")
|
|
return
|
|
}
|
|
inv := []models.ToolMigrationWorkspace{}
|
|
readable := 0
|
|
var lastErr error
|
|
for i, c := range creds {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
sess, err := p.Open(ctx, c)
|
|
if err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
list, err := sess.Workspaces(ctx)
|
|
if err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
for _, ws := range list {
|
|
ws.Key = i
|
|
entry := models.ToolMigrationWorkspace{Workspace: ws}
|
|
if got, err := sess.Inventory(ctx, ws); err != nil {
|
|
entry.Error = providerMessage(err)
|
|
} else {
|
|
entry.Inventory = got
|
|
readable++
|
|
}
|
|
inv = append(inv, entry)
|
|
}
|
|
}
|
|
if ctx.Err() != nil {
|
|
// Shutting down or the lease was lost: the next holder scans again.
|
|
return
|
|
}
|
|
if readable == 0 {
|
|
msg := "Nothing could be read with these keys."
|
|
if lastErr != nil {
|
|
msg = providerMessage(lastErr)
|
|
}
|
|
failScan(msg)
|
|
return
|
|
}
|
|
raw, err := json.Marshal(inv)
|
|
if err != nil {
|
|
failScan("The scan could not be saved.")
|
|
return
|
|
}
|
|
_, _ = s.d.Repo.FinishScan(work, job.ID, job.Token, raw, models.ToolMigrationReady, "")
|
|
}
|
|
|
|
// providerMessage is a provider error as a person reads it.
|
|
func providerMessage(err error) string {
|
|
var apiErr *source.APIError
|
|
switch {
|
|
case errors.As(err, &apiErr):
|
|
return apiErr.Error()
|
|
case errors.Is(err, source.ErrKey), errors.Is(err, source.ErrForbidden), errors.Is(err, source.ErrRateLimited),
|
|
errors.Is(err, source.ErrUnavailable), errors.Is(err, source.ErrUnsupported):
|
|
return err.Error()
|
|
}
|
|
return "The other tool could not be read."
|
|
}
|
|
|
|
// transient errors are retried by letting the lease lapse.
|
|
func transient(err error) bool {
|
|
return errors.Is(err, source.ErrRateLimited) || errors.Is(err, source.ErrUnavailable) ||
|
|
errors.Is(err, context.DeadlineExceeded)
|
|
}
|
|
|
|
// cursor is a target's resume point inside its current phase.
|
|
type cursor struct {
|
|
Kind source.Kind `json:"kind,omitempty"`
|
|
// Campaign indexes the source campaign list in ID order; past its end is the account-level leads.
|
|
Campaign int `json:"campaign,omitempty"`
|
|
Page string `json:"page,omitempty"`
|
|
// LeadsServed is true once any form of the leads listing answered.
|
|
LeadsServed bool `json:"leads_served,omitempty"`
|
|
}
|
|
|
|
// stop ends a target's run early without finishing it.
|
|
var (
|
|
errLost = errors.New("claim lost")
|
|
errInterrupted = errors.New("shutting down")
|
|
)
|
|
|
|
// failure is a target error worth showing to a person.
|
|
type failure struct{ msg string }
|
|
|
|
func (f *failure) Error() string { return f.msg }
|
|
|
|
func fail(format string, args ...any) error { return &failure{msg: fmt.Sprintf(format, args...)} }
|
|
|
|
// run is one claimed target in progress.
|
|
type run struct {
|
|
s *Service
|
|
job *repository.ToolMigrationTargetJob
|
|
parent context.Context
|
|
ctx context.Context
|
|
opts models.ToolMigrationOptions
|
|
info source.Info
|
|
sess source.Session
|
|
ws source.Workspace
|
|
actor uuid.UUID
|
|
label string
|
|
cur cursor
|
|
counts models.ToolMigrationCounts
|
|
followup models.ToolMigrationFollowup
|
|
mailboxImportID *uuid.UUID
|
|
categoryID *uuid.UUID
|
|
// campaignList caches the source's campaigns for this run.
|
|
campaignList []source.Campaign
|
|
}
|
|
|
|
func (s *Service) runTarget(ctx context.Context, job *repository.ToolMigrationTargetJob) {
|
|
r := &run{s: s, job: job, parent: ctx, ctx: context.WithoutCancel(ctx), label: job.Label,
|
|
mailboxImportID: job.MailboxImportID, categoryID: job.CategoryID,
|
|
ws: source.Workspace{ID: job.WorkspaceID, Name: job.WorkspaceName, Key: job.KeyIndex}}
|
|
_ = json.Unmarshal(job.Counts, &r.counts)
|
|
_ = json.Unmarshal(job.Followup, &r.followup)
|
|
if len(job.Cursor) > 0 {
|
|
_ = json.Unmarshal(job.Cursor, &r.cur)
|
|
}
|
|
|
|
err := r.prepare()
|
|
if err == nil {
|
|
err = r.apply()
|
|
}
|
|
switch {
|
|
case errors.Is(err, errLost), errors.Is(err, errInterrupted), err != nil && ctx.Err() != nil:
|
|
// Stopping, canceled, removed or claimed by another holder: the next holder carries on.
|
|
return
|
|
case err != nil && transient(err) && job.Attempts < maxAttempts:
|
|
// The lease lapses and a later pass resumes from the checkpoint.
|
|
log.Warn().Str("migration_id", job.MigrationID.String()).Msg("tool migration: the provider is unavailable, retrying later")
|
|
return
|
|
case err != nil:
|
|
var f *failure
|
|
msg := "This workspace could not be brought over."
|
|
if errors.As(err, &f) {
|
|
msg = f.msg
|
|
} else if transient(err) {
|
|
msg = providerMessage(err)
|
|
} else {
|
|
errs.CaptureException(err)
|
|
}
|
|
r.finish(models.ToolMigrationTargetFailed, msg)
|
|
default:
|
|
r.finish(models.ToolMigrationTargetDone, "")
|
|
}
|
|
}
|
|
|
|
// prepare re-checks the creator's standing and opens the source.
|
|
func (r *run) prepare() error {
|
|
job := r.job
|
|
if job.Attempts > maxAttempts {
|
|
return fail("This workspace stopped unexpectedly several times. What it brought over stays.")
|
|
}
|
|
if job.CreatedBy == nil {
|
|
return fail("The member who started this migration no longer has an account.")
|
|
}
|
|
r.actor = *job.CreatedBy
|
|
if job.Sealed == nil {
|
|
return fail("The keys for this migration are gone.")
|
|
}
|
|
if err := json.Unmarshal(job.Options, &r.opts); err != nil {
|
|
return fail("The migration's choices could not be read.")
|
|
}
|
|
if xerr := r.s.requireMember(r.ctx, job.OrgID, r.actor, neededPerms(r.opts.Kinds)); xerr != nil {
|
|
return fail("The member who started this migration can no longer write to this workspace: %s", xerr.Message)
|
|
}
|
|
if job.AgencyID != nil {
|
|
if xerr := r.s.stillAgencyAdmin(r.ctx, *job.AgencyID, r.actor); xerr != nil {
|
|
return fail("%s", xerr.Message)
|
|
}
|
|
if xerr := r.s.stillClient(r.ctx, *job.AgencyID, job.OrgID, r.actor); xerr != nil {
|
|
return fail("%s", xerr.Message)
|
|
}
|
|
}
|
|
p, ok := r.s.d.Registry.Get(source.ProviderID(job.Provider))
|
|
if !ok {
|
|
return fail("Warmbly no longer migrates from this tool.")
|
|
}
|
|
r.info = p.Info()
|
|
creds, err := r.s.openCredentials(*job.Sealed)
|
|
if err != nil {
|
|
errs.CaptureException(err)
|
|
return fail("The keys for this migration could not be read.")
|
|
}
|
|
if job.KeyIndex < 0 || job.KeyIndex >= len(creds) {
|
|
return fail("The key for this workspace is missing.")
|
|
}
|
|
sess, err := p.Open(r.parent, creds[job.KeyIndex])
|
|
if err != nil {
|
|
if transient(err) {
|
|
return err
|
|
}
|
|
return fail("%s", providerMessage(err))
|
|
}
|
|
r.sess = sess
|
|
return nil
|
|
}
|
|
|
|
// stillAgencyAdmin checks the member who started an agency migration still manages the agency.
|
|
func (s *Service) stillAgencyAdmin(ctx context.Context, agencyID, userID uuid.UUID) *errx.Error {
|
|
if s.d.Agencies == nil {
|
|
return errx.New(errx.ServiceUnavailable, "Agencies are not available on this instance.")
|
|
}
|
|
a, xerr := s.d.Agencies.Get(ctx, agencyID, userID)
|
|
if xerr != nil || !a.Role.CanManage() {
|
|
return errx.New(errx.Forbidden, "The member who started this migration is no longer an owner or admin of the agency.")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// stillClient checks the workspace still belongs to the agency.
|
|
func (s *Service) stillClient(ctx context.Context, agencyID, orgID, userID uuid.UUID) *errx.Error {
|
|
if s.d.AgencyClients == nil {
|
|
return errx.New(errx.ServiceUnavailable, "Agencies are not available on this instance.")
|
|
}
|
|
list, err := s.d.AgencyClients.ClientMemberships(ctx, agencyID, userID)
|
|
if err != nil {
|
|
return errx.InternalError()
|
|
}
|
|
for _, m := range list {
|
|
if m.OrganizationID == orgID {
|
|
return nil
|
|
}
|
|
}
|
|
return errx.New(errx.Forbidden, "This workspace is no longer one of the agency's clients.")
|
|
}
|
|
|
|
// apply runs every chosen phase from the checkpoint on.
|
|
func (r *run) apply() error {
|
|
phases := make([]source.Kind, 0, len(r.opts.Kinds))
|
|
for _, k := range source.AllKinds {
|
|
if slices.Contains(r.opts.Kinds, k) && r.info.Supports(k) {
|
|
phases = append(phases, k)
|
|
}
|
|
}
|
|
start := 0
|
|
if r.cur.Kind != "" {
|
|
if i := slices.Index(phases, r.cur.Kind); i >= 0 {
|
|
start = i
|
|
}
|
|
}
|
|
for _, k := range phases[start:] {
|
|
if r.cur.Kind != k {
|
|
r.cur = cursor{Kind: k}
|
|
}
|
|
if err := r.save(); err != nil {
|
|
return err
|
|
}
|
|
var err error
|
|
switch k {
|
|
case source.KindBlocklist:
|
|
err = r.blocklist()
|
|
case source.KindMailboxes:
|
|
err = r.mailboxes()
|
|
case source.KindCampaigns:
|
|
err = r.campaigns()
|
|
case source.KindLeads:
|
|
err = r.leads()
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// save checkpoints the target and renews its lease, stopping when it was canceled or the process is stopping.
|
|
func (r *run) save() error {
|
|
if r.parent.Err() != nil {
|
|
return errInterrupted
|
|
}
|
|
cur, _ := json.Marshal(r.cur)
|
|
held, err := r.s.d.Repo.SaveTarget(r.ctx, r.job.ID, r.job.Token, leaseDuration, r.progress(cur))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !held {
|
|
return errLost
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *run) progress(cur []byte) repository.ToolMigrationProgress {
|
|
counts, _ := json.Marshal(r.counts)
|
|
followup, _ := json.Marshal(r.followup)
|
|
return repository.ToolMigrationProgress{
|
|
Phase: string(r.cur.Kind), Cursor: cur, Counts: counts, Followup: followup,
|
|
MailboxImportID: r.mailboxImportID, CategoryID: r.categoryID,
|
|
}
|
|
}
|
|
|
|
func (r *run) finish(status models.ToolMigrationTargetStatus, msg string) {
|
|
phase := r.cur.Kind
|
|
if status == models.ToolMigrationTargetDone {
|
|
r.cur.Kind = ""
|
|
}
|
|
p := r.progress(nil)
|
|
if status != models.ToolMigrationTargetDone {
|
|
p.Phase = string(phase)
|
|
}
|
|
ended, err := r.s.d.Repo.FinishTarget(r.ctx, r.job.ID, r.job.Token, status, p, msg)
|
|
if err != nil || !ended {
|
|
return
|
|
}
|
|
r.s.audit(r.ctx, r.job.OrgID, r.actor, models.AuditActionUpdate, r.job.MigrationID, map[string]string{"status": string(status)})
|
|
if _, err := r.s.d.Repo.Finalize(r.ctx, r.job.MigrationID); err != nil {
|
|
log.Warn().Str("migration_id", r.job.MigrationID.String()).Msg("tool migration: could not close the migration")
|
|
}
|
|
}
|
|
|
|
// note records something a person has to know, once.
|
|
func (r *run) note(format string, args ...any) {
|
|
msg := fmt.Sprintf(format, args...)
|
|
if slices.Contains(r.followup.Notes, msg) {
|
|
return
|
|
}
|
|
if len(r.followup.Notes) >= maxNotes {
|
|
return
|
|
}
|
|
r.followup.Notes = append(r.followup.Notes, msg)
|
|
}
|