diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cc542e9a..b08662a94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -466,6 +466,18 @@ jobs: grep -q "^--- PASS: $test " workspace-isolation.log done + - name: Integration inbound secret sealing (PostgreSQL) + if: matrix.suite == 'live-3' + env: + WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable + run: | + set -o pipefail + go test -race -v -p 1 ./internal/app/integration ./internal/app/orgtransfer -run '^TestLive(InboundSecretSealedAcrossWriteBackfillAndRead|InboundSecretUnsealedRowStillResolves|ArchiveCarriesInboundSecretsOnlySealed|ImportFlagsEachIntegrationOnceAndClosesItsInboundURL|ReconnectFlagClearsSealedInboundSecret)$' -count=1 | tee inbound-secret.log + if grep -q -- '--- SKIP:' inbound-secret.log; then exit 1; fi + for test in TestLiveInboundSecretSealedAcrossWriteBackfillAndRead TestLiveInboundSecretUnsealedRowStillResolves TestLiveArchiveCarriesInboundSecretsOnlySealed TestLiveImportFlagsEachIntegrationOnceAndClosesItsInboundURL TestLiveReconnectFlagClearsSealedInboundSecret; do + grep -q "^--- PASS: $test " inbound-secret.log + done + - name: Monitoring database observation regressions (PostgreSQL) if: matrix.suite == 'live-2' env: diff --git a/cmd/backend/main.go b/cmd/backend/main.go index fd1eaa55c..ca7da6613 100644 --- a/cmd/backend/main.go +++ b/cmd/backend/main.go @@ -2210,6 +2210,10 @@ func main() { if m, ok := integrationServiceForHandler.(interface{ StartSigningSecretMigration(context.Context) }); ok { elector.Go("signing_secret_migration", m.StartSigningSecretMigration) } + // Inbound URL secrets are sealed with the org DEK; this seals any a writer stored without that copy. + if m, ok := integrationServiceForHandler.(interface{ StartInboundSecretSealing(context.Context) }); ok { + elector.Go("inbound_secret_sealing", m.StartInboundSecretSealing) + } go elector.Run(ctx) // Danger zone: schedule + execute delayed deletions (orgs, accounts). diff --git a/docs/content/docs/guides/clay.mdx b/docs/content/docs/guides/clay.mdx index 227d2f054..e72cdc648 100644 --- a/docs/content/docs/guides/clay.mdx +++ b/docs/content/docs/guides/clay.mdx @@ -140,4 +140,4 @@ To send to more than one Clay table, use an [automation](/guides/automations/) w ## What moves with a workspace export -The Clay connection moves with a [workspace export](/guides/workspace-export-import/) like other integrations, with its defaults and its chosen events. The table webhook and token travel only in an archive that includes credentials; otherwise paste them again. The inbound URL keeps its path on the new instance, so change the address in your Clay columns to the new instance's, or make a new URL there. +The Clay connection moves with a [workspace export](/guides/workspace-export-import/) like other integrations, with its defaults and its chosen events. The table webhook and token travel only in an archive that includes credentials; otherwise paste them again. The inbound URL travels the same way, sealed inside an archive that includes credentials. It then keeps its path on the new instance, so change the address in your Clay columns to the new instance's, or make a new URL there. Without credentials the connection arrives marked for reconnection and its old URL accepts nothing; reconnect it on the new instance and paste the new URL into your Clay columns. diff --git a/docs/content/docs/guides/workspace-export-import.mdx b/docs/content/docs/guides/workspace-export-import.mdx index 4b15dc14c..b965ea665 100644 --- a/docs/content/docs/guides/workspace-export-import.mdx +++ b/docs/content/docs/guides/workspace-export-import.mdx @@ -238,6 +238,7 @@ An import runs as one transaction. If anything fails, nothing lands and the work - **Check campaign schedules.** Per-contact progress travels, so a running campaign resumes at the step it reached rather than restarting. A campaign's choice to send in each lead's local time travels with it, and the destination works out every contact's location again from the fields that came across, usually within minutes of the import; until it has, [region segments](/guides/regions-and-timezones/) on the destination count those contacts as having no location. - **Expect the daily send counters to be honoured.** Today's counts come across, so a mailbox cannot double its volume by being migrated mid-day. - **Reconnect each integration the import report names.** Those arrived without their credentials and show as needing a reconnect under **Integrations**. A Salesforce connection syncs nothing until it is reauthorized on the destination; see [Salesforce](/guides/salesforce/#what-moves-with-a-workspace-export). +- **Check inbound URLs.** The secret in a [Calendly or Cal.com](/guides/meetings/#connecting-calendly-or-calcom) or [Clay](/guides/clay/) inbound URL travels only sealed inside an export that carries credentials, and the URL keeps its path on the destination, so give the provider the destination's address. Exported without credentials, the connection arrives marked for reconnection, the import report names it once, and its old URL accepts nothing. Reconnect it and paste the new URL into the provider. - **Reconnect Slack, then link again.** A Slack install belongs to the Slack app of the instance that made it, and Slack sends that app's messages to that instance only. Reconnect under **Integrations > Slack** on the destination, and each member links their Slack account again (the bot offers a link button the first time they message it). Assistant conversations held in Slack arrive in the dashboard history, where you can continue them. - **Check HubSpot or Pipedrive mode.** The mode, its choices and the links between Warmbly and CRM records travel, so the destination keeps updating the same records instead of creating duplicates. Its token only keeps working when the archive carried credentials and the destination uses the same HubSpot or Pipedrive app; otherwise reconnect the CRM there. Writes still queued on the source do not travel, so let sync health ([HubSpot](/guides/hubspot/#sync-health-and-fixing-errors), [Pipedrive](/guides/pipedrive/#sync-health-and-fixing-errors)) show nothing pending before exporting. The destination starts its own pull, and in Pipedrive mode registers its own webhooks. - **Rotate each webhook's signing secret.** The secret is encrypted at rest with the source instance's key, so it travels only in an export that carries credentials. Exported without them, the endpoint arrives with no secret and its deliveries will not verify at your receiver. Open each endpoint and use **Rotate secret**, then put the new value in your receiver. diff --git a/internal/app/integration/inbound_secret_live_test.go b/internal/app/integration/inbound_secret_live_test.go new file mode 100644 index 000000000..6829f5fb1 --- /dev/null +++ b/internal/app/integration/inbound_secret_live_test.go @@ -0,0 +1,230 @@ +package integration + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "os" + "path" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/warmbly/warmbly/internal/app/cipher" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +// Inbound URL secrets are sealed with the organization DEK and found by digest. +// +// WARMBLY_TEST_DB=postgres://warmbly:warmbly@localhost:15432/?sslmode=disable \ +// go test ./internal/app/integration/ -run TestLiveInboundSecret -v + +const inboundFixtureSigningKey = "fixture-signing-key" + +type inboundFixture struct { + pool *pgxpool.Pool + svc *service + org uuid.UUID + user uuid.UUID +} + +func newInboundFixture(t *testing.T) inboundFixture { + t.Helper() + dsn := os.Getenv("WARMBLY_TEST_DB") + if dsn == "" { + t.Skip("WARMBLY_TEST_DB not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(pool.Close) + f := inboundFixture{pool: pool, org: uuid.New(), user: uuid.New()} + tag := f.org.String()[:8] + for _, q := range []struct { + sql string + args []any + }{ + {`INSERT INTO users (id, first_name, last_name, email, password_hash) VALUES ($1, 'Inbound', 'Live', $2, 'x')`, + []any{f.user, "inbound-" + tag + "@test.local"}}, + {`INSERT INTO organizations (id, name, slug, owner_user_id) VALUES ($1, 'Inbound', $2, $3)`, + []any{f.org, "inbound-" + tag, f.user}}, + } { + if _, err := pool.Exec(ctx, q.sql, q.args...); err != nil { + t.Fatalf("fixture: %v", err) + } + } + t.Cleanup(func() { + ctx := context.Background() + _, _ = pool.Exec(ctx, `DELETE FROM integration_connections WHERE organization_id = $1`, f.org) + _, _ = pool.Exec(ctx, `DELETE FROM organizations WHERE id = $1`, f.org) + _, _ = pool.Exec(ctx, `DELETE FROM users WHERE id = $1`, f.user) + }) + key := bytes.Repeat([]byte{7}, 32) + f.svc = NewService(repository.NewIntegrationRepository(pool), cipher.NewStatic(key), nil).(*service) + return f +} + +// deliver runs a signed Cal.com delivery through the lookup and signature check the inbound handler uses. +func (f inboundFixture) deliver(t *testing.T, secret string) (uuid.UUID, bool) { + t.Helper() + ctx := context.Background() + conn, err := f.svc.Repo().GetConnectionByInboundSecret(ctx, models.IntegrationCalCom, secret) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if conn == nil { + return uuid.Nil, false + } + body := []byte(`{"triggerEvent":"BOOKING_CREATED"}`) + mac := hmac.New(sha256.New, []byte(inboundFixtureSigningKey)) + mac.Write(body) + header := func(name string) string { + if name == "X-Cal-Signature-256" { + return hex.EncodeToString(mac.Sum(nil)) + } + return "" + } + key, err := f.svc.InboundSigningKey(ctx, conn) + if err != nil { + t.Fatalf("signing key: %v", err) + } + if key == "" || !VerifyInboundSignature(models.IntegrationCalCom, key, header, body, time.Now()) { + t.Fatal("signed delivery did not verify") + } + return conn.ID, true +} + +type inboundColumns struct { + plain, sealed, digest *string +} + +func (f inboundFixture) columns(t *testing.T, id uuid.UUID) inboundColumns { + t.Helper() + var c inboundColumns + if err := f.pool.QueryRow(context.Background(), ` + SELECT inbound_secret, inbound_secret_encrypted, inbound_secret_digest + FROM integration_connections WHERE id = $1`, id).Scan(&c.plain, &c.sealed, &c.digest); err != nil { + t.Fatalf("read connection: %v", err) + } + return c +} + +// assertSealed checks the row holds secret sealed under the org DEK, its digest, and the clear copy the previous release reads. +func (f inboundFixture) assertSealed(t *testing.T, id uuid.UUID, secret string) { + t.Helper() + c := f.columns(t, id) + if c.sealed == nil || *c.sealed == "" || *c.sealed == secret { + t.Fatal("inbound secret has no sealed copy") + } + opened, err := f.svc.open(context.Background(), f.org, *c.sealed) + if err != nil || opened != secret { + t.Fatalf("sealed copy does not open to the secret (err %v)", err) + } + if c.digest == nil || *c.digest != repository.InboundSecretDigest(secret) { + t.Fatal("digest does not match the secret") + } + if c.plain == nil || *c.plain != secret { + t.Fatal("clear copy for the previous release is missing") + } +} + +func (f inboundFixture) connectCalCom(t *testing.T) (uuid.UUID, string) { + t.Helper() + ctx := context.Background() + conn, err := f.svc.Connect(ctx, f.org, f.user, models.IntegrationCalCom, "Bookings", nil) + if err != nil { + t.Fatalf("connect: %v", err) + } + if _, err := f.svc.SetInboundSigningKey(ctx, f.org, conn.ID, inboundFixtureSigningKey); err != nil { + t.Fatalf("set signing key: %v", err) + } + return conn.ID, path.Base(conn.InboundWebhookURL) +} + +func TestLiveInboundSecretSealedAcrossWriteBackfillAndRead(t *testing.T) { + f := newInboundFixture(t) + ctx := context.Background() + + id, secret := f.connectCalCom(t) + f.assertSealed(t, id, secret) + if got, ok := f.deliver(t, secret); !ok || got != id { + t.Fatal("delivery to a freshly minted URL did not resolve") + } + + url, err := f.svc.RotateInboundSecret(ctx, f.org, id) + if err != nil { + t.Fatalf("rotate: %v", err) + } + rotated := path.Base(url) + f.assertSealed(t, id, rotated) + if _, ok := f.deliver(t, secret); ok { + t.Fatal("rotated-out URL still resolves") + } + if got, ok := f.deliver(t, rotated); !ok || got != id { + t.Fatal("rotated URL did not resolve") + } + + // The previous release rotates by writing only the clear column; the sealed copy retires with it. + prior := "calcom_" + hex.EncodeToString(bytes.Repeat([]byte{0xab}, 24)) + if _, err := f.pool.Exec(ctx, `UPDATE integration_connections SET inbound_secret = $2 WHERE id = $1`, id, prior); err != nil { + t.Fatal(err) + } + if c := f.columns(t, id); c.sealed != nil || c.digest != nil { + t.Fatal("a clear-only rotation left the earlier sealed copy in place") + } + if _, ok := f.deliver(t, rotated); ok { + t.Fatal("URL rotated out by the previous release still resolves") + } + if got, ok := f.deliver(t, prior); !ok || got != id { + t.Fatal("URL minted by the previous release did not resolve before sealing") + } + + if _, err := f.svc.SealInboundSecrets(ctx); err != nil { + t.Fatalf("seal pass: %v", err) + } + f.assertSealed(t, id, prior) + if got, ok := f.deliver(t, prior); !ok || got != id { + t.Fatal("URL did not resolve after sealing") + } +} + +func TestLiveInboundSecretUnsealedRowStillResolves(t *testing.T) { + f := newInboundFixture(t) + ctx := context.Background() + + id, _ := f.connectCalCom(t) + // A row as the previous release writes it: the secret in the clear and nothing else. + legacy := "calcom_" + hex.EncodeToString(bytes.Repeat([]byte{0xcd}, 24)) + if _, err := f.pool.Exec(ctx, ` + UPDATE integration_connections + SET inbound_secret = $2, inbound_secret_encrypted = NULL, inbound_secret_digest = NULL + WHERE id = $1`, id, legacy); err != nil { + t.Fatal(err) + } + if got, ok := f.deliver(t, legacy); !ok || got != id { + t.Fatal("unsealed row did not resolve") + } + + if _, err := f.svc.SealInboundSecrets(ctx); err != nil { + t.Fatalf("seal pass: %v", err) + } + f.assertSealed(t, id, legacy) + if got, ok := f.deliver(t, legacy); !ok || got != id { + t.Fatal("row did not resolve after sealing") + } + // A second pass finds nothing left to seal on this row and changes nothing. + before := f.columns(t, id) + if _, err := f.svc.SealInboundSecrets(ctx); err != nil { + t.Fatalf("second seal pass: %v", err) + } + if after := f.columns(t, id); *after.sealed != *before.sealed { + t.Fatal("a second pass resealed an already sealed row") + } +} diff --git a/internal/app/integration/service.go b/internal/app/integration/service.go index 20d787fc7..ec21c7858 100644 --- a/internal/app/integration/service.go +++ b/internal/app/integration/service.go @@ -411,13 +411,16 @@ func (s *service) Connect(ctx context.Context, orgID, userID uuid.UUID, provider } } - var inboundSecret string + var inboundSecret, inboundSealed string var err error if mintsInboundURL(provider) { inboundSecret, err = generateInboundSecret(provider) if err != nil { return nil, err } + if inboundSealed, err = s.seal(ctx, orgID, inboundSecret); err != nil { + return nil, err + } } configEnc, err := s.sealConfig(ctx, orgID, config) @@ -468,9 +471,10 @@ func (s *service) Connect(ctx context.Context, orgID, userID uuid.UUID, provider } if err := s.repo.UpsertConnection(ctx, &repository.ConnectionWrite{ - Conn: conn, - ConfigEncrypted: configEnc, - InboundSecret: inboundSecret, + Conn: conn, + ConfigEncrypted: configEnc, + InboundSecret: inboundSecret, + InboundSecretSealed: inboundSealed, }); err != nil { return nil, err } @@ -774,7 +778,11 @@ func (s *service) RotateInboundSecret(ctx context.Context, orgID, id uuid.UUID) if err != nil { return "", err } - if err := s.repo.SetInboundSecret(ctx, orgID, id, secret); err != nil { + sealed, err := s.seal(ctx, orgID, secret) + if err != nil { + return "", err + } + if err := s.repo.SetInboundSecret(ctx, orgID, id, secret, sealed); err != nil { return "", err } return BuildInboundURL(conn.Provider, secret), nil @@ -1606,6 +1614,59 @@ func (s *service) StartSigningSecretMigration(ctx context.Context) { } } +// inboundSealingInterval paces the sweep that seals inbound secrets a writer stored without a sealed copy. +const inboundSealingInterval = 10 * time.Minute + +// StartInboundSecretSealing seals every inbound URL secret that has no sealed +// copy, then repeats, so rows written by a previous release mid-rollout are sealed too. +func (s *service) StartInboundSecretSealing(ctx context.Context) { + for { + if _, err := s.SealInboundSecrets(ctx); err != nil && ctx.Err() == nil { + log.Warn().Err(err).Msg("integration: sealing inbound secrets failed") + } + select { + case <-ctx.Done(): + return + case <-time.After(inboundSealingInterval): + } + } +} + +// SealInboundSecrets runs one resumable pass of the inbound secret sealing and reports how many rows it sealed. +func (s *service) SealInboundSecrets(ctx context.Context) (int, error) { + var cursor uuid.UUID + sealedRows := 0 + for { + batch, err := s.repo.ListUnsealedInboundSecrets(ctx, cursor, 100) + if err != nil { + return sealedRows, err + } + if len(batch) == 0 { + return sealedRows, nil + } + for _, u := range batch { + cursor = u.ID + sealed, err := s.seal(ctx, u.OrganizationID, u.Secret) + if err != nil { + log.Warn().Err(err).Str("connection_id", u.ID.String()).Msg("integration: sealing inbound secret failed") + continue + } + ok, err := s.repo.SealInboundSecret(ctx, u.ID, u.Secret, sealed) + if err != nil { + return sealedRows, err + } + if ok { + sealedRows++ + } + } + select { + case <-ctx.Done(): + return sealedRows, ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } +} + // SendTestEvent fires a synthetic event through the connection's notify/webhook // automations (Slack, Discord, generic webhook), reusing the real delivery path // so signing + payload shape match production. CRM-upsert automations are diff --git a/internal/app/orgtransfer/export.go b/internal/app/orgtransfer/export.go index 568786edf..c89983edf 100644 --- a/internal/app/orgtransfer/export.go +++ b/internal/app/orgtransfer/export.go @@ -135,7 +135,7 @@ func (s *service) ExportTo( Name: t.Name, Group: t.Group, Columns: slices.DeleteFunc(columnNames(cols), func(column string) bool { - return slices.Contains(t.ExcludeOnExport, column) + return slices.Contains(t.ExcludeOnExport, column) || slices.Contains(t.derivedColumns(), column) }), } if archiveKey != nil { @@ -227,12 +227,17 @@ func (s *service) exportRow( } for _, sc := range t.Secrets { + // A row not yet sealed carries its value only in the clear column, which stands in for it. + var clear string + if sc.Plaintext != "" { + clear = jsonString(obj[sc.Plaintext]) + } raw, ok := obj[sc.Column] - if !ok { + if !ok && clear == "" { continue } stored := jsonString(raw) - if stored == "" { + if stored == "" && clear == "" { continue } // A guarded column only holds ciphertext when its flag is set. @@ -247,17 +252,20 @@ func (s *service) exportRow( continue } + var plain string var openErr error - if bytea[sc.Column] { - if stored, openErr = decodeBytea(stored); openErr == nil && stored == "" { + if bytea[sc.Column] && stored != "" { + if stored, openErr = decodeBytea(stored); openErr == nil && stored == "" && clear == "" { obj[sc.Column] = json.RawMessage(`null`) continue } } - var plain string - if openErr == nil { + if openErr == nil && stored != "" { plain, openErr = s.openSecret(ctx, sc, stored, orgCipher) } + if (stored == "" || openErr != nil) && clear != "" { + plain, openErr = clear, nil + } if openErr != nil { // One unreadable credential must not sink the export. Blank it and // let the mailbox or integration arrive needing a reconnect. @@ -275,6 +283,10 @@ func (s *service) exportRow( obj[sc.Column] = enc } + for _, column := range t.derivedColumns() { + delete(obj, column) + } + for _, bc := range t.Blobs { v := jsonString(obj[bc.Column]) if v == "" { diff --git a/internal/app/orgtransfer/import.go b/internal/app/orgtransfer/import.go index b71e63781..f7a2db2af 100644 --- a/internal/app/orgtransfer/import.go +++ b/internal/app/orgtransfer/import.go @@ -196,7 +196,7 @@ func (s *service) ImportFrom( } if selected[models.OrgDataGroupAutomations] { - stale, err := s.repo.MarkIntegrationsNeedingReconnect(ctx, tx, orgID, integrationReconnectDetail) + stale, err := s.repo.MarkIntegrationsNeedingReconnect(ctx, tx, orgID, integrationReconnectDetail, inboundReconnectDetail) if err != nil { return nil, err } @@ -315,6 +315,17 @@ func (s *service) importTable( unknown++ } } + // Columns rebuilt from a sealed one are written whenever that one arrives. + for _, sc := range t.Secrets { + if !slices.Contains(insertCols, sc.Column) { + continue + } + for _, c := range []string{sc.Plaintext, sc.Digest} { + if c != "" && writable[c] && !slices.Contains(insertCols, c) { + insertCols = append(insertCols, c) + } + } + } if len(insertCols) == 0 { return 0, []string{fmt.Sprintf("No column of %s matched this instance, so it was skipped.", t.Name)}, nil } @@ -493,17 +504,20 @@ func (s *service) importRow( } stored := jsonString(raw) if stored == "" { + setDerived(obj, sc, "") continue } if !IsSealed(stored) { // Exported without secrets, so the column is already blank or was // never ciphertext. Nothing to re-key. + setDerived(obj, sc, "") continue } if ic.archiveKey == nil { // Sealed, but no passphrase. Blank it rather than storing a value // this instance's keys can never open. blankSecret(obj, sc) + setDerived(obj, sc, "") continue } plain, oerr := OpenValue(ic.archiveKey, stored) @@ -522,6 +536,7 @@ func (s *service) importRow( return nil, merr } obj[sc.Column] = enc + setDerived(obj, sc, plain) } return json.Marshal(obj) @@ -530,13 +545,44 @@ func (s *service) importRow( // integrationReconnectDetail is the health detail an integration without its credentials carries. const integrationReconnectDetail = "Credentials did not come across with the workspace import: reconnect this integration." -// integrationReconnectWarning names one connection by its provider and label, never by a credential. +// inboundReconnectDetail is the health detail of a connection whose inbound URL alone did not travel. +const inboundReconnectDetail = "The inbound URL did not come across with the workspace import: reconnect this integration for a new one." + +// integrationReconnectWarning is the one warning per connection, naming its provider and label, never a credential. func integrationReconnectWarning(c repository.IntegrationReconnect) string { name := c.Provider if c.Label != "" { name = fmt.Sprintf("%s (%s)", c.Provider, c.Label) } - return fmt.Sprintf("The %s integration arrived without its credentials and is marked for reconnection.", name) + missing := "its credentials" + switch { + case c.CredentialsMissing && c.InboundClosed: + missing = "its credentials or inbound URL" + case !c.CredentialsMissing: + missing = "its inbound URL" + } + return fmt.Sprintf("The %s integration arrived without %s and is marked for reconnection.", name, missing) +} + +// setDerived rebuilds the clear and digest columns of a sealed one from its plaintext; "" clears them. +func setDerived(obj map[string]json.RawMessage, sc SecretColumn, plain string) { + set := func(column, value string) { + if column == "" { + return + } + if value == "" { + obj[column] = json.RawMessage(`null`) + return + } + enc, _ := json.Marshal(value) + obj[column] = enc + } + set(sc.Plaintext, plain) + digest := "" + if plain != "" { + digest = repository.InboundSecretDigest(plain) + } + set(sc.Digest, digest) } // sealSecret re-encrypts a plaintext under this instance's key for that domain. diff --git a/internal/app/orgtransfer/integration_inbound_secret_live_test.go b/internal/app/orgtransfer/integration_inbound_secret_live_test.go new file mode 100644 index 000000000..a051a3643 --- /dev/null +++ b/internal/app/orgtransfer/integration_inbound_secret_live_test.go @@ -0,0 +1,323 @@ +package orgtransfer + +import ( + "archive/zip" + "bytes" + "context" + "encoding/hex" + "encoding/json" + "io" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/warmbly/warmbly/internal/app/cipher" + "github.com/warmbly/warmbly/internal/infrastructure/db" + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/repository" +) + +// Integration inbound secrets travel in an archive only as passphrase-sealed values. +func TestLiveArchiveCarriesInboundSecretsOnlySealed(t *testing.T) { + for _, passphrase := range []string{"inbound secret archive passphrase", ""} { + name := "with passphrase" + if passphrase == "" { + name = "without passphrase" + } + t.Run(name, func(t *testing.T) { + _, pool := liveImportService(t) + ctx := context.Background() + src := newTenantFixture(t, pool) + srcKeys := cipher.NewStatic(bytes.Repeat([]byte{1}, 32)) + rekeyed := cipher.NewStatic(bytes.Repeat([]byte{2}, 32)) + keys := perOrgKeys{src.org: srcKeys} + svc := NewService(repository.NewOrgTransferRepository(&db.DB{Pool: pool}), keys, nil, nil, InstanceInfo{}) + + sealedSecret := "calcom_" + hex.EncodeToString(bytes.Repeat([]byte{0x11}, 24)) + clearSecret := "clay_" + hex.EncodeToString(bytes.Repeat([]byte{0x22}, 24)) + srcCipher, err := srcKeys.Cipher(ctx, src.org) + if err != nil { + t.Fatal(err) + } + sealedAtRest, err := srcCipher.Encrypt(ctx, sealedSecret) + if err != nil { + t.Fatal(err) + } + sealedConn, clearConn := uuid.New(), uuid.New() + // One row sealed as this release writes it, one only in the clear as the previous release does. + if _, err := pool.Exec(ctx, ` + INSERT INTO integration_connections (id, organization_id, provider, label, status, auth_method, inbound_secret, inbound_secret_encrypted, inbound_secret_digest, health) + VALUES ($1, $3, 'cal_com', 'Bookings', 'connected', 'webhook', $4, $5, $6, 'healthy'), + ($2, $3, 'clay', 'Enrichment', 'connected', 'webhook', $7, NULL, NULL, 'healthy')`, + sealedConn, clearConn, src.org, sealedSecret, sealedAtRest, repository.InboundSecretDigest(sealedSecret), clearSecret); err != nil { + t.Fatal(err) + } + + var archive bytes.Buffer + if _, err := svc.ExportTo(ctx, src.org, ExportOptions{Passphrase: passphrase, Groups: []models.OrgDataGroup{models.OrgDataGroupAutomations}}, &archive, nil); err != nil { + t.Fatalf("export: %v", err) + } + rows := archiveConnectionRows(t, archive.Bytes(), sealedSecret, clearSecret) + if len(rows) != 2 { + t.Fatalf("archive holds %d connections, want 2", len(rows)) + } + for _, row := range rows { + for _, c := range []string{"inbound_secret", "inbound_secret_digest"} { + if _, ok := row[c]; ok { + t.Fatalf("archive carries %s", c) + } + } + sealed := jsonString(row["inbound_secret_encrypted"]) + if passphrase != "" && !IsSealed(sealed) { + t.Fatal("inbound secret is not passphrase-sealed in the archive") + } + if passphrase == "" && sealed != "" { + t.Fatal("an archive without a passphrase carries an inbound secret") + } + } + + // The workspace comes back under a different DEK, as it would on another instance. + if _, err := pool.Exec(ctx, `DELETE FROM integration_connections WHERE organization_id = $1`, src.org); err != nil { + t.Fatal(err) + } + keys[src.org] = rekeyed + result, err := svc.ImportFrom(ctx, src.org, bytes.NewReader(archive.Bytes()), ImportOptions{ + Passphrase: passphrase, + Groups: []models.OrgDataGroup{models.OrgDataGroupAutomations}, + Conflict: models.OrgImportConflictSkip, + ActorUserID: src.owner, + }, nil) + if err != nil { + t.Fatalf("import: %v", err) + } + + integrations := repository.NewIntegrationRepository(pool) + rekeyedCipher, err := rekeyed.Cipher(ctx, src.org) + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + id uuid.UUID + provider models.IntegrationProvider + secret string + }{{sealedConn, models.IntegrationCalCom, sealedSecret}, {clearConn, models.IntegrationClay, clearSecret}} { + var org uuid.UUID + var status string + var plain, sealed, digest *string + if err := pool.QueryRow(ctx, ` + SELECT organization_id, status, inbound_secret, inbound_secret_encrypted, inbound_secret_digest + FROM integration_connections WHERE id = $1`, c.id).Scan(&org, &status, &plain, &sealed, &digest); err != nil { + t.Fatalf("read imported connection: %v", err) + } + if org != src.org { + t.Fatal("imported connection is not in its workspace") + } + found, err := integrations.GetConnectionByInboundSecret(ctx, c.provider, c.secret) + if err != nil { + t.Fatal(err) + } + if passphrase == "" { + if status != string(models.IntegrationStatusReauthRequired) || plain != nil || sealed != nil || digest != nil { + t.Fatalf("uncarried inbound URL not closed: status=%s", status) + } + if found != nil { + t.Fatal("an uncarried inbound URL still resolves") + } + if !hasWarning(result.Warnings, string(c.provider)) { + t.Fatalf("import did not name the %s connection for reconnection", c.provider) + } + continue + } + if status != string(models.IntegrationStatusConnected) { + t.Fatalf("carried connection arrived %s", status) + } + if sealed == nil || plain == nil || *plain != c.secret || digest == nil || *digest != repository.InboundSecretDigest(c.secret) { + t.Fatal("imported connection is missing its sealed secret, clear copy or digest") + } + if opened, err := rekeyedCipher.Decrypt(ctx, *sealed); err != nil || opened != c.secret { + t.Fatal("imported secret does not open under the destination key") + } + if _, err := srcCipher.Decrypt(ctx, *sealed); err == nil { + t.Fatal("imported secret still opens under the source key") + } + if found == nil || found.ID != c.id || found.OrganizationID != src.org { + t.Fatal("imported inbound URL does not resolve to the destination connection") + } + } + }) + } +} + +// archiveConnectionRows returns the archive's integration_connections rows, failing if a secret appears anywhere in it. +func archiveConnectionRows(t *testing.T, archive []byte, secrets ...string) []map[string]json.RawMessage { + t.Helper() + zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive))) + if err != nil { + t.Fatal(err) + } + var rows []map[string]json.RawMessage + for _, entry := range zr.File { + r, err := entry.Open() + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(r) + _ = r.Close() + if err != nil { + t.Fatal(err) + } + for _, s := range secrets { + if bytes.Contains(data, []byte(s)) { + t.Fatalf("%s carries an inbound secret in the clear", entry.Name) + } + } + if entry.Name != dataPath("integration_connections") { + continue + } + for _, line := range bytes.Split(bytes.TrimSpace(data), []byte("\n")) { + if len(line) == 0 { + continue + } + var row map[string]json.RawMessage + if err := json.Unmarshal(line, &row); err != nil { + t.Fatal(err) + } + rows = append(rows, row) + } + } + return rows +} + +func hasWarning(warnings []string, needle string) bool { + for _, w := range warnings { + if strings.Contains(w, needle) { + return true + } + } + return false +} + +// A connection whose config did not travel is flagged once and its inbound URL closes in every column, sealed or not. +func TestLiveImportFlagsEachIntegrationOnceAndClosesItsInboundURL(t *testing.T) { + for _, passphrase := range []string{"inbound secret archive passphrase", ""} { + name := "with passphrase" + if passphrase == "" { + name = "without passphrase" + } + t.Run(name, func(t *testing.T) { + keys := perOrgKeys{} + svc, pool := keyedImportService(t, keys) + ctx := context.Background() + src := newTenantFixture(t, pool) + keys[src.org] = staticKey(1) + + secret := "calcom_" + hex.EncodeToString(bytes.Repeat([]byte{0x33}, 24)) + conn := uuid.New() + // The config is sealed under a key this workspace does not hold, so the export cannot carry it; + // the inbound secret is sealed under its own key, so with a passphrase it travels. + if _, err := pool.Exec(ctx, ` + INSERT INTO integration_connections (id, organization_id, provider, label, status, auth_method, config_encrypted, inbound_secret, inbound_secret_encrypted, inbound_secret_digest, health) + VALUES ($1, $2, 'cal_com', 'Bookings', 'connected', 'webhook', $3, $4, $5, $6, 'healthy')`, + conn, src.org, []byte(sealFixture(t, staticKey(9), `{"inbound_signing_key":"fixture-signing-key"}`)), + secret, sealFixture(t, keys[src.org], secret), repository.InboundSecretDigest(secret)); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM integration_connections WHERE organization_id = $1`, src.org) + }) + + groups := []models.OrgDataGroup{models.OrgDataGroupAutomations} + var archive bytes.Buffer + if _, err := svc.ExportTo(ctx, src.org, ExportOptions{Passphrase: passphrase, Groups: groups}, &archive, nil); err != nil { + t.Fatalf("export: %v", err) + } + if _, err := pool.Exec(ctx, `DELETE FROM integration_connections WHERE organization_id = $1`, src.org); err != nil { + t.Fatal(err) + } + keys[src.org] = staticKey(2) + result, err := svc.ImportFrom(ctx, src.org, bytes.NewReader(archive.Bytes()), ImportOptions{ + Passphrase: passphrase, Groups: groups, Conflict: models.OrgImportConflictSkip, ActorUserID: src.owner, + }, nil) + if err != nil { + t.Fatalf("import: %v", err) + } + + assertInboundClosed(t, pool, conn, secret) + var named []string + for _, w := range result.Warnings { + if strings.Contains(w, "cal_com (Bookings)") { + named = append(named, w) + } + } + if len(named) != 1 || !strings.Contains(named[0], "without its credentials or inbound URL") { + t.Fatalf("want one warning naming the connection's credentials and inbound URL, got %d", len(named)) + } + }) + } +} + +// The flag step clears the sealed copy and digest itself, also on a row whose clear column is already NULL and fires no trigger. +func TestLiveReconnectFlagClearsSealedInboundSecret(t *testing.T) { + _, pool := liveImportService(t) + ctx := context.Background() + src := newTenantFixture(t, pool) + secret := "calcom_" + hex.EncodeToString(bytes.Repeat([]byte{0x44}, 24)) + conn := uuid.New() + if _, err := pool.Exec(ctx, ` + INSERT INTO integration_connections (id, organization_id, provider, label, status, auth_method, config_encrypted, inbound_secret, inbound_secret_encrypted, inbound_secret_digest, health) + VALUES ($1, $2, 'cal_com', '', 'connected', 'webhook', '\x'::bytea, NULL, $3, $4, 'healthy')`, + conn, src.org, sealFixture(t, staticKey(1), secret), repository.InboundSecretDigest(secret)); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM integration_connections WHERE id = $1`, conn) + }) + if found, err := repository.NewIntegrationRepository(pool).GetConnectionByInboundSecret(ctx, models.IntegrationCalCom, secret); err != nil || found == nil { + t.Fatalf("fixture URL does not resolve before the flag: %v", err) + } + + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback(context.Background()) }() + stale, err := repository.NewOrgTransferRepository(&db.DB{Pool: pool}).MarkIntegrationsNeedingReconnect(ctx, tx, src.org, integrationReconnectDetail, inboundReconnectDetail) + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + if len(stale) != 1 || !stale[0].CredentialsMissing || !stale[0].InboundClosed { + t.Fatalf("want the connection flagged once with its inbound URL closed, got %+v", stale) + } + assertInboundClosed(t, pool, conn, secret) +} + +// assertInboundClosed checks a connection awaits reconnection and its old inbound URL is refused. +func assertInboundClosed(t *testing.T, pool *pgxpool.Pool, conn uuid.UUID, secret string) { + t.Helper() + ctx := context.Background() + var status string + var plain, sealed, digest *string + if err := pool.QueryRow(ctx, ` + SELECT status, inbound_secret, inbound_secret_encrypted, inbound_secret_digest + FROM integration_connections WHERE id = $1`, conn).Scan(&status, &plain, &sealed, &digest); err != nil { + t.Fatal(err) + } + if status != string(models.IntegrationStatusReauthRequired) { + t.Fatalf("flagged connection is %s, want reauth_required", status) + } + if plain != nil || sealed != nil || digest != nil { + t.Fatalf("closed inbound URL kept a value: clear=%t sealed=%t digest=%t", plain != nil, sealed != nil, digest != nil) + } + found, err := repository.NewIntegrationRepository(pool).GetConnectionByInboundSecret(ctx, models.IntegrationCalCom, secret) + if err != nil { + t.Fatal(err) + } + if found != nil { + t.Fatal("a closed inbound URL still resolves") + } +} diff --git a/internal/app/orgtransfer/integration_tracked_links_live_test.go b/internal/app/orgtransfer/integration_tracked_links_live_test.go index 88cfe570b..b0fc8b3b7 100644 --- a/internal/app/orgtransfer/integration_tracked_links_live_test.go +++ b/internal/app/orgtransfer/integration_tracked_links_live_test.go @@ -277,6 +277,18 @@ func TestLiveArchiveCarriesIntegrationCredentialsAndDirectMailLinks(t *testing.T reconnect = append(reconnect, w) } } + // Each connection is named once, whatever of its credentials and inbound URL stayed behind. + for _, name := range []string{"apollo (Prospecting)", "The hubspot integration", "calendly (Bookings)"} { + n := 0 + for _, w := range result.Warnings { + if strings.Contains(w, name) { + n++ + } + } + if n > 1 { + t.Fatalf("%s is named in %d import warnings, want one", name, n) + } + } apollo, hubspot, calendly := read(srcConns.apollo), read(srcConns.hubspot), read(srcConns.calendly) for _, c := range []conn{apollo, hubspot, calendly} { diff --git a/internal/app/orgtransfer/spec.go b/internal/app/orgtransfer/spec.go index f24b99213..26cd9112e 100644 --- a/internal/app/orgtransfer/spec.go +++ b/internal/app/orgtransfer/spec.go @@ -59,6 +59,11 @@ type SecretColumn struct { Guard string // PlaintextPrefix marks a value still stored in the clear from before sealing; it travels as is. PlaintextPrefix string + // Plaintext names a column still written in the clear beside this one for the previous release. + // It never travels: it stands in for this column on a row not yet sealed, and is rebuilt from it on import. + Plaintext string + // Digest names a lookup column holding repository.InboundSecretDigest of the value; rebuilt on import, never exported. + Digest string } // JSONColumnExclusion names instance-local keys inside a stored JSON envelope. @@ -120,6 +125,19 @@ type Table struct { Note string } +// derivedColumns are the columns rebuilt from a sealed one on import, which never travel. +func (t *Table) derivedColumns() []string { + var out []string + for _, sc := range t.Secrets { + for _, c := range []string{sc.Plaintext, sc.Digest} { + if c != "" { + out = append(out, c) + } + } + } + return out +} + // OwnerScope is the WHERE fragment deciding whether an existing row belongs to // the organization, which an import checks before writing or referencing it. func (t *Table) OwnerScope() string { @@ -661,6 +679,7 @@ var Tables = []Table{ {Column: "config_encrypted", Domain: KeyDomainOrgDEK}, {Column: "access_token_encrypted", Domain: KeyDomainOrgDEK}, {Column: "refresh_token_encrypted", Domain: KeyDomainOrgDEK}, + {Column: "inbound_secret_encrypted", Domain: KeyDomainOrgDEK, Plaintext: "inbound_secret", Digest: "inbound_secret_digest"}, }, ResetOnImport: []string{"last_synced_at", "last_error", "last_error_at", "health_checked_at"}, }, diff --git a/internal/infrastructure/db/migrations/000371_integration_inbound_secret_sealed.down.sql b/internal/infrastructure/db/migrations/000371_integration_inbound_secret_sealed.down.sql new file mode 100644 index 000000000..be3723ef4 --- /dev/null +++ b/internal/infrastructure/db/migrations/000371_integration_inbound_secret_sealed.down.sql @@ -0,0 +1,6 @@ +DROP TRIGGER IF EXISTS integration_inbound_secret_retire_sealed ON integration_connections; +DROP FUNCTION IF EXISTS integration_inbound_secret_retire_sealed(); +DROP INDEX IF EXISTS idx_integration_connections_inbound_digest; +ALTER TABLE integration_connections + DROP COLUMN IF EXISTS inbound_secret_digest, + DROP COLUMN IF EXISTS inbound_secret_encrypted; diff --git a/internal/infrastructure/db/migrations/000371_integration_inbound_secret_sealed.up.sql b/internal/infrastructure/db/migrations/000371_integration_inbound_secret_sealed.up.sql new file mode 100644 index 000000000..3ea932b71 --- /dev/null +++ b/internal/infrastructure/db/migrations/000371_integration_inbound_secret_sealed.up.sql @@ -0,0 +1,30 @@ +-- The inbound URL secret is sealed with the organization DEK and found by its SHA-256 digest. +ALTER TABLE integration_connections + ADD COLUMN IF NOT EXISTS inbound_secret_encrypted text, + ADD COLUMN IF NOT EXISTS inbound_secret_digest text; + +CREATE INDEX IF NOT EXISTS idx_integration_connections_inbound_digest + ON integration_connections (provider, inbound_secret_digest) + WHERE inbound_secret_digest IS NOT NULL; + +-- A writer that changes only inbound_secret (the previous release) retires the sealed copy, so a rotated URL never keeps resolving. +CREATE FUNCTION integration_inbound_secret_retire_sealed() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.inbound_secret IS DISTINCT FROM OLD.inbound_secret + AND NEW.inbound_secret_digest IS NOT DISTINCT FROM OLD.inbound_secret_digest THEN + NEW.inbound_secret_encrypted := NULL; + NEW.inbound_secret_digest := NULL; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER integration_inbound_secret_retire_sealed +BEFORE UPDATE OF inbound_secret ON integration_connections +FOR EACH ROW EXECUTE FUNCTION integration_inbound_secret_retire_sealed(); + +COMMENT ON COLUMN public.integration_connections.inbound_secret_encrypted IS + 'Inbound URL secret sealed with the organization DEK.'; +COMMENT ON COLUMN public.integration_connections.inbound_secret_digest IS + 'Hex SHA-256 of the inbound URL secret, the lookup key for deliveries.'; diff --git a/internal/repository/pg_integration.go b/internal/repository/pg_integration.go index b5e889bbd..fe8268f8f 100644 --- a/internal/repository/pg_integration.go +++ b/internal/repository/pg_integration.go @@ -2,7 +2,9 @@ package repository import ( "context" + "crypto/sha256" "database/sql" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -27,6 +29,8 @@ type ConnectionWrite struct { AccessTokenEnc string // base64 ciphertext; "" = leave unchanged RefreshTokenEnc string // base64 ciphertext; "" = leave unchanged InboundSecret string // "" = leave unchanged + // InboundSecretSealed is InboundSecret sealed with the org DEK; set whenever InboundSecret is. + InboundSecretSealed string } // ConnectionSecrets carries the encrypted secret material for a connection so @@ -127,7 +131,11 @@ type IntegrationRepository interface { // config and records in display_fields whether deliveries must be signed. SetInboundSigningConfig(ctx context.Context, orgID, connID uuid.UUID, configEncrypted []byte, signed bool) error // SetInboundSecret replaces a Calendly/Cal.com connection's inbound URL secret. - SetInboundSecret(ctx context.Context, orgID, connID uuid.UUID, secret string) error + SetInboundSecret(ctx context.Context, orgID, connID uuid.UUID, secret, sealed string) error + // ListUnsealedInboundSecrets pages, by id, the connections whose inbound secret has no sealed copy yet. + ListUnsealedInboundSecrets(ctx context.Context, after uuid.UUID, limit int) ([]UnsealedInboundSecret, error) + // SealInboundSecret stores the sealed copy and digest of a secret, only while the row still holds that secret unsealed. + SealInboundSecret(ctx context.Context, id uuid.UUID, secret, sealed string) (bool, error) // Sync runs CreateSyncRun(ctx context.Context, run *models.IntegrationSyncRun) error @@ -191,19 +199,23 @@ func (r *integrationRepository) UpsertConnection(ctx context.Context, w *Connect connected_by_user_id, access_token_encrypted, refresh_token_encrypted, token_expires_at, granted_scopes, external_account_id, external_account_name, health, health_detail, health_checked_at, - created_at, updated_at + created_at, updated_at, + inbound_secret_encrypted, inbound_secret_digest ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, - $20, $20 + $20, $20, + $21, $22 ) ON CONFLICT (organization_id, provider, label) DO UPDATE SET status = EXCLUDED.status, auth_method = EXCLUDED.auth_method, inbound_secret = COALESCE(EXCLUDED.inbound_secret, integration_connections.inbound_secret), + inbound_secret_encrypted = CASE WHEN EXCLUDED.inbound_secret IS NULL THEN integration_connections.inbound_secret_encrypted ELSE EXCLUDED.inbound_secret_encrypted END, + inbound_secret_digest = CASE WHEN EXCLUDED.inbound_secret IS NULL THEN integration_connections.inbound_secret_digest ELSE EXCLUDED.inbound_secret_digest END, config_encrypted = COALESCE(EXCLUDED.config_encrypted, integration_connections.config_encrypted), display_fields = EXCLUDED.display_fields, connected_by_user_id = COALESCE(EXCLUDED.connected_by_user_id, integration_connections.connected_by_user_id), @@ -224,6 +236,7 @@ func (r *integrationRepository) UpsertConnection(ctx context.Context, w *Connect c.TokenExpiresAt, normalizeScopes(c.GrantedScopes), nullIfEmptyStr(c.ExternalAccountID), nullIfEmptyStr(c.ExternalAccountName), c.Health, c.HealthDetail, c.HealthCheckedAt, now, + nullIfEmptyStr(w.InboundSecretSealed), nullIfEmptyStr(inboundDigest(w.InboundSecret)), ) return err } @@ -336,9 +349,13 @@ func (r *integrationRepository) GetConnectionByInboundSecret(ctx context.Context if secret == "" { return nil, nil } + // The digest is the lookup; a row not yet sealed is still found by its secret. row := r.db.QueryRow(ctx, `SELECT `+connectionPublicCols+` - FROM integration_connections WHERE provider = $1 AND inbound_secret = $2 LIMIT 1`, - string(provider), secret) + FROM integration_connections + WHERE provider = $1 + AND (inbound_secret_digest = $2 OR (inbound_secret_digest IS NULL AND inbound_secret = $3)) + LIMIT 1`, + string(provider), inboundDigest(secret), secret) var c models.IntegrationConnection if err := scanConnectionInto(row, &c); err != nil { if isNoRows(err) { @@ -936,11 +953,15 @@ func (r *integrationRepository) SetInboundSigningConfig(ctx context.Context, org return nil } -func (r *integrationRepository) SetInboundSecret(ctx context.Context, orgID, connID uuid.UUID, secret string) error { +func (r *integrationRepository) SetInboundSecret(ctx context.Context, orgID, connID uuid.UUID, secret, sealed string) error { + if secret == "" || sealed == "" { + return errors.New("inbound secret and its sealed copy are both required") + } tag, err := r.db.Exec(ctx, ` - UPDATE integration_connections SET inbound_secret = $3, updated_at = now() + UPDATE integration_connections + SET inbound_secret = $3, inbound_secret_encrypted = $4, inbound_secret_digest = $5, updated_at = now() WHERE organization_id = $1 AND id = $2 AND provider IN ('calendly', 'cal_com', 'clay')`, - orgID, connID, secret) + orgID, connID, secret, sealed, inboundDigest(secret)) if err != nil { return err } @@ -950,6 +971,61 @@ func (r *integrationRepository) SetInboundSecret(ctx context.Context, orgID, con return nil } +// InboundSecretDigest is the lookup key stored for an inbound URL secret: its hex SHA-256. +func InboundSecretDigest(secret string) string { + sum := sha256.Sum256([]byte(secret)) + return hex.EncodeToString(sum[:]) +} + +func inboundDigest(secret string) string { + if secret == "" { + return "" + } + return InboundSecretDigest(secret) +} + +// UnsealedInboundSecret is a connection whose inbound secret still has no sealed copy. +type UnsealedInboundSecret struct { + ID uuid.UUID + OrganizationID uuid.UUID + Secret string +} + +func (r *integrationRepository) ListUnsealedInboundSecrets(ctx context.Context, after uuid.UUID, limit int) ([]UnsealedInboundSecret, error) { + rows, err := r.db.Query(ctx, ` + SELECT id, organization_id, inbound_secret FROM integration_connections + WHERE id > $1 AND inbound_secret IS NOT NULL AND inbound_secret <> '' AND inbound_secret_digest IS NULL + ORDER BY id LIMIT $2`, after, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var out []UnsealedInboundSecret + for rows.Next() { + var u UnsealedInboundSecret + if err := rows.Scan(&u.ID, &u.OrganizationID, &u.Secret); err != nil { + return nil, err + } + out = append(out, u) + } + return out, rows.Err() +} + +func (r *integrationRepository) SealInboundSecret(ctx context.Context, id uuid.UUID, secret, sealed string) (bool, error) { + if secret == "" || sealed == "" { + return false, errors.New("inbound secret and its sealed copy are both required") + } + tag, err := r.db.Exec(ctx, ` + UPDATE integration_connections + SET inbound_secret_encrypted = $3, inbound_secret_digest = $4 + WHERE id = $1 AND inbound_secret = $2 AND inbound_secret_digest IS NULL`, + id, secret, sealed, InboundSecretDigest(secret)) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + // ErrInboundConnectionNotFound means no Calendly, Cal.com or Clay connection // with that id exists in the organization. var ErrInboundConnectionNotFound = errors.New("inbound connection not found") diff --git a/internal/repository/pg_orgtransfer.go b/internal/repository/pg_orgtransfer.go index adabae2f3..2b86f47d2 100644 --- a/internal/repository/pg_orgtransfer.go +++ b/internal/repository/pg_orgtransfer.go @@ -89,8 +89,8 @@ type OrgTransferRepository interface { // did not travel, so the dashboard prompts for a reconnect instead of the // worker failing every send against an empty password. MarkMailboxesNeedingReconnect(ctx context.Context, tx pgx.Tx, orgID uuid.UUID) (int64, error) - // MarkIntegrationsNeedingReconnect flags connections whose credentials did not travel. - MarkIntegrationsNeedingReconnect(ctx context.Context, tx pgx.Tx, orgID uuid.UUID, detail string) ([]IntegrationReconnect, error) + // MarkIntegrationsNeedingReconnect flags, once each, connections whose credentials or inbound secret did not travel. + MarkIntegrationsNeedingReconnect(ctx context.Context, tx pgx.Tx, orgID uuid.UUID, credentialsDetail, inboundDetail string) ([]IntegrationReconnect, error) // Begin starts the import transaction. Begin(ctx context.Context) (pgx.Tx, error) @@ -901,27 +901,42 @@ func (r *orgTransferRepository) MarkMailboxesNeedingReconnect(ctx context.Contex type IntegrationReconnect struct { Provider string Label string + // CredentialsMissing reports an API key, OAuth token or config the archive blanked. + CredentialsMissing bool + // InboundClosed reports that the connection had an inbound URL and it was closed. + InboundClosed bool } -// MarkIntegrationsNeedingReconnect moves every connection holding a credential -// the archive blanked (an empty string, where the app itself writes NULL) to -// reauth_required, and closes its inbound URL so it is re-minted on reconnect. -func (r *orgTransferRepository) MarkIntegrationsNeedingReconnect(ctx context.Context, tx pgx.Tx, orgID uuid.UUID, detail string) ([]IntegrationReconnect, error) { +// MarkIntegrationsNeedingReconnect moves every connection holding a value the archive blanked ("" where the app writes NULL) to reauth_required. +// A blank config or inbound secret closes the inbound URL in its clear, sealed and digest columns alike. +func (r *orgTransferRepository) MarkIntegrationsNeedingReconnect(ctx context.Context, tx pgx.Tx, orgID uuid.UUID, credentialsDetail, inboundDetail string) ([]IntegrationReconnect, error) { rows, err := tx.Query(ctx, ` - UPDATE integration_connections - SET status = CASE WHEN status = 'disconnected' THEN status ELSE 'reauth_required' END, + WITH stale AS ( + SELECT id, + COALESCE(config_encrypted = '\x'::bytea OR access_token_encrypted = '' OR refresh_token_encrypted = '', false) AS credentials_missing, + COALESCE(config_encrypted = '\x'::bytea OR inbound_secret_encrypted = '', false) AS close_inbound, + (inbound_secret IS NOT NULL OR inbound_secret_encrypted IS NOT NULL OR inbound_secret_digest IS NOT NULL) AS had_inbound + FROM integration_connections + WHERE organization_id = $1 + AND (config_encrypted = '\x'::bytea OR access_token_encrypted = '' OR refresh_token_encrypted = '' OR inbound_secret_encrypted = '') + FOR UPDATE + ) + UPDATE integration_connections ic + SET status = CASE WHEN ic.status = 'disconnected' THEN ic.status ELSE 'reauth_required' END, health = 'down', - health_detail = $2, + health_detail = CASE WHEN s.credentials_missing THEN $2 ELSE $3 END, health_checked_at = NOW(), - inbound_secret = CASE WHEN config_encrypted = '\x'::bytea THEN NULL ELSE inbound_secret END, - config_encrypted = NULLIF(config_encrypted, '\x'::bytea), - access_token_encrypted = NULLIF(access_token_encrypted, ''), - refresh_token_encrypted = NULLIF(refresh_token_encrypted, ''), + inbound_secret = CASE WHEN s.close_inbound THEN NULL ELSE ic.inbound_secret END, + inbound_secret_encrypted = CASE WHEN s.close_inbound THEN NULL ELSE ic.inbound_secret_encrypted END, + inbound_secret_digest = CASE WHEN s.close_inbound THEN NULL ELSE ic.inbound_secret_digest END, + config_encrypted = NULLIF(ic.config_encrypted, '\x'::bytea), + access_token_encrypted = NULLIF(ic.access_token_encrypted, ''), + refresh_token_encrypted = NULLIF(ic.refresh_token_encrypted, ''), updated_at = NOW() - WHERE organization_id = $1 - AND (config_encrypted = '\x'::bytea OR access_token_encrypted = '' OR refresh_token_encrypted = '') - RETURNING provider, label - `, orgID, detail) + FROM stale s + WHERE ic.id = s.id AND ic.organization_id = $1 + RETURNING ic.provider, ic.label, s.credentials_missing, s.close_inbound AND s.had_inbound + `, orgID, credentialsDetail, inboundDetail) if err != nil { return nil, fmt.Errorf("flag integrations needing reconnect: %w", err) } @@ -929,7 +944,7 @@ func (r *orgTransferRepository) MarkIntegrationsNeedingReconnect(ctx context.Con var out []IntegrationReconnect for rows.Next() { var c IntegrationReconnect - if err := rows.Scan(&c.Provider, &c.Label); err != nil { + if err := rows.Scan(&c.Provider, &c.Label, &c.CredentialsMissing, &c.InboundClosed); err != nil { return nil, err } out = append(out, c)