warmbly/internal/app/contact/import.go

package contact

import (
	"context"
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/warmbly/warmbly/internal/email"
	"github.com/warmbly/warmbly/internal/errx"
	"github.com/warmbly/warmbly/internal/models"
	"github.com/warmbly/warmbly/internal/utils"
	"github.com/xuri/excelize/v2"
)

// ImportPreview parses the uploaded file enough to drive the column
// mapping UI. It does NOT persist anything. The same file is uploaded
// a second time on commit; storing the parsed buffer between calls
// would either pin memory or require a tmp store, neither of which is
// worth it for the typical (small) file size.
func (s *contactService) ImportPreview(ctx context.Context, r io.Reader, filename string) (*models.ContactImportPreview, *errx.Error) {
	rows, format, xerr := parseSpreadsheet(r, filename)
	if xerr != nil {
		return nil, xerr
	}
	if len(rows) == 0 {
		return nil, errx.New(errx.BadRequest, "the uploaded file is empty")
	}

	headers, hasHeader := detectHeaders(rows[0])
	dataStart := 0
	if hasHeader {
		dataStart = 1
	}

	// Sample slice for the UI to render. Cap at preview limit.
	sampleEnd := dataStart + models.MaxContactImportPreviewRows
	if sampleEnd > len(rows) {
		sampleEnd = len(rows)
	}
	sample := make([][]string, 0, sampleEnd-dataStart)
	for i := dataStart; i < sampleEnd; i++ {
		sample = append(sample, padRow(rows[i], len(headers)))
	}

	totalRows := len(rows) - dataStart

	return &models.ContactImportPreview{
		Filename:         filename,
		Format:           format,
		TotalRows:        totalRows,
		Columns:          headers,
		HasHeader:        hasHeader,
		SampleRows:       sample,
		SuggestedMapping: suggestMapping(headers),
	}, nil
}

// ImportCommit re-parses the file and writes the upsert. We don't share
// state with ImportPreview on purpose — keeping the path stateless
// makes the commit safe to retry without an opaque "session id".
func (s *contactService) ImportCommit(
	ctx context.Context,
	userID string,
	r io.Reader,
	filename string,
	opts *models.ContactImportCommit,
) (*models.ContactImportResult, *errx.Error) {
	startedAt := time.Now().UTC()

	if opts == nil {
		return nil, errx.New(errx.BadRequest, "missing import options")
	}
	if len(opts.Mapping) == 0 {
		return nil, errx.New(errx.BadRequest, "no column mapping provided")
	}
	uid, perr := uuid.Parse(userID)
	if perr != nil {
		return nil, errx.ErrUuid
	}

	dedup := opts.Dedup
	switch dedup {
	case models.ContactImportDedupSkip,
		models.ContactImportDedupUpdate,
		models.ContactImportDedupCreateDuplicate:
	case "":
		dedup = models.ContactImportDedupSkip
	default:
		return nil, errx.New(errx.BadRequest, "unknown dedup strategy: "+string(dedup))
	}

	subscribedDefault := true
	if opts.SubscribedDefault != nil {
		subscribedDefault = *opts.SubscribedDefault
	}

	// Validate category IDs are well-formed UUIDs. Ownership scoping
	// happens later inside the repo (the INSERT joins against
	// categories.user_id) so we don't need to round-trip the DB here.
	catIDs, xerr := parseLocalCategoryIDs(opts.CategoryIDs)
	if xerr != nil {
		return nil, xerr
	}

	rows, _, xerr := parseSpreadsheet(r, filename)
	if xerr != nil {
		return nil, xerr
	}
	if len(rows) == 0 {
		return &models.ContactImportResult{
			StartedAt: startedAt,
			EndedAt:   time.Now().UTC(),
		}, nil
	}

	dataStart := 0
	if opts.HasHeader {
		dataStart = 1
	}
	data := rows[dataStart:]
	if len(data) > models.MaxContactImportRows {
		return nil, errx.New(errx.BadRequest,
			fmt.Sprintf("too many rows; max %d per import", models.MaxContactImportRows))
	}

	// Build the parsed contacts up front so we can pre-check
	// collisions in one DB round trip instead of N.
	type pendingRow struct {
		line    int
		raw     []string
		contact models.AddContact
		ok      bool
		errMsg  string
	}

	parsed := make([]pendingRow, 0, len(data))
	for i, row := range data {
		line := i + dataStart + 1 // 1-based for "open in Excel and jump"
		p := pendingRow{line: line, raw: row}

		contact, err := buildAddContact(row, opts.Mapping, subscribedDefault, opts.CampaignIDs, opts.CategoryIDs)
		if err != "" {
			p.errMsg = err
			parsed = append(parsed, p)
			continue
		}
		contact.Email = strings.TrimSpace(contact.Email)
		if contact.Email == "" || !email.IsValid(contact.Email) {
			p.errMsg = "missing or invalid email"
			parsed = append(parsed, p)
			continue
		}
		contact.Email = strings.ToLower(contact.Email)
		p.contact = contact
		p.ok = true
		parsed = append(parsed, p)
	}

	// Pre-check existing emails in one shot so we can route rows to
	// the right path (skip / update / dup).
	emails := make([]string, 0, len(parsed))
	for i := range parsed {
		if parsed[i].ok {
			emails = append(emails, parsed[i].contact.Email)
		}
	}
	existing, xerr := s.contactRepository.GetByEmailsAndUser(ctx, uid, emails)
	if xerr != nil {
		return nil, xerr
	}

	res := &models.ContactImportResult{
		Total:     len(parsed),
		StartedAt: startedAt,
		Errors:    make([]models.ContactImportRowError, 0),
	}

	// Bucket rows by target action. We send fresh inserts through
	// contactRepository.Add in batches and fall back to per-row
	// Update for the "update existing" path so we can compute the
	// merged custom_fields correctly.
	toInsert := make([]models.AddContact, 0, len(parsed))
	toInsertLines := make([]int, 0, len(parsed))
	toUpdate := make([]pendingRow, 0)

	for _, p := range parsed {
		if !p.ok {
			res.Failed++
			res.Errors = append(res.Errors, models.ContactImportRowError{
				Line: p.line, Email: p.contact.Email, Values: p.raw, Reason: p.errMsg,
			})
			continue
		}
		_, dup := existing[p.contact.Email]
		switch {
		case !dup:
			toInsert = append(toInsert, p.contact)
			toInsertLines = append(toInsertLines, p.line)
		case dedup == models.ContactImportDedupSkip:
			res.Skipped++
		case dedup == models.ContactImportDedupUpdate:
			toUpdate = append(toUpdate, p)
		case dedup == models.ContactImportDedupCreateDuplicate:
			// We can't actually create a duplicate because of the
			// unique (user_id, lower(email)) index. We treat this as
			// "update" so the data isn't lost, and surface a soft
			// warning per row. This is a deliberate, friendlier
			// behaviour than failing the whole batch.
			toUpdate = append(toUpdate, p)
		}
	}

	// Insert in chunks so a 50k row import doesn't blow up a single
	// pgx batch. 500 lines up with the Search page size.
	for start := 0; start < len(toInsert); start += 500 {
		end := start + 500
		if end > len(toInsert) {
			end = len(toInsert)
		}
		chunk := toInsert[start:end]
		inserted, xerr := s.Add(ctx, userID, chunk)
		if xerr != nil {
			// Per-row reasons are easier to act on than a "batch
			// failed" — record each as failed with the same reason.
			for i, p := range chunk {
				res.Failed++
				res.Errors = append(res.Errors, models.ContactImportRowError{
					Line:   toInsertLines[start+i],
					Email:  p.Email,
					Reason: xerr.Message,
				})
			}
			continue
		}
		res.Imported += len(inserted)
	}

	for _, p := range toUpdate {
		// Find the existing contact id and merge.
		ex := existing[p.contact.Email]
		idStr := ex.ID.String()

		update := &models.UpdateContact{
			FirstName: optString(p.contact.FirstName, ex.FirstName),
			LastName:  optString(p.contact.LastName, ex.LastName),
			Company:   optString(p.contact.Company, ex.Company),
			Phone:     optString(p.contact.Phone, ex.Phone),
		}
		if len(p.contact.CustomFields) > 0 {
			merged := make(map[string]string, len(p.contact.CustomFields))
			for k, v := range p.contact.CustomFields {
				merged[k] = v
			}
			update.CustomFields = &merged
		}
		if len(catIDs) > 0 {
			ids := make([]string, len(catIDs))
			for i, id := range catIDs {
				ids[i] = id.String()
			}
			update.AddCategories = ids
		}
		if _, xerr := s.contactRepository.Update(ctx, userID, idStr, update); xerr != nil {
			res.Failed++
			res.Errors = append(res.Errors, models.ContactImportRowError{
				Line:   p.line,
				Email:  p.contact.Email,
				Reason: xerr.Message,
			})
			continue
		}
		res.Updated++

		// Attach campaigns separately if the caller requested it.
		if len(p.contact.Campaigns) > 0 {
			if _, xerr := s.contactRepository.BulkUpdate(ctx, userID, &models.BulkEditContactsData{
				Contacts:     []string{idStr},
				AddCampaigns: p.contact.Campaigns,
			}); xerr != nil {
				// Non-fatal — the contact was updated, the link
				// failed. Surface as a row-level warning.
				res.Errors = append(res.Errors, models.ContactImportRowError{
					Line:   p.line,
					Email:  p.contact.Email,
					Reason: "contact updated but campaign link failed: " + xerr.Message,
				})
			}
		}
	}

	res.EndedAt = time.Now().UTC()
	return res, nil
}

// parseSpreadsheet returns rows as a 2-D slice and the detected format.
// CSV is decoded with the stdlib (forgiving about trailing commas /
// quoting), XLSX is decoded with excelize. Anything else 400s.
func parseSpreadsheet(r io.Reader, filename string) ([][]string, string, *errx.Error) {
	ext := strings.ToLower(filepath.Ext(filename))
	switch ext {
	case ".csv", ".tsv", ".txt", "":
		reader := csv.NewReader(r)
		reader.FieldsPerRecord = -1 // tolerate ragged rows; we pad
		reader.LazyQuotes = true
		if ext == ".tsv" {
			reader.Comma = '\t'
		}
		rows, err := reader.ReadAll()
		if err != nil {
			return nil, "csv", errx.New(errx.BadRequest, "failed to parse CSV: "+err.Error())
		}
		return rows, "csv", nil
	case ".xlsx", ".xlsm":
		f, err := excelize.OpenReader(r)
		if err != nil {
			return nil, "xlsx", errx.New(errx.BadRequest, "failed to parse XLSX: "+err.Error())
		}
		defer f.Close()
		sheetName := f.GetSheetName(f.GetActiveSheetIndex())
		if sheetName == "" {
			names := f.GetSheetList()
			if len(names) == 0 {
				return nil, "xlsx", errx.New(errx.BadRequest, "workbook has no sheets")
			}
			sheetName = names[0]
		}
		rows, err := f.GetRows(sheetName)
		if err != nil {
			return nil, "xlsx", errx.New(errx.BadRequest, "failed to read XLSX rows: "+err.Error())
		}
		return rows, "xlsx", nil
	}
	return nil, "", errx.New(errx.BadRequest, "unsupported file type: "+ext)
}

// detectHeaders applies a simple heuristic: if every cell in the first
// row looks like text (no @, no digit-heavy noise), treat it as headers.
// Users can override this in the UI; this is just the smart default.
func detectHeaders(first []string) ([]string, bool) {
	if len(first) == 0 {
		return nil, false
	}
	looksLikeHeader := true
	for _, cell := range first {
		c := strings.TrimSpace(cell)
		if c == "" {
			continue
		}
		// An "@" in the first row almost certainly means it's a data
		// row (email address) — Excel-exported CSVs sometimes ship
		// without headers at all.
		if strings.Contains(c, "@") {
			looksLikeHeader = false
			break
		}
	}
	if looksLikeHeader {
		out := make([]string, len(first))
		for i, c := range first {
			out[i] = strings.TrimSpace(c)
			if out[i] == "" {
				out[i] = "Column " + strconv.Itoa(i+1)
			}
		}
		return out, true
	}
	// No header → synthesise.
	out := make([]string, len(first))
	for i := range first {
		out[i] = "Column " + strconv.Itoa(i+1)
	}
	return out, false
}

// padRow returns a copy of `row` padded to `n` columns. Excel and Sheets
// both export ragged rows when trailing cells are empty; padding makes
// downstream code simpler.
func padRow(row []string, n int) []string {
	if len(row) >= n {
		return row[:n]
	}
	out := make([]string, n)
	copy(out, row)
	return out
}

// suggestMapping uses fuzzy header matches to pick a target for each
// column. Anything we don't recognise becomes ignore — better than
// inventing a custom-field key the user didn't ask for.
func suggestMapping(headers []string) []models.ContactImportColumnMapping {
	out := make([]models.ContactImportColumnMapping, len(headers))
	for i, h := range headers {
		out[i] = guessTarget(i, h)
	}
	return out
}

// guessTarget runs against ~the set of header aliases we've seen in the
// wild from Salesforce, HubSpot, Mailchimp, Apollo, Lemlist, raw
// gmail-contact CSVs. The match is case-insensitive + ignores spaces
// and punctuation.
func guessTarget(idx int, header string) models.ContactImportColumnMapping {
	key := strings.ToLower(header)
	key = strings.NewReplacer(" ", "", "_", "", "-", "", ".", "").Replace(key)
	switch key {
	case "email", "emailaddress", "e-mail", "mail", "emailaddress1", "primaryemail":
		return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetEmail}
	case "firstname", "givenname", "fname", "first":
		return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetFirstName}
	case "lastname", "familyname", "surname", "lname", "last":
		return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetLastName}
	case "company", "companyname", "organization", "organisation", "employer", "account", "accountname":
		return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetCompany}
	case "phone", "phonenumber", "mobile", "cell", "phone1":
		return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetPhone}
	case "subscribed", "optin", "optedin", "subscribe":
		return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetSubscribed}
	case "categories", "category", "tags", "tag", "labels", "label":
		return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetCategories}
	}
	return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetIgnore}
}

// buildAddContact applies the column mapping to a single row. Returns
// either a fully-populated AddContact or a reason string explaining why
// the row was rejected. We don't bail on the first bad field — we
// gather everything so the user sees one good error.
func buildAddContact(
	row []string,
	mapping []models.ContactImportColumnMapping,
	subscribedDefault bool,
	defaultCampaignIDs []string,
	defaultCategoryIDs []string,
) (models.AddContact, string) {
	ac := models.AddContact{
		CustomFields: map[string]string{},
		Campaigns:    append([]string{}, defaultCampaignIDs...),
		Categories:   append([]string{}, defaultCategoryIDs...),
	}
	subscribedSet := false
	for _, m := range mapping {
		if m.Index < 0 || m.Index >= len(row) {
			continue
		}
		val := strings.TrimSpace(row[m.Index])
		if val == "" {
			continue
		}
		switch m.Target {
		case models.ContactImportTargetIgnore:
			continue
		case models.ContactImportTargetEmail:
			ac.Email = val
		case models.ContactImportTargetFirstName:
			ac.FirstName = val
		case models.ContactImportTargetLastName:
			ac.LastName = val
		case models.ContactImportTargetCompany:
			ac.Company = val
		case models.ContactImportTargetPhone:
			ac.Phone = val
		case models.ContactImportTargetSubscribed:
			subscribedSet = true
			b, perr := parseBoolish(val)
			if perr != "" {
				return models.AddContact{}, perr
			}
			_ = b // not used: we don't have a way to push it into AddContact yet
		case models.ContactImportTargetCategories:
			// Comma-separated list of category names — caller could
			// also pass IDs but names are friendlier for CSV
			// round-trips. For now we ignore names from the file
			// (we'd need a lookup); the bulk category assignment
			// applied by `opts.CategoryIDs` covers the common case.
			_ = val
		default:
			if strings.HasPrefix(string(m.Target), "custom:") {
				key := strings.TrimPrefix(string(m.Target), "custom:")
				if key == "" || !utils.IsValidJSONKey(key) {
					return models.AddContact{}, "invalid custom field key: " + key
				}
				ac.CustomFields[key] = val
			}
			if m.CustomKey != "" {
				if !utils.IsValidJSONKey(m.CustomKey) {
					return models.AddContact{}, "invalid custom field key: " + m.CustomKey
				}
				ac.CustomFields[m.CustomKey] = val
			}
		}
	}
	_ = subscribedSet // AddContact doesn't carry subscribed; default applies at row creation
	_ = subscribedDefault
	return ac, ""
}

// parseBoolish accepts the strings real CSV exporters emit for boolean
// columns. Empty/unknown values are treated as default (caller decides
// what default means).
func parseBoolish(v string) (bool, string) {
	switch strings.ToLower(strings.TrimSpace(v)) {
	case "", "1", "true", "t", "yes", "y", "subscribed", "opted in", "opt-in":
		return true, ""
	case "0", "false", "f", "no", "n", "unsubscribed", "opted out", "opt-out":
		return false, ""
	}
	return false, "could not parse subscribed value: " + v
}

// optString returns a pointer to `incoming` if non-empty, else `fallback`.
// Used in the update path to avoid blanking a populated field with an
// empty CSV cell — the importer's job is to enrich, not erase.
func optString(incoming, fallback string) *string {
	if strings.TrimSpace(incoming) == "" {
		return nil
	}
	v := incoming
	_ = fallback
	return &v
}

// parseLocalCategoryIDs is the import-package twin of pg_contact's
// parseCategoryIDs. Kept private and small so we don't depend on the
// repository package's internals.
func parseLocalCategoryIDs(raw []string) ([]uuid.UUID, *errx.Error) {
	if len(raw) == 0 {
		return nil, nil
	}
	seen := make(map[uuid.UUID]struct{}, len(raw))
	out := make([]uuid.UUID, 0, len(raw))
	for _, s := range raw {
		s = strings.TrimSpace(s)
		if s == "" {
			continue
		}
		id, err := uuid.Parse(s)
		if err != nil {
			return nil, errx.ErrUuid
		}
		if _, dup := seen[id]; dup {
			continue
		}
		seen[id] = struct{}{}
		out = append(out, id)
	}
	return out, nil
}

var _ = errors.New