From 2bdbfe5f6805cd975be1b76e1d0d4aeafaf94a8a Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Tue, 22 Sep 2026 20:07:57 -0700 Subject: [PATCH] feat: list the workspace's existing custom fields in the contact import and Google Sheets column mapper (searchable, with inline create), auto-map headers to existing fields ignoring case and separators in one shared server-side suggester, flag new fields, near-duplicates and columns sharing a field, and document the matching --- docs/content/docs/api/endpoints.mdx | 4 +- docs/content/docs/api/reference/contacts.mdx | 10 +- .../docs/api/reference/integrations.mdx | 2 +- docs/content/docs/guides/contacts-crm.mdx | 14 +- docs/public/openapi.json | 1 + internal/api/handler/contact_io.go | 8 +- internal/app/contact/import.go | 70 +++- internal/app/contact/import_live_test.go | 40 +++ internal/app/contact/import_suggest_test.go | 67 ++++ internal/app/contact/service.go | 2 +- internal/app/leadsync/service.go | 38 +- .../components/app/contacts/ImportWizard.tsx | 332 ++++++++++++++---- .../app/contacts/importShared.test.ts | 45 +++ .../components/app/contacts/importShared.ts | 51 +++ 14 files changed, 572 insertions(+), 112 deletions(-) create mode 100644 internal/app/contact/import_suggest_test.go diff --git a/docs/content/docs/api/endpoints.mdx b/docs/content/docs/api/endpoints.mdx index e91399101..93a4c7d4c 100644 --- a/docs/content/docs/api/endpoints.mdx +++ b/docs/content/docs/api/endpoints.mdx @@ -125,9 +125,9 @@ The three `/campaigns/:id/leads/:contactId/…` calls hold one contact's flow in | GET | `/contacts/:id/research` | `AI_RESEARCH` | | POST | `/contacts/research/batch` | `AI_RESEARCH` | -`GET /contacts/custom-fields` lists the distinct custom-field keys used across your contacts (frequency-ranked), for building personalization pickers. It returns a flat string array under `data`, capped at 200 keys. +`GET /contacts/custom-fields` lists the distinct custom-field keys used across your contacts (frequency-ranked), for building personalization pickers and import column mappers. It returns a flat string array under `data`, capped at 200 keys. -The import pair is two steps over the same file: preview parses it and suggests a column mapping, commit applies the mapping you chose. Commit takes the stricter `BULK_CONTACTS` scope because one call writes up to 50,000 rows. A mapping problem, an unusable custom-field name or no `email` column, is a `400` on the whole request; see [contacts](/api/reference/contacts/). +The import pair is two steps over the same file: preview parses it and suggests a column mapping (matching headers to your existing custom fields), commit applies the mapping you chose. Commit takes the stricter `BULK_CONTACTS` scope because one call writes up to 50,000 rows. A mapping problem, an unusable custom-field name or no `email` column, is a `400` on the whole request; see [contacts](/api/reference/contacts/). AI contact research charges credits (2 per run, billable even when it finds nothing) and only saves cited findings. See the [AI contact research](/guides/ai-contact-research/) guide. The batch endpoint accepts up to 500 contact ids and drains in the background. diff --git a/docs/content/docs/api/reference/contacts.mdx b/docs/content/docs/api/reference/contacts.mdx index 87c89bdaa..9b83a9bc3 100644 --- a/docs/content/docs/api/reference/contacts.mdx +++ b/docs/content/docs/api/reference/contacts.mdx @@ -361,21 +361,23 @@ Send the file as `multipart/form-data` with a `file` form field. Uploads are cap "filename": "leads.csv", "format": "csv", "total_rows": 1243, - "columns": ["Email", "First", "Last", "Company"], + "columns": ["Email", "First", "Last", "Company", "industry", "Notes"], "has_header": true, "sample_rows": [ - ["dana@acme.com", "Dana", "Reyes", "Acme"] + ["dana@acme.com", "Dana", "Reyes", "Acme", "Real Estate", "Met at SaaStr"] ], "suggested_mapping": [ { "index": 0, "target": "email" }, { "index": 1, "target": "first_name" }, { "index": 2, "target": "last_name" }, - { "index": 3, "target": "company" } + { "index": 3, "target": "company" }, + { "index": 4, "target": "custom", "custom_key": "Industry" }, + { "index": 5, "target": "ignore" } ] } ``` -`sample_rows` is capped at 20 rows. `suggested_mapping` is a default the client may override. +`sample_rows` is capped at 20 rows. `suggested_mapping` is a default the client may override. A header that names one of the workspace's existing custom fields, ignoring case, spaces, underscores, dashes and dots, is suggested as `custom` with that field's stored spelling in `custom_key` (above, `industry` maps to the existing `Industry`). Each field is suggested for one column at most, standard fields are matched first, and a header that matches nothing is `ignore`. `GET /contacts/custom-fields` ([endpoints](/api/endpoints/#contacts)) lists the fields a mapping can target. ## Commit an import diff --git a/docs/content/docs/api/reference/integrations.mdx b/docs/content/docs/api/reference/integrations.mdx index ddefcc3be..f89f494e2 100644 --- a/docs/content/docs/api/reference/integrations.mdx +++ b/docs/content/docs/api/reference/integrations.mdx @@ -1018,7 +1018,7 @@ Auth: **Scope** `WRITE_CONTACTS` · **Org permission** `manage_contacts`. `POST /lead-sync/google/preview` -Returns an import-preview-shaped payload (columns, sample rows, total rows, header detection, suggested mapping) so the frontend reuses its contact-import column mapper verbatim. +Returns an import-preview-shaped payload (columns, sample rows, total rows, header detection, suggested mapping) so the frontend reuses its contact-import column mapper verbatim. The mapping is suggested exactly as the [file import preview](/api/reference/contacts/#preview-an-import) suggests it, including headers matched to the workspace's existing custom fields. Auth: **Scope** `WRITE_CONTACTS` · **Org permission** `manage_contacts`. diff --git a/docs/content/docs/guides/contacts-crm.mdx b/docs/content/docs/guides/contacts-crm.mdx index 7e3f53fc5..4fddc9a5a 100644 --- a/docs/content/docs/guides/contacts-crm.mdx +++ b/docs/content/docs/guides/contacts-crm.mdx @@ -19,7 +19,17 @@ You must map at least one column to **Email**. | First name, Last name, Company, Phone | Standard identity fields | | Subscribed | `yes`, `true`, `1`, `subscribed` and their opposites (`no`, `false`, `0`, `unsubscribed`). Applies to new contacts and to existing ones you update; a value the importer cannot read fails only that row | | Categories | Category names, separated by commas or semicolons. Names you do not have yet are created, up to 100 new names per import | -| Custom field | Anything else; you name it | +| Custom field | Anything else: one of your existing custom fields, or a new one you name | + +### Mapping into your custom fields + +The mapping menu lists the custom fields your workspace already has under **Your custom fields**, most used first, with a search box at the top. Type to filter; type a name nobody has yet and the menu offers to create it. Choosing an existing field writes the column into it, so the values line up with the contacts you already have and every template, filter and segment that reads the field picks them up. + +A column whose header names an existing field is mapped to it before you open the menu. The match ignores case, spaces, underscores, dashes and dots, so a `company url` column lands on your `company_url` field in its stored spelling instead of starting a second field. Only one column is matched to each field, and standard fields (Email, Company, and so on) are matched first. + +Each custom mapping says what it will do: **Fills your existing field**, **Creates a new field**, or, when the name you typed differs from an existing field only in case or separators, **You already have `Industry`**, with **Use it** to switch to that field. **Keep N more as custom fields** uses the same matching, so the columns it claims land on your existing fields where it can. + +Two columns may fill the same field, which is how a `Phone` and a `Mobile` column become one number. Each row names the other column, and where both have a value the one further right is kept; a blank cell never overwrites the other column's value. `Dana@Acme.com` and `dana@acme.com` are the same contact. Choose **Skip existing** (leave untouched), **Update existing** (merge new values in), or **Create duplicates** (force a new one, falling back to update if a uniqueness rule blocks it). @@ -51,7 +61,7 @@ This is not address verification. It reads the addresses; verification asks the ## Custom fields -For anything beyond identity: industry, plan tier, account owner. Create them by mapping a column to "Use as custom field" during import, or on a contact's Details tab. +For anything beyond identity: industry, plan tier, account owner. Create them by mapping a column to a new custom field during import, or on a contact's Details tab. A later import can map into the same field by picking it from the mapping menu (see [Mapping into your custom fields](#mapping-into-your-custom-fields)). They feed personalization, which is the main reason to get imports right. diff --git a/docs/public/openapi.json b/docs/public/openapi.json index aa6ec10cd..245f1137e 100644 --- a/docs/public/openapi.json +++ b/docs/public/openapi.json @@ -25204,6 +25204,7 @@ }, "suggested_mapping": { "type": "array", + "description": "A default the client may override. A header naming an existing custom field, ignoring case, spaces, underscores, dashes and dots, is suggested as target custom with the field's stored spelling in custom_key; one column per field, standard fields matched first.", "items": { "$ref": "#/components/schemas/ContactImportColumnMapping" } diff --git a/internal/api/handler/contact_io.go b/internal/api/handler/contact_io.go index 0d7837156..6b47b2708 100644 --- a/internal/api/handler/contact_io.go +++ b/internal/api/handler/contact_io.go @@ -102,6 +102,12 @@ func (h *Handler) ImportPreviewContacts(c *gin.Context) { errx.Handle(c, errx.ErrUser) return } + // The suggestion matches headers to this workspace's custom fields. + orgID := middleware.GetOrganizationID(c) + if orgID == nil { + errx.Handle(c, errx.New(errx.BadRequest, "no organization selected")) + return + } c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxImportUploadBytes) file, header, err := c.Request.FormFile("file") @@ -111,7 +117,7 @@ func (h *Handler) ImportPreviewContacts(c *gin.Context) { } defer file.Close() - preview, xerr := h.ContactService.ImportPreview(c.Request.Context(), file, header.Filename) + preview, xerr := h.ContactService.ImportPreview(c.Request.Context(), *orgID, file, header.Filename) if xerr != nil { errx.Handle(c, xerr) return diff --git a/internal/app/contact/import.go b/internal/app/contact/import.go index 36c4ea00f..14b3c1258 100644 --- a/internal/app/contact/import.go +++ b/internal/app/contact/import.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "time" + "unicode" "github.com/google/uuid" "github.com/warmbly/warmbly/internal/app/orgrisk" @@ -35,7 +36,7 @@ const ( xlsxUnzipXMLLimitBytes = 64 << 20 // 64 MiB for any single XML part ) -func (s *contactService) ImportPreview(ctx context.Context, r io.Reader, filename string) (*models.ContactImportPreview, *errx.Error) { +func (s *contactService) ImportPreview(ctx context.Context, orgID uuid.UUID, r io.Reader, filename string) (*models.ContactImportPreview, *errx.Error) { rows, format, xerr := parseSpreadsheet(r, filename) if xerr != nil { return nil, xerr @@ -69,10 +70,21 @@ func (s *contactService) ImportPreview(ctx context.Context, r io.Reader, filenam Columns: headers, HasHeader: hasHeader, SampleRows: sample, - SuggestedMapping: suggestMapping(headers, sample), + SuggestedMapping: SuggestMapping(headers, sample, s.existingCustomFieldKeys(ctx, orgID)), }, nil } +// existingCustomFieldKeys is the workspace's custom-field keys for the +// suggester. A failed read only costs the suggestion, never the preview. +func (s *contactService) existingCustomFieldKeys(ctx context.Context, orgID uuid.UUID) []string { + keys, err := s.contactRepository.DistinctCustomFieldKeys(ctx, orgID) + if err != nil { + log.Warn().Str("organization_id", orgID.String()).Msg("could not read custom field keys for the import suggestion") + return nil + } + return keys +} + // importColumn is one validated mapping entry: exactly one destination // for one column index. Building these up front means a bad mapping is a // single actionable 400 instead of the same message repeated once per row. @@ -873,10 +885,12 @@ func padRow(row []string, n int) []string { return out } -// suggestMapping uses fuzzy header matches to pick a target for each -// column. Anything we don't recognise becomes ignore — better than -// inventing a custom-field key the user didn't ask for. -func suggestMapping(headers []string, sample [][]string) []models.ContactImportColumnMapping { +// SuggestMapping picks a target for each column from its header and sample: +// a standard field, a verification verdict, or a custom field the workspace +// already has (existingKeys, most used first). Anything else becomes ignore, +// better than inventing a custom-field key the user didn't ask for. Every +// importer that shows a column mapper calls this, so they suggest alike. +func SuggestMapping(headers []string, sample [][]string, existingKeys []string) []models.ContactImportColumnMapping { out := make([]models.ContactImportColumnMapping, len(headers)) for i, h := range headers { out[i] = guessTarget(i, h) @@ -909,9 +923,53 @@ func suggestMapping(headers []string, sample [][]string) []models.ContactImportC } out[i] = models.ContactImportColumnMapping{Index: i, Target: models.ContactImportTargetVerificationStatus, VerificationProvider: provider} } + matchExistingCustomFields(out, headers, existingKeys) return out } +// matchExistingCustomFields maps each still-ignored column whose header names +// an existing custom field, ignoring case and separators, onto that field's +// stored spelling, so "industry" in a file lands on "Industry" instead of +// starting a second field. Each field is claimed by one column at most. +func matchExistingCustomFields(out []models.ContactImportColumnMapping, headers, existingKeys []string) { + if len(existingKeys) == 0 { + return + } + byFold := make(map[string]string, len(existingKeys)) + for _, k := range existingKeys { + f := FoldCustomFieldKey(k) + if _, taken := byFold[f]; f != "" && !taken { + byFold[f] = k + } + } + claimed := make(map[string]bool, len(out)) + for i, h := range headers { + if out[i].Target != models.ContactImportTargetIgnore { + continue + } + f := FoldCustomFieldKey(h) + key, ok := byFold[f] + if !ok || claimed[f] { + continue + } + claimed[f] = true + out[i] = models.ContactImportColumnMapping{Index: i, Target: models.ContactImportTargetCustom, CustomKey: key} + } +} + +// FoldCustomFieldKey reduces a header or field name to lowercase letters and +// digits, the form two spellings of one field ("Company URL", "company_url") +// share. +func FoldCustomFieldKey(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(s) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + } + } + return b.String() +} + // guessTarget runs against ~the set of header aliases we've seen in the // wild from Salesforce, HubSpot, Mailchimp, Apollo, Lemlist, raw // gmail-contact CSVs. The match is case-insensitive + ignores spaces diff --git a/internal/app/contact/import_live_test.go b/internal/app/contact/import_live_test.go index 1f063c5c8..7b5c3260a 100644 --- a/internal/app/contact/import_live_test.go +++ b/internal/app/contact/import_live_test.go @@ -885,3 +885,43 @@ func TestLiveImportReportsAPinThatDidNotLand(t *testing.T) { t.Fatalf("first note = %+v, want the segment-pin reason", first) } } + +// Issue #649: a second import lands on the custom fields the first one made, +// in their stored spelling, instead of starting near-duplicate fields. +func TestLiveImportPreviewSuggestsExistingCustomFields(t *testing.T) { + f := newImportFixture(t) + if _, msg := f.commit(t, "Email,Industry,Company URL\ndana@acme.com,Real Estate,https://acme.com\n", &models.ContactImportCommit{ + Mapping: []models.ContactImportColumnMapping{col(0, models.ContactImportTargetEmail), customCol(1, "Industry"), customCol(2, "company_url")}, + Dedup: models.ContactImportDedupSkip, HasHeader: true, + }); msg != "" { + t.Fatalf("seed import: %s", msg) + } + + preview, xerr := f.svc.ImportPreview(context.Background(), f.org, + strings.NewReader("email,industry,company-url,Notes\nlee@beta.io,SaaS,https://beta.io,hi\n"), "next.csv") + if xerr != nil { + t.Fatalf("preview: %s", xerr.Message) + } + want := []models.ContactImportColumnMapping{ + col(0, models.ContactImportTargetEmail), + customCol(1, "Industry"), + customCol(2, "company_url"), + col(3, models.ContactImportTargetIgnore), + } + for i, w := range want { + if preview.SuggestedMapping[i] != w { + t.Errorf("column %d: got %+v, want %+v", i, preview.SuggestedMapping[i], w) + } + } + + // Another workspace's fields are never suggested. + other := newImportFixture(t) + preview, xerr = other.svc.ImportPreview(context.Background(), other.org, + strings.NewReader("email,industry\nlee@beta.io,SaaS\n"), "next.csv") + if xerr != nil { + t.Fatalf("preview: %s", xerr.Message) + } + if got := preview.SuggestedMapping[1].Target; got != models.ContactImportTargetIgnore { + t.Fatalf("another workspace's field was suggested: %q", got) + } +} diff --git a/internal/app/contact/import_suggest_test.go b/internal/app/contact/import_suggest_test.go new file mode 100644 index 000000000..c2cbda4ab --- /dev/null +++ b/internal/app/contact/import_suggest_test.go @@ -0,0 +1,67 @@ +package contact + +import ( + "testing" + + "github.com/warmbly/warmbly/internal/models" +) + +func TestSuggestMappingMatchesExistingCustomFields(t *testing.T) { + headers := []string{"Email", "industry", "Company URL", "Total Score", "Company", "INDUSTRY", "Notes"} + existing := []string{"Industry", "company_url", "industry", "Company"} + + got := SuggestMapping(headers, nil, existing) + + want := []models.ContactImportColumnMapping{ + {Index: 0, Target: models.ContactImportTargetEmail}, + // The stored spelling wins, and the most used of two spellings. + {Index: 1, Target: models.ContactImportTargetCustom, CustomKey: "Industry"}, + {Index: 2, Target: models.ContactImportTargetCustom, CustomKey: "company_url"}, + // No field by that name: left for the user, never invented. + {Index: 3, Target: models.ContactImportTargetIgnore}, + // A standard field outranks a custom field of the same name. + {Index: 4, Target: models.ContactImportTargetCompany}, + // One column per field; a second spelling of it stays ignored. + {Index: 5, Target: models.ContactImportTargetIgnore}, + {Index: 6, Target: models.ContactImportTargetIgnore}, + } + if len(got) != len(want) { + t.Fatalf("got %d mappings, want %d", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("column %d (%q): got %+v, want %+v", i, headers[i], got[i], want[i]) + } + } +} + +func TestSuggestMappingKeepsVerificationAheadOfCustomFields(t *testing.T) { + headers := []string{"email", "ZeroBounce Status"} + got := SuggestMapping(headers, [][]string{{"a@x.com", "valid"}}, []string{"ZeroBounce Status"}) + if got[1].Target != models.ContactImportTargetVerificationStatus { + t.Fatalf("verdict column mapped to %q, want verification_status", got[1].Target) + } +} + +func TestSuggestMappingWithoutExistingFields(t *testing.T) { + got := SuggestMapping([]string{"email", "Industry"}, nil, nil) + if got[1].Target != models.ContactImportTargetIgnore { + t.Fatalf("unknown header mapped to %q, want ignore", got[1].Target) + } +} + +func TestFoldCustomFieldKey(t *testing.T) { + for in, want := range map[string]string{ + "Company URL": "companyurl", + "company_url": "companyurl", + "company-url": "companyurl", + " Company.URL ": "companyurl", + "Revenue ($)": "revenue", + "###": "", + "Straße Nummer2": "straßenummer2", + } { + if got := FoldCustomFieldKey(in); got != want { + t.Errorf("FoldCustomFieldKey(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/app/contact/service.go b/internal/app/contact/service.go index 5b1a36fdf..31d4f6801 100644 --- a/internal/app/contact/service.go +++ b/internal/app/contact/service.go @@ -37,7 +37,7 @@ type ContactService interface { // ImportPreview parses an uploaded CSV/XLSX file and reports back // the columns + first N rows + suggested mapping — no DB writes. - ImportPreview(ctx context.Context, file io.Reader, filename string) (*models.ContactImportPreview, *errx.Error) + ImportPreview(ctx context.Context, orgID uuid.UUID, file io.Reader, filename string) (*models.ContactImportPreview, *errx.Error) // ValidateImportMapping reports whether a column mapping is usable: // exactly the checks ImportCommit runs before it touches a row. Callers diff --git a/internal/app/leadsync/service.go b/internal/app/leadsync/service.go index 23303128b..8bf2ec7cb 100644 --- a/internal/app/leadsync/service.go +++ b/internal/app/leadsync/service.go @@ -110,7 +110,7 @@ func (s *service) Preview(ctx context.Context, orgID, connID uuid.UUID, sheetID, Columns: headers, HasHeader: true, SampleRows: sample, - SuggestedMapping: suggestMapping(headers), + SuggestedMapping: contact.SuggestMapping(headers, sample, s.existingCustomFieldKeys(ctx, orgID)), }, nil } @@ -415,34 +415,12 @@ func padRow(row []string, n int) []string { return out } -// suggestMapping applies the same fuzzy header heuristics the contact importer -// uses so the dashboard's preview arrives with sensible defaults. -func suggestMapping(headers []string) []models.ContactImportColumnMapping { - out := make([]models.ContactImportColumnMapping, len(headers)) - for i, h := range headers { - out[i] = guessTarget(i, h) +// existingCustomFieldKeys is the workspace's custom-field keys for the +// suggester. A failed read only costs the suggestion, never the preview. +func (s *service) existingCustomFieldKeys(ctx context.Context, orgID uuid.UUID) []string { + keys, xerr := s.contacts.ListCustomFieldKeys(ctx, orgID) + if xerr != nil { + return nil } - return out -} - -func guessTarget(idx int, header string) models.ContactImportColumnMapping { - key := strings.ToLower(header) - key = strings.NewReplacer(" ", "", "_", "", "-", "", ".", "").Replace(key) - switch key { - case "email", "emailaddress", "mail", "primaryemail": - return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetEmail} - case "firstname", "givenname", "fname", "first": - return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetFirstName} - case "lastname", "familyname", "surname", "lname", "last": - return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetLastName} - case "company", "companyname", "organization", "organisation", "employer", "account", "accountname": - return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetCompany} - case "phone", "phonenumber", "mobile", "cell": - return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetPhone} - case "subscribed", "optin", "optedin", "subscribe": - return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetSubscribed} - case "categories", "category", "tags", "tag", "labels", "label": - return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetCategories} - } - return models.ContactImportColumnMapping{Index: idx, Target: models.ContactImportTargetIgnore} + return keys } diff --git a/web/src/components/app/contacts/ImportWizard.tsx b/web/src/components/app/contacts/ImportWizard.tsx index 75e53aed9..3c12b04ef 100644 --- a/web/src/components/app/contacts/ImportWizard.tsx +++ b/web/src/components/app/contacts/ImportWizard.tsx @@ -24,10 +24,13 @@ import { AlertTriangleIcon, ArrowLeftIcon, ArrowRightIcon, + BracesIcon, CheckCircle2Icon, + CheckIcon, DownloadIcon, FileSpreadsheetIcon, Loader2Icon, + PlusIcon, ShieldCheckIcon, UploadCloudIcon, XIcon, @@ -56,16 +59,23 @@ import CategoryPicker from "./CategoryPicker"; import { CampaignMultiPicker, SegmentMultiPicker } from "@/components/app/segments/SegmentPickers"; import { useSegments } from "@/lib/api/hooks/app/segments"; import { downloadBlob } from "@/lib/api/client/app/contacts/exportContacts"; +import useCustomFieldKeys from "@/lib/api/hooks/app/contacts/useCustomFieldKeys"; import { CUSTOM_KEY_RULES, DEDUP_OPTIONS, STANDARD_TARGETS, VERIFICATION_VOCABULARY_LABELS, + customKeyOf, + customKeyStatus, describeError, + foldCustomKey, isCustomTarget, isValidCustomKey, mappingProblem, + matchExistingKey, + normalizeCustomKey, suggestCustomKey, + targetIdentity, } from "./importShared"; interface Props { @@ -516,16 +526,43 @@ export function MapStep({ return mapping.find((m) => m.index === idx) ?? { index: idx, target: "ignore" }; } + const { data: existingKeys = [] } = useCustomFieldKeys(); + + // Which columns write to each destination, so a row can say when another + // column shares its field. Column numbers are 1-based, as on screen. + const writers = new Map(); + for (const m of mapping) { + const id = targetIdentity(m); + if (id === null || m.index >= preview.columns.length) continue; + writers.set(id, [...(writers.get(id) ?? []), m.index + 1]); + } + function takenKeysFor(idx: number): Map { + const taken = new Map(); + for (const [id, cols] of writers) { + const other = cols.find((c) => c !== idx + 1); + if (id.startsWith("custom:") && other !== undefined) taken.set(id.slice(7), other); + } + return taken; + } + // Columns we didn't recognise default to Ignore, which means a CRM export // with a dozen extra columns is a dozen dropdowns. Offer the obvious bulk - // action for the ones whose header is already a usable field name. + // action for the ones whose header is already a usable field name, landing + // on the workspace's own spelling when the field already exists. // Only with a real header row: without one the columns are synthesised // ("Column 4"), which is a legal field name but never the one you want. - const claimable = !hasHeader - ? [] - : preview.columns - .map((header, idx) => ({ idx, key: suggestCustomKey(header) })) - .filter(({ idx, key }) => key !== "" && getMapping(idx).target === "ignore"); + const claimable: { idx: number; key: string }[] = []; + if (hasHeader) { + const claimed = new Set([...writers.keys()]); + preview.columns.forEach((header, idx) => { + if (getMapping(idx).target !== "ignore") return; + const key = matchExistingKey(header, existingKeys) ?? suggestCustomKey(header); + // A field another column already fills is left for the user to decide. + if (key === "" || claimed.has(`custom:${key}`)) return; + claimed.add(`custom:${key}`); + claimable.push({ idx, key }); + }); + } function claimAllAsCustom() { setMapping((cur) => { @@ -597,6 +634,9 @@ export function MapStep({ value={m} header={col} onChange={(next) => updateMapping(idx, next)} + existingKeys={existingKeys} + takenKeys={takenKeysFor(idx)} + writers={writers.get(targetIdentity(m) ?? "") ?? []} /> {m.target === "verification_status" && ( @@ -625,79 +665,241 @@ export function MapStep({ ); } +// MappingNote says what a custom mapping will do to the workspace's fields +// (fill an existing one, create one, or nearly duplicate one) and when another +// column writes to the same place. +function MappingNote({ + mapping, + existingKeys, + writers, + column, + onUseExisting, +}: { + mapping: ImportColumnMapping; + existingKeys: string[]; + // Every column writing where this one does, in the order the importer + // applies them: the last non-empty cell is the one kept. + writers: number[]; + column: number; + onUseExisting: (key: string) => void; +}) { + const sharedWith = writers.filter((c) => c !== column); + const kept = writers[writers.length - 1]; + const key = customKeyOf(mapping); + const status = isCustomTarget(mapping.target) && isValidCustomKey(key) ? customKeyStatus(key, existingKeys) : null; + if (!status && sharedWith.length === 0) return null; + return ( +
+ {status?.kind === "existing" && ( +

+ + Fills your existing field +

+ )} + {status?.kind === "new" && ( +

+ + Creates a new field +

+ )} + {status?.kind === "similar" && ( +

+ + + You already have {status.existing}. + + +

+ )} + {sharedWith.length > 0 && ( +

+ Also filled by column {sharedWith.join(", ")}. {sharedWith.length === 1 ? "When both have a value" : "When several do"}, column {kept}'s is kept. +

+ )} +
+ ); +} + export function TargetPicker({ value, onChange, header, + existingKeys = [], + takenKeys, + writers = [], }: { value: ImportColumnMapping; onChange: (next: ImportColumnMapping) => void; /** The column's header, used to pre-fill the custom-field name. */ header?: string; + /** The workspace's custom fields, most used first. */ + existingKeys?: string[]; + /** Custom fields other columns already write to, with that column's number. */ + takenKeys?: Map; + /** Every column writing where this one does, in the order they are applied. */ + writers?: number[]; }) { - // Custom-field rows are tagged with target="custom" (sentinel); the - // user-typed name lives in custom_key. We also accept the legacy - // "custom:" form in case a saved mapping comes in that shape. + const [open, setOpen] = React.useState(false); + const [query, setQuery] = React.useState(""); + // Set once the user names a new field, so the name box stays put when + // what they type happens to match an existing field. + const [naming, setNaming] = React.useState(false); + const isCustom = isCustomTarget(value.target.toString()); + const customKey = isCustom ? customKeyOf(value) : ""; + const rawKey = value.custom_key ?? customKey; + const keyExists = isCustom && existingKeys.includes(customKey); + const isExisting = keyExists && !naming; + const keyInvalid = isCustom && rawKey.trim() !== "" && !isValidCustomKey(rawKey); const stdLabel = STANDARD_TARGETS.find((t) => t.id === value.target)?.label; - const customKey = value.custom_key ?? ""; - const keyInvalid = isCustom && customKey.trim() !== "" && !isValidCustomKey(customKey); - const label = isCustom - ? customKey - ? `Custom: ${customKey}` - : "Custom field…" - : stdLabel ?? "Ignore"; + const label = isExisting ? customKey : isCustom ? (keyExists ? "Existing field" : "New field") : stdLabel ?? "Ignore"; + + const q = normalizeCustomKey(query); + const fq = foldCustomKey(q); + const matches = (text: string) => fq === "" || foldCustomKey(text).includes(fq); + const standard = STANDARD_TARGETS.filter((t) => matches(t.label)); + const custom = existingKeys.filter(matches); + // Typing a name nobody has yet offers to create it, the way a tag input does. + const canCreate = q !== "" && isValidCustomKey(q) && !existingKeys.includes(q); + + function setMenuOpen(o: boolean) { + setOpen(o); + if (!o) setQuery(""); + } + + function pickStandard(id: string) { + setNaming(false); + onChange({ index: value.index, target: id }); + } + + function pickExisting(key: string) { + setNaming(false); + onChange({ index: value.index, target: "custom", custom_key: key }); + } + + function pickNew(name: string) { + setNaming(true); + onChange({ index: value.index, target: "custom", custom_key: name }); + } + + function pickFirst() { + if (standard.length > 0) pickStandard(standard[0].id); + else if (custom.length > 0) pickExisting(custom[0]); + else if (canCreate) pickNew(q); + else return; + setMenuOpen(false); + } return ( -
- - - - - - Standard - {STANDARD_TARGETS.map((t) => ( - - onChange({ index: value.index, target: t.id }) - } - > - {t.label} - - ))} - Custom - - onChange({ - index: value.index, - target: "custom", - // Start from the column header: "Company Mobile" - // is a valid field name, so there is nothing to - // type in the common case. - custom_key: customKey || suggestCustomKey(header ?? ""), - }) - } - > - Use as custom field… - - - - {isCustom && ( - - onChange({ index: value.index, target: "custom", custom_key: v }) - } - placeholder="field name" - invalid={keyInvalid} - title={keyInvalid ? CUSTOM_KEY_RULES : undefined} - className="w-24 md:w-32" - /> - )} -
+ <> +
+ + + : isCustom ? : undefined} + label={label} + title={keyExists ? `Existing custom field: ${customKey}` : undefined} + className="flex-1 min-w-0" + /> + + +
+ setQuery(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + pickFirst(); + } + }} + placeholder={existingKeys.length > 0 ? "Search or name a new field…" : "Search or name a field…"} + autoFocus + aria-label="Search fields" + className="w-full h-5 bg-transparent text-[16px] md:text-[12px] text-slate-900 placeholder:text-slate-400 outline-none" + /> +
+
+ {standard.length > 0 && Standard} + {standard.map((t) => ( + pickStandard(t.id)} + > + {t.label} + + ))} + {custom.length > 0 && Your custom fields} + {custom.map((key) => { + const takenBy = takenKeys?.get(key); + return ( + } + selected={isCustom && customKey === key} + onSelect={() => pickExisting(key)} + trailing={ + isCustom && customKey === key ? undefined : takenBy !== undefined ? ( + column {takenBy} + ) : undefined + } + > + {key} + + ); + })} + New + {canCreate ? ( + } onSelect={() => pickNew(q)}> + Create “{q}” + + ) : ( + } + selected={isCustom && !isExisting} + // Start from the column header: "Company Mobile" + // is a valid field name, so there is nothing to + // type in the common case. + onSelect={() => pickNew(isCustom && !isExisting ? rawKey : suggestCustomKey(header ?? ""))} + > + New custom field… + + )} + {q !== "" && !canCreate && standard.length === 0 && custom.length === 0 && ( +

{CUSTOM_KEY_RULES}

+ )} +
+ + + {isCustom && !isExisting && ( + { + setNaming(true); + onChange({ index: value.index, target: "custom", custom_key: v }); + }} + placeholder="field name" + invalid={keyInvalid} + title={keyInvalid ? CUSTOM_KEY_RULES : undefined} + className="w-24 md:w-32" + /> + )} +
+ + ); } diff --git a/web/src/components/app/contacts/importShared.test.ts b/web/src/components/app/contacts/importShared.test.ts index 3abb510ad..cc1235383 100644 --- a/web/src/components/app/contacts/importShared.test.ts +++ b/web/src/components/app/contacts/importShared.test.ts @@ -5,10 +5,15 @@ import { describe, it, expect } from "vitest"; import { + customKeyOf, + customKeyStatus, + foldCustomKey, isValidCustomKey, mappingProblem, + matchExistingKey, normalizeCustomKey, suggestCustomKey, + targetIdentity, } from "./importShared"; import type { ImportColumnMapping } from "@/lib/api/client/app/contacts/importContacts"; @@ -68,3 +73,43 @@ describe("mappingProblem", () => { expect(mappingProblem([email, { index: 2, target: "custom:plan/tier" }])).toContain("plan/tier"); }); }); + +describe("existing custom fields", () => { + const existing = ["Industry", "company_url", "industry", "Total Score"]; + + it("finds the field a header names, ignoring case and separators", () => { + expect(matchExistingKey("Industry", existing)).toBe("Industry"); + expect(matchExistingKey("INDUSTRY", existing)).toBe("Industry"); + expect(matchExistingKey("industry", existing)).toBe("industry"); + expect(matchExistingKey("Company URL", existing)).toBe("company_url"); + expect(matchExistingKey("total-score", existing)).toBe("Total Score"); + expect(matchExistingKey("Website", existing)).toBeUndefined(); + expect(matchExistingKey("###", existing)).toBeUndefined(); + }); + + it("tells an existing field from a near-duplicate and a new one", () => { + expect(customKeyStatus("company_url", existing)).toEqual({ kind: "existing" }); + expect(customKeyStatus("Company-URL", existing)).toEqual({ kind: "similar", existing: "company_url" }); + expect(customKeyStatus("Website", existing)).toEqual({ kind: "new" }); + }); + + it("folds the way the server does", () => { + expect(foldCustomKey(" Company.URL ")).toBe("companyurl"); + expect(foldCustomKey("Revenue ($)")).toBe("revenue"); + }); + + it("reads both spellings of a custom mapping", () => { + expect(customKeyOf({ index: 1, target: "custom", custom_key: " Company Mobile " })).toBe("Company Mobile"); + expect(customKeyOf({ index: 1, target: "custom:Role" })).toBe("Role"); + expect(customKeyOf({ index: 1, target: "custom:Role", custom_key: "" })).toBe("Role"); + expect(customKeyOf({ index: 1, target: "email" })).toBe(""); + }); + + it("names where a mapping writes, except targets that take many columns", () => { + expect(targetIdentity({ index: 1, target: "custom", custom_key: "Industry" })).toBe("custom:Industry"); + expect(targetIdentity({ index: 1, target: "phone" })).toBe("phone"); + expect(targetIdentity({ index: 1, target: "categories" })).toBeNull(); + expect(targetIdentity({ index: 1, target: "ignore" })).toBeNull(); + expect(targetIdentity({ index: 1, target: "custom", custom_key: "" })).toBeNull(); + }); +}); diff --git a/web/src/components/app/contacts/importShared.ts b/web/src/components/app/contacts/importShared.ts index 0a2189b5f..fce8782ad 100644 --- a/web/src/components/app/contacts/importShared.ts +++ b/web/src/components/app/contacts/importShared.ts @@ -114,6 +114,57 @@ export function isCustomTarget(target: string): boolean { return target === "custom" || target.startsWith("custom:"); } +// customKeyOf is the field a custom mapping writes to, reading the legacy +// "custom:" spelling too. "" for a non-custom mapping. +export function customKeyOf(m: ImportColumnMapping): string { + if (!isCustomTarget(m.target)) return ""; + const explicit = normalizeCustomKey(m.custom_key ?? ""); + return explicit || normalizeCustomKey(m.target.startsWith("custom:") ? m.target.slice(7) : ""); +} + +// Mirrors contact.FoldCustomFieldKey: the form two spellings of one field +// ("Company URL", "company_url") share. +export function foldCustomKey(key: string): string { + return key.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, ""); +} + +// matchExistingKey finds the workspace field a header or typed name refers to: +// the exact name first, then one that differs only in case or separators. +// `existing` is most-used first, so of two such spellings the common one wins. +export function matchExistingKey(name: string, existing: string[]): string | undefined { + const n = normalizeCustomKey(name); + if (n === "") return undefined; + if (existing.includes(n)) return n; + const f = foldCustomKey(n); + if (f === "") return undefined; + return existing.find((k) => foldCustomKey(k) === f); +} + +export type CustomKeyStatus = + | { kind: "existing" } + | { kind: "similar"; existing: string } + | { kind: "new" }; + +// customKeyStatus says whether a custom mapping writes into a field the +// workspace already has, a near-duplicate of one, or a brand new field. +export function customKeyStatus(key: string, existing: string[]): CustomKeyStatus { + const k = normalizeCustomKey(key); + if (existing.includes(k)) return { kind: "existing" }; + const match = matchExistingKey(k, existing); + return match ? { kind: "similar", existing: match } : { kind: "new" }; +} + +// targetIdentity names where a mapping writes, so two columns writing to the +// same place can be spotted. null for targets that take any number of columns. +export function targetIdentity(m: ImportColumnMapping): string | null { + if (m.target === "ignore" || m.target === "categories") return null; + if (isCustomTarget(m.target)) { + const key = customKeyOf(m); + return key ? `custom:${key}` : null; + } + return m.target; +} + // mappingProblem returns the first reason the mapping cannot be committed, or // null when it is good to go. Same order of checks as the server so the two // never disagree about which column is at fault.