diff --git a/docs/content/docs/api/endpoints.mdx b/docs/content/docs/api/endpoints.mdx index 409327872..971c2ca72 100644 --- a/docs/content/docs/api/endpoints.mdx +++ b/docs/content/docs/api/endpoints.mdx @@ -204,7 +204,7 @@ AI contact research charges credits (2 per run, billable even when it finds noth Leads are prospect contacts, so the [lead finder](/guides/lead-finder/) uses the contact scopes and the **View contacts** and **Manage contacts** member permissions. `POST /leads/search` runs one page of a source search over an `area` (`bbox`, `circle` or `polygon`) and stores what it finds; send `search_id` alone to fetch the next page of the same search. `POST /leads/query`, `/leads/points` and `/leads/facets` are reads with a `filter` body; `query` pages with an opaque `cursor` and a `limit` from 1 to 200 and answers `data` plus `pagination` with a `total`. Bulk calls take a selection: `lead_ids` (at most 5,000), or `all: true` with a `filter`. -`POST /leads/enrich` starts a background run that reads each lead's website for emails, socials and (unless `people` is `false`) the people it names, then works out and checks their addresses, and answers `202` with the run; follow it at `/leads/enrichments/:id`. It costs one credit per website read, refunded when the site never answers. `GET /leads/:id` carries the lead's `people`. `POST /leads/:id/people` adds a person by `name` or `first_name` and `last_name`, and without an `email` works out their address at the lead's domain on the spot for one credit, refunded when none turns up; `find-email` does the same for someone already listed. People are matched by name, so adding someone again lands on the same person. `POST /leads/contacts` takes `who` (`business`, `people` or `both`), `include_guessed`, `segment_ids`, `category_ids`, `campaign_ids`, `skip_active_in_other_campaigns` and `keep_existing` (leave contacts you already have as they are); `POST /leads/export` takes `rows` (`leads` or `people`). A search through a source the instance pays for costs one credit a page. `POST /leads/search` and `/leads/enrich` honor `Idempotency-Key`; the other writes set values or update existing contacts, so a retry lands on the same state. `POST /leads/export` answers the file itself (`format` is `csv`, `xlsx` or `json`). +`POST /leads/enrich` starts a background run that reads each lead's website for emails, socials and (unless `people` is `false`) the people it names, then works out and checks their addresses, and answers `202` with the run; follow it at `/leads/enrichments/:id`. It costs one credit per website read, refunded when the site never answers. `GET /leads/:id` carries the lead's `people`. `POST /leads/:id/people` adds a person by `name` or `first_name` and `last_name`, and without an `email` works out their address at the lead's domain on the spot for one credit, refunded when none turns up; `find-email` does the same for someone already listed. People are matched by name, so adding someone again lands on the same person. `POST /leads/contacts` takes `who` (`business`, `people` or `both`), `include_guessed`, `segment_ids`, `category_ids`, `campaign_ids`, `skip_active_in_other_campaigns` and `keep_existing` (leave contacts you already have as they are); `POST /leads/export` takes `rows` (`leads` or `people`). A search through a source the instance pays for costs one credit a page. `POST /leads/search` and `/leads/enrich` honor `Idempotency-Key`. A paid address lookup, `find-email` or adding a person without an `email`, charges again when retried, so retry it only after reading the person back. The other writes set values or update existing contacts, so a retry lands on the same state. `POST /leads/export` answers the file itself (`format` is `csv`, `xlsx` or `json`). ### Segments diff --git a/docs/content/docs/guides/lead-finder.mdx b/docs/content/docs/guides/lead-finder.mdx index 48bb2f72c..d0f801331 100644 --- a/docs/content/docs/guides/lead-finder.mdx +++ b/docs/content/docs/guides/lead-finder.mdx @@ -147,7 +147,7 @@ A lead without an email is left out, as is a suppressed address. **Only verified ## Export -**Export** downloads the selected leads, or every lead matching the filter, as CSV, Excel or JSON. Pick the columns: every field on this page is available, including all addresses found, the people found, socials, the website's platform and signals, the score, status, tags and notes. +**Export** downloads the selected leads, or every lead matching the filter, as CSV, Excel or JSON. Pick the columns: every field on this page is available, including every address found other than invalid or suppressed ones, the people found with their addresses unless suppressed, socials, the website's platform and signals, the score, status, tags and notes. Choose **One row per person** to get a row for each person found instead, starting with their name, first and last name, title, seniority, address, its status and confidence, LinkedIn, phone and the page they were found on, followed by the business columns you picked. Businesses with nobody found are left out of that file. diff --git a/internal/app/leadfinder/crawl.go b/internal/app/leadfinder/crawl.go index d0b9c4974..91e8d0a16 100644 --- a/internal/app/leadfinder/crawl.go +++ b/internal/app/leadfinder/crawl.go @@ -440,7 +440,10 @@ func parsePage(src string) *page { p.phones = append(p.phones, d) } case strings.Contains(lower, "/cdn-cgi/l/email-protection#"): - addEmail(decodeCloudflareEmail(href[strings.Index(lower, "#")+1:])) + // Sliced at the fragment of href itself: lowercasing can change byte lengths. + if i := strings.IndexByte(href, '#'); i >= 0 { + addEmail(decodeCloudflareEmail(href[i+1:])) + } case href != "" && !strings.HasPrefix(lower, "javascript:"): p.links = append(p.links, link{href: href}) if tt == html.StartTagToken { diff --git a/internal/app/leadfinder/export.go b/internal/app/leadfinder/export.go index 80b34a51b..e76469499 100644 --- a/internal/app/leadfinder/export.go +++ b/internal/app/leadfinder/export.go @@ -353,7 +353,8 @@ func writeXLSX(w io.Writer, t *exportTable) error { continue } } - row[i] = cellSafe(v) + // Excel stores these as text, never formulas, so they are written as they are. + row[i] = v } cell, _ := excelize.CoordinatesToCellName(1, r+2) if err := sw.SetRow(cell, row); err != nil { diff --git a/internal/app/leadfinder/people_service.go b/internal/app/leadfinder/people_service.go index b7040047b..149dffb92 100644 --- a/internal/app/leadfinder/people_service.go +++ b/internal/app/leadfinder/people_service.go @@ -493,7 +493,13 @@ func (s *Service) findAndSave(ctx context.Context, orgID uuid.UUID, l *models.Le log.Warn().Err(err).Msg("leadfinder: could not store a domain's address pattern") } } - return s.savePerson(ctx, orgID, l.ID, p) + saved, err := s.savePerson(ctx, orgID, l.ID, p) + if err != nil && searching && s.metered() && found { + if _, rerr := s.Credits.RefundCharge(context.WithoutCancel(pctx), orgID, key, "lead_person_email_refund"); rerr != nil { + log.Warn().Err(rerr).Msg("leadfinder: could not refund a person search that was not saved") + } + } + return saved, err } func (s *Service) savePerson(ctx context.Context, orgID, leadID uuid.UUID, p models.LeadPerson) (*models.LeadPerson, error) { diff --git a/internal/app/leadfinder/service.go b/internal/app/leadfinder/service.go index e3fc4eb0b..5dd0cab4f 100644 --- a/internal/app/leadfinder/service.go +++ b/internal/app/leadfinder/service.go @@ -297,7 +297,8 @@ func (s *Service) Geocode(ctx context.Context, q string, near *models.LatLng, la } res, err := s.geocoder.Search(ctx, q, near, 6, lang) if err != nil { - return nil, errx.New(errx.ServiceUnavailable, err.Error()) + log.Warn().Err(err).Msg("leadfinder: geocoding failed") + return nil, errx.New(errx.ServiceUnavailable, "Places could not be looked up right now. Try again shortly.") } return res, nil } @@ -536,7 +537,8 @@ func providerErr(id models.LeadProvider, err error) error { case errors.Is(err, context.DeadlineExceeded): return errx.NewWithIdentifier(errx.ServiceUnavailable, "lead_provider_unavailable", name+" took too long. Try a smaller area.") } - return errx.NewWithIdentifier(errx.ServiceUnavailable, "lead_provider_unavailable", err.Error()) + log.Warn().Err(err).Str("provider", name).Msg("leadfinder: provider search failed") + return errx.NewWithIdentifier(errx.ServiceUnavailable, "lead_provider_unavailable", name+" could not be reached. Try again shortly.") } // --- leads ------------------------------------------------------------------ diff --git a/internal/repository/lead_finder_live_test.go b/internal/repository/lead_finder_live_test.go index b5c41efb7..9cb843a35 100644 --- a/internal/repository/lead_finder_live_test.go +++ b/internal/repository/lead_finder_live_test.go @@ -17,7 +17,7 @@ import ( // go test ./internal/repository/ -run LiveLeadFinder -v func TestLiveLeadFinder(t *testing.T) { _, pool := liveContactDB(t) - requireSchemaVersion(t, pool, 355) + requireSchemaVersion(t, pool, 361) ctx := context.Background() repo := NewLeadFinderRepository(pool)