warmbly/internal/pkg/mailhost/detect.go

package mailhost

import (
	"context"
	"errors"
	"net"
	"net/http"
	"sort"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/idna"

	"github.com/warmbly/warmbly/internal/pkg/safehttp"
)

const (
	// detectBudget bounds one domain's whole chain; DetectMany applies it per domain.
	detectBudget  = 12 * time.Second
	mxTimeout     = 4 * time.Second
	srvTimeout    = 3 * time.Second
	fetchTimeout  = 4 * time.Second
	maxBody       = 64 << 10
	manyWorkers   = 32
	defaultISPDB  = "https://autoconfig.thunderbird.net/v1.1/"
	maxDomainSize = 253
)

// Resolver is the DNS surface Detect needs; *net.Resolver satisfies it.
type Resolver interface {
	LookupMX(ctx context.Context, name string) ([]*net.MX, error)
	LookupSRV(ctx context.Context, service, proto, name string) (string, []*net.SRV, error)
}

// Cache stores detections by normalized domain.
type Cache interface {
	Get(ctx context.Context, domain string) (Detection, bool)
	Set(ctx context.Context, domain string, d Detection)
}

// Detector runs the detection chain. The zero value is not usable; use NewDetector.
type Detector struct {
	resolver Resolver
	client   *http.Client
	cache    Cache
	// ispdbBase is the Thunderbird ISPDB prefix; the domain is appended.
	ispdbBase string
	// autoconfigURLs builds the provider-hosted autoconfig URLs for a domain.
	autoconfigURLs func(domain string) []string
}

// NewDetector builds a Detector. A nil resolver uses net.DefaultResolver, a nil
// client an SSRF-hardened one (domains are user input), a nil cache none.
func NewDetector(r Resolver, httpClient *http.Client, c Cache) *Detector {
	if r == nil {
		r = net.DefaultResolver
	}
	if httpClient == nil {
		httpClient = safehttp.Client(fetchTimeout)
	}
	return &Detector{
		resolver:       r,
		client:         httpClient,
		cache:          c,
		ispdbBase:      defaultISPDB,
		autoconfigURLs: defaultAutoconfigURLs,
	}
}

// WithoutISPDB stops the detector asking Thunderbird's ISPDB, a third party;
// the domain's own DNS and autoconfig files are still read.
func (d *Detector) WithoutISPDB() *Detector {
	d.ispdbBase = ""
	return d
}

func defaultAutoconfigURLs(domain string) []string {
	return []string{
		"https://autoconfig." + domain + "/mail/config-v1.1.xml?emailaddress=info@" + domain,
		"https://" + domain + "/.well-known/autoconfig/mail/config-v1.1.xml",
	}
}

// NormalizeDomain lower-cases and trims a domain (or the domain of an address)
// and converts it to ASCII. It returns "" when the result is not a hostname.
func NormalizeDomain(s string) string {
	s = strings.TrimSpace(s)
	if i := strings.LastIndexByte(s, '@'); i >= 0 {
		s = s[i+1:]
	}
	s = strings.TrimSuffix(strings.ToLower(s), ".")
	if s == "" {
		return ""
	}
	ascii, err := idna.Lookup.ToASCII(s)
	if err != nil {
		return ""
	}
	if !validHostname(ascii) || !strings.Contains(ascii, ".") {
		return ""
	}
	return ascii
}

// validHostname accepts LDH labels only, so a value can go into a URL or a DNS query.
func validHostname(s string) bool {
	if s == "" || len(s) > maxDomainSize {
		return false
	}
	for _, label := range strings.Split(s, ".") {
		if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
			return false
		}
		for i := 0; i < len(label); i++ {
			c := label[i]
			if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' && c != '_' {
				return false
			}
		}
	}
	return true
}

// Detect works out where domain's mail is hosted. It never fails: what cannot
// be determined comes back with Source "none".
func (d *Detector) Detect(ctx context.Context, domain string) Detection {
	norm := NormalizeDomain(domain)
	if norm == "" {
		return Detection{Domain: strings.ToLower(strings.TrimSpace(domain)), Source: SourceNone, PasswordAuth: Unsupported}
	}
	if d.cache != nil {
		if hit, ok := d.cache.Get(ctx, norm); ok {
			return hit
		}
	}
	ctx, cancel := context.WithTimeout(ctx, detectBudget)
	defer cancel()
	det, cacheable := d.detect(ctx, norm)
	if cacheable && d.cache != nil {
		d.cache.Set(ctx, norm, det)
	}
	return det
}

// detect runs the chain; cacheable is false when a lookup failed transiently.
func (d *Detector) detect(ctx context.Context, domain string) (Detection, bool) {
	if h, v, ok := known(domain); ok {
		return detection(domain, h, v, SourceKnown, nil), true
	}

	mx, mxErr := d.lookupMX(ctx, domain)
	transient := mxErr != nil && !notFound(mxErr)
	for _, host := range mx {
		if h, v := classify(host); h != Unknown {
			return detection(domain, Refine(h, domain), v, SourceMX, mx), true
		}
	}

	if s := d.autoconfig(ctx, domain); s != nil {
		return fromSettings(domain, SourceAutoconfig, s, mx), true
	}
	if s := d.ispdb(ctx, domain, domain); s != nil {
		return fromSettings(domain, SourceISPDB, s, mx), true
	}
	if len(mx) > 0 {
		if reg, _ := registrableDomain(mx[0]); reg != "" && reg != domain {
			if s := d.ispdb(ctx, reg, domain); s != nil {
				return fromSettings(domain, SourceISPDB, s, mx), true
			}
		}
	}
	if s, err := d.srv(ctx, domain); s != nil {
		return fromSettings(domain, SourceSRV, s, mx), true
	} else if err != nil {
		transient = true
	}

	det := Detection{Domain: domain, Host: Unknown, Source: SourceNone, PasswordAuth: Unsupported, MX: mx}
	if len(mx) > 0 {
		det.Host = Other
		det.PasswordAuth = Password
	}
	// A deadline that ran out says nothing about the domain.
	if ctx.Err() != nil {
		transient = true
	}
	return det, !transient
}

// fromSettings names the provider behind discovered servers.
func fromSettings(domain, source string, s *Settings, mx []string) Detection {
	h, v := classify(s.SMTP.Host)
	if h == Unknown {
		h, v = classify(s.IMAP.Host)
	}
	if h == Unknown {
		return Detection{Domain: domain, Host: Other, Source: source, Settings: s, PasswordAuth: Password, MX: mx}
	}
	det := detection(domain, Refine(h, domain), v, source, mx)
	det.Settings = s
	return det
}

func (d *Detector) lookupMX(ctx context.Context, domain string) ([]string, error) {
	ctx, cancel := context.WithTimeout(ctx, mxTimeout)
	defer cancel()
	recs, err := d.resolver.LookupMX(ctx, domain)
	// The Go resolver can return usable records alongside an error for bad ones.
	sort.SliceStable(recs, func(i, j int) bool { return recs[i].Pref < recs[j].Pref })
	out := make([]string, 0, len(recs))
	seen := map[string]bool{}
	for _, r := range recs {
		if r == nil {
			continue
		}
		h := normHost(r.Host)
		// A null MX (RFC 7505) is a domain that receives no mail.
		if h == "" || seen[h] {
			continue
		}
		seen[h] = true
		out = append(out, h)
	}
	if len(out) > 0 {
		return out, nil
	}
	return nil, err
}

func notFound(err error) bool {
	var dnsErr *net.DNSError
	return errors.As(err, &dnsErr) && dnsErr.IsNotFound
}

// registrableDomain is host's eTLD+1.
func registrableDomain(host string) (string, bool) {
	label, suffix := registrable(host)
	if label == "" {
		return "", false
	}
	return label + "." + suffix, true
}

// DetectMany detects every distinct domain with bounded concurrency. The map
// is keyed by each input string as given; empty inputs are skipped.
func (d *Detector) DetectMany(ctx context.Context, domains []string) map[string]Detection {
	byNorm := map[string][]string{}
	var order []string
	for _, in := range domains {
		if strings.TrimSpace(in) == "" {
			continue
		}
		n := NormalizeDomain(in)
		if n == "" {
			n = "\x00" + in
		}
		if _, ok := byNorm[n]; !ok {
			order = append(order, n)
		}
		byNorm[n] = append(byNorm[n], in)
	}

	results := make([]Detection, len(order))
	sem := make(chan struct{}, manyWorkers)
	var wg sync.WaitGroup
	for i, n := range order {
		wg.Add(1)
		go func(i int, in string) {
			defer wg.Done()
			select {
			case sem <- struct{}{}:
			case <-ctx.Done():
				results[i] = d.Detect(ctx, in)
				return
			}
			defer func() { <-sem }()
			results[i] = d.Detect(ctx, in)
		}(i, byNorm[n][0])
	}
	wg.Wait()

	out := make(map[string]Detection, len(domains))
	for i, n := range order {
		for _, in := range byNorm[n] {
			out[in] = results[i]
		}
	}
	return out
}