feat: sign in with addresses on internationalized domains (bulwarkmail/webmail#1100)

src/components/settings/AccountSettings.tsx

@@ -16,6 +16,7 @@ import { sharedAccountSettingsTabs } from '../../lib/capabilities';
 import { jmapClient } from '../../api/jmap-client';
 import { fetchMailQuota, type MailQuota } from '../../api/quota';
 import { MAX_ACCOUNTS, accountRowText } from '../../lib/account-utils';
+import { toUnicodeEmail } from '../../lib/idn';
 import type { RootStackParamList } from '../../navigation/types';
 
 function formatFileSize(bytes: number): string {
@@ -184,10 +185,10 @@ export function AccountSettings() {
                     : t('settings.account.accounts.switch_to', 'Switch to this account')}
                 >
                   <View style={styles.rowTitle}>
-                    <Text style={styles.rowName} numberOfLines={1}>{rowText.title}</Text>
+                    <Text style={styles.rowName} numberOfLines={1}>{toUnicodeEmail(rowText.title)}</Text>
                     {isActive && <Check size={14} color={c.success} />}
                   </View>
-                  {rowText.subtitle ? <Text style={styles.rowSub} numberOfLines={1}>{rowText.subtitle}</Text> : null}
+                  {rowText.subtitle ? <Text style={styles.rowSub} numberOfLines={1}>{toUnicodeEmail(rowText.subtitle)}</Text> : null}
                   {isDefault && (
                     <Text style={styles.defaultBadge}>
                       {t('settings.account.accounts.default_badge', 'Default account')}

src/lib/__tests__/idn.test.ts

@@ -0,0 +1,126 @@
+// Ported from the webmail's lib/__tests__/idn.test.ts (#1100).
+import { describe, it, expect } from 'vitest';
+import {
+  toAsciiDomain,
+  toAsciiEmail,
+  toUnicodeDomain,
+  toUnicodeEmail,
+  toUnicodeEmailOnOwnDomains,
+} from '../idn';
+
+// Pairs as Stalwart (idna crate, UTS #46) and browsers produce them.
+const PAIRS: Array<[string, string]> = [
+  ['bücher.de', 'xn--bcher-kva.de'],
+  ['münchen.de', 'xn--mnchen-3ya.de'],
+  ['ノード.com', 'xn--gdkj2l.com'],
+  ['ドメイン名例.jp', 'xn--eckwd4c7cu47r2wf.jp'],
+  ['пример.рф', 'xn--e1afmkfd.xn--p1ai'],
+  ['straße.de', 'xn--strae-oqa.de'],
+  ['𠀋.jp', 'xn--u50i.jp'],
+];
+
+describe('toAsciiDomain', () => {
+  it.each(PAIRS)('converts %s to %s', (unicode, ascii) => {
+    expect(toAsciiDomain(unicode)).toBe(ascii);
+  });
+
+  it('lowercases and maps like a browser does', () => {
+    expect(toAsciiDomain('Bücher.DE')).toBe('xn--bcher-kva.de');
+    expect(toAsciiDomain('ΑΒΓ.gr')).toBe('xn--mxacd.gr');
+    expect(toAsciiDomain(' example.com ')).toBe('example.com');
+  });
+
+  it('rejects input that is not a bare host name', () => {
+    expect(toAsciiDomain('')).toBeNull();
+    expect(toAsciiDomain('a b.de')).toBeNull();
+    expect(toAsciiDomain('evil.com/path')).toBeNull();
+    expect(toAsciiDomain('evil.com:25')).toBeNull();
+    expect(toAsciiDomain('user@evil.com')).toBeNull();
+    expect(toAsciiDomain('%65vil.com')).toBeNull();
+    expect(toAsciiDomain('xn--zz.de')).toBeNull();
+  });
+});
+
+describe('toAsciiEmail', () => {
+  it('converts only the domain of an IDN address', () => {
+    expect(toAsciiEmail('alice@ノード.com')).toBe('alice@xn--gdkj2l.com');
+    expect(toAsciiEmail('Jörg@Bücher.de')).toBe('Jörg@xn--bcher-kva.de');
+  });
+
+  it('leaves ASCII addresses and bare login names exactly as typed', () => {
+    expect(toAsciiEmail('Alice@Example.COM')).toBe('Alice@Example.COM');
+    expect(toAsciiEmail('alice@xn--bcher-kva.de')).toBe('alice@xn--bcher-kva.de');
+    expect(toAsciiEmail('alice')).toBe('alice');
+    expect(toAsciiEmail('jörg')).toBe('jörg');
+    expect(toAsciiEmail('')).toBe('');
+  });
+
+  it('splits at the last @ and keeps unconvertible input unchanged', () => {
+    expect(toAsciiEmail('"a@b"@bücher.de')).toBe('"a@b"@xn--bcher-kva.de');
+    expect(toAsciiEmail('alice@bü cher.de')).toBe('alice@bü cher.de');
+  });
+});
+
+describe('toUnicodeDomain', () => {
+  it.each(PAIRS)('shows %s for %s', (unicode, ascii) => {
+    expect(toUnicodeDomain(ascii)).toBe(unicode);
+  });
+
+  it('decodes uppercase ACE labels to a lowercase domain', () => {
+    expect(toUnicodeDomain('XN--BCHER-KVA.DE')).toBe('bücher.de');
+    expect(toUnicodeDomain('mail.xn--Bcher-kva.de')).toBe('mail.bücher.de');
+  });
+
+  it('leaves plain ASCII domains alone, case included', () => {
+    expect(toUnicodeDomain('Example.COM')).toBe('Example.COM');
+    expect(toUnicodeDomain('')).toBe('');
+  });
+
+  it('keeps labels that are not valid, canonical punycode', () => {
+    expect(toUnicodeDomain('xn--zz.de')).toBe('xn--zz.de');
+    expect(toUnicodeDomain('xn--.de')).toBe('xn--.de');
+    // Decodes to plain "abc", whose ASCII form is not "xn--abc-".
+    expect(toUnicodeDomain('xn--abc-.de')).toBe('xn--abc-.de');
+    // Decode to "À" and fullwidth "ａｂｃü", which UTS #46 maps to other
+    // labels; a registrable domain can never contain them.
+    expect(toUnicodeDomain('xn--3ba.de')).toBe('xn--3ba.de');
+    expect(toUnicodeDomain('xn--tda9921kdae.de')).toBe('xn--tda9921kdae.de');
+    expect(toUnicodeDomain('xn--0ca.de')).toBe('à.de');
+  });
+});
+
+describe('toUnicodeEmail', () => {
+  it('converts only the domain', () => {
+    expect(toUnicodeEmail('alice@xn--gdkj2l.com')).toBe('alice@ノード.com');
+    expect(toUnicodeEmail('Alice+Tag@xn--bcher-kva.de')).toBe('Alice+Tag@bücher.de');
+  });
+
+  it('returns other input unchanged', () => {
+    expect(toUnicodeEmail('alice@example.com')).toBe('alice@example.com');
+    expect(toUnicodeEmail('Alice Example')).toBe('Alice Example');
+    expect(toUnicodeEmail('alice')).toBe('alice');
+  });
+});
+
+describe('toUnicodeEmailOnOwnDomains', () => {
+  const own = ['alice@xn--gdkj2l.com', null, undefined, ''];
+
+  it('shows addresses on the user\'s own IDN domains in Unicode', () => {
+    expect(toUnicodeEmailOnOwnDomains('alice@xn--gdkj2l.com', own)).toBe('alice@ノード.com');
+    expect(toUnicodeEmailOnOwnDomains('bob@XN--GDKJ2L.COM', own)).toBe('bob@ノード.com');
+  });
+
+  it('recognizes the domain when the user\'s own address is in Unicode', () => {
+    expect(toUnicodeEmailOnOwnDomains('bob@xn--bcher-kva.de', ['alice@bücher.de'])).toBe('bob@bücher.de');
+  });
+
+  it('keeps everyone else\'s IDN domain in ASCII form', () => {
+    expect(toUnicodeEmailOnOwnDomains('eve@xn--bcher-kva.de', own)).toBe('eve@xn--bcher-kva.de');
+    expect(toUnicodeEmailOnOwnDomains('eve@sub.xn--gdkj2l.com', own)).toBe('eve@sub.xn--gdkj2l.com');
+    expect(toUnicodeEmailOnOwnDomains('eve@xn--bcher-kva.de', [])).toBe('eve@xn--bcher-kva.de');
+  });
+
+  it('leaves ASCII addresses untouched', () => {
+    expect(toUnicodeEmailOnOwnDomains('Bob@Example.com', ['alice@example.com'])).toBe('Bob@Example.com');
+  });
+});

src/lib/__tests__/server-discovery.test.ts

@@ -197,6 +197,14 @@ describe('discoverServerForEmail', () => {
     expect(mockSecureFetch).not.toHaveBeenCalled();
   });
 
+  it('probes an internationalized domain in its ASCII form (#1100)', async () => {
+    respondFor({ 'https://xn--bcher-kva.de': 401 });
+    await expect(discoverServerForEmail('jörg@Bücher.de')).resolves.toBe('https://xn--bcher-kva.de');
+    expect(emailDomain('jörg@bücher.de')).toBe('xn--bcher-kva.de');
+    expect(normalizeServerUrl('mail.bücher.de/jmap/session')).toBe('https://mail.xn--bcher-kva.de');
+    expect(normalizeServerUrl('https://Mail.Bücher.de:8443')).toBe('https://mail.xn--bcher-kva.de:8443');
+  });
+
   it('stops searching as soon as the sign-in is cancelled (U-133)', async () => {
     // Every host hangs; only the cancel can end the search before the deadline.
     mockSecureFetch.mockImplementation(() => new Promise(() => undefined));

src/lib/idn.ts

@@ -0,0 +1,250 @@
+/**
+ * Internationalized domain names (IDN) in email addresses (webmail
+ * lib/idn.ts, #1100).
+ *
+ * Stalwart keeps domains in their ASCII (punycode, `xn--`) form and maps a
+ * Unicode domain to it on lookup, so the app does the same: whatever the
+ * user types is converted to ASCII before it is sent anywhere, and the ASCII
+ * form is turned back into Unicode only for display.
+ *
+ * The webmail converts through the WHATWG URL parser, which React Native's
+ * URL doesn't implement for hosts, so ToASCII is done here: NFKC and
+ * lowercasing (the UTS #46 mapping for everything a mail domain contains in
+ * practice), then RFC 3492 for each non-ASCII label. A decoded domain is
+ * only shown when it converts back to exactly the ASCII form it came from.
+ */
+
+// Characters that make the input something other than a bare host (path,
+// query, port, userinfo, percent-escapes, IPv6) and the URL parser's other
+// forbidden host code points.
+const NON_HOST_CHARS = /[\s/\\?#@:%[\]<>^|]/;
+const ACE_LABEL = /(^|\.)xn--/i;
+const NON_ASCII = /[^\x00-\x7f]/;
+// UTS #46 label separators: full stop, ideographic, fullwidth, halfwidth.
+const SEPARATORS = /[.。．｡]/;
+
+function mapLabel(label: string): string {
+  let mapped = label;
+  try {
+    mapped = mapped.normalize('NFKC');
+  } catch {
+    // An engine without normalize(): the label as typed.
+  }
+  return mapped.toLowerCase();
+}
+
+/**
+ * The ASCII form of a domain (`bücher.de` -> `xn--bcher-kva.de`), lowercased
+ * and mapped. Null when the input is not a valid host name.
+ */
+export function toAsciiDomain(domain: string): string | null {
+  const trimmed = domain.trim();
+  if (!trimmed || NON_HOST_CHARS.test(trimmed)) return null;
+  const labels = trimmed.split(SEPARATORS);
+  const out: string[] = [];
+  for (let i = 0; i < labels.length; i++) {
+    const label = mapLabel(labels[i]);
+    if (!label) {
+      // Only a trailing dot may leave an empty label.
+      if (i === labels.length - 1 && i > 0) {
+        out.push('');
+        continue;
+      }
+      return null;
+    }
+    if (NON_HOST_CHARS.test(label)) return null;
+    if (!NON_ASCII.test(label)) {
+      // An ACE label must be valid, canonical punycode of a non-ASCII label.
+      if (label.startsWith('xn--')) {
+        const decoded = decodePunycode(label.slice(4));
+        if (decoded === null || !NON_ASCII.test(decoded) || encodePunycode(decoded) !== label.slice(4)) return null;
+      }
+      out.push(label);
+      continue;
+    }
+    const encoded = encodePunycode(label);
+    if (encoded === null) return null;
+    out.push(`xn--${encoded}`);
+  }
+  return out.join('.');
+}
+
+/**
+ * The address with its domain in ASCII form. Only a non-ASCII domain is
+ * touched, so ASCII addresses and bare login names come back unchanged; the
+ * local part is never changed (it is case-sensitive to the server).
+ */
+export function toAsciiEmail(address: string): string {
+  const at = address.lastIndexOf('@');
+  if (at <= 0) return address;
+  const domain = address.slice(at + 1);
+  if (!NON_ASCII.test(domain)) return address;
+  const ascii = toAsciiDomain(domain);
+  return ascii ? `${address.slice(0, at)}@${ascii}` : address;
+}
+
+/**
+ * The Unicode form of a domain for display (`xn--bcher-kva.de` -> `bücher.de`).
+ * Returns the input unchanged when it has no `xn--` label or does not decode
+ * to a valid, canonical domain.
+ */
+export function toUnicodeDomain(domain: string): string {
+  if (!ACE_LABEL.test(domain)) return domain;
+  // Punycode keeps the case of its ASCII letters, so decode the lowercased
+  // domain or `XN--BCHER-KVA.DE` would come out as `BüCHER.DE`.
+  const lower = domain.toLowerCase();
+  const decoded: string[] = [];
+  for (const label of lower.split('.')) {
+    if (!label.startsWith('xn--')) {
+      decoded.push(label);
+      continue;
+    }
+    const unicode = decodePunycode(label.slice(4));
+    if (unicode === null) return domain;
+    decoded.push(unicode);
+  }
+  const unicode = decoded.join('.');
+  return toAsciiDomain(unicode) === lower ? unicode : domain;
+}
+
+/** The address with its domain in Unicode form, for display. */
+export function toUnicodeEmail(address: string): string {
+  const at = address.lastIndexOf('@');
+  if (at <= 0) return address;
+  const domain = address.slice(at + 1);
+  const unicode = toUnicodeDomain(domain);
+  return unicode === domain ? address : `${address.slice(0, at)}@${unicode}`;
+}
+
+/**
+ * `toUnicodeEmail` for an address in a message, applied only when it is on
+ * one of the user's own domains (taken from `ownAddresses`). Anyone else's
+ * IDN domain keeps its xn-- form: a Unicode domain can be drawn to look like
+ * a familiar one, and the ASCII form gives such a look-alike away.
+ */
+export function toUnicodeEmailOnOwnDomains(
+  address: string,
+  ownAddresses: readonly (string | null | undefined)[],
+): string {
+  const domain = address.slice(address.lastIndexOf('@') + 1).toLowerCase();
+  if (!ACE_LABEL.test(domain)) return address;
+  const own = ownAddresses.some((ownAddress) => {
+    if (!ownAddress) return false;
+    const ascii = toAsciiEmail(ownAddress);
+    return ascii.slice(ascii.lastIndexOf('@') + 1).toLowerCase() === domain;
+  });
+  return own ? toUnicodeEmail(address) : address;
+}
+
+// RFC 3492 section 5 parameters.
+const BASE = 36;
+const T_MIN = 1;
+const T_MAX = 26;
+const SKEW = 38;
+const DAMP = 700;
+const INITIAL_BIAS = 72;
+const INITIAL_N = 0x80;
+const MAX_INT = 0x7fffffff;
+
+function adapt(delta: number, numPoints: number, firstTime: boolean): number {
+  let d = firstTime ? Math.floor(delta / DAMP) : Math.floor(delta / 2);
+  d += Math.floor(d / numPoints);
+  let k = 0;
+  while (d > ((BASE - T_MIN) * T_MAX) / 2) {
+    d = Math.floor(d / (BASE - T_MIN));
+    k += BASE;
+  }
+  return k + Math.floor(((BASE - T_MIN + 1) * d) / (d + SKEW));
+}
+
+function digitValue(code: number): number {
+  if (code >= 0x30 && code <= 0x39) return code - 0x16; // 0-9 -> 26-35
+  if (code >= 0x41 && code <= 0x5a) return code - 0x41; // A-Z -> 0-25
+  if (code >= 0x61 && code <= 0x7a) return code - 0x61; // a-z -> 0-25
+  return BASE;
+}
+
+function digitChar(d: number): string {
+  // 0-25 -> a-z, 26-35 -> 0-9
+  return String.fromCharCode(d < 26 ? 0x61 + d : 0x16 + d);
+}
+
+/** RFC 3492 section 6.3 encoding of one label (without `xn--`). */
+function encodePunycode(input: string): string | null {
+  const codePoints = Array.from(input, (ch) => ch.codePointAt(0)!);
+  let output = '';
+  for (const cp of codePoints) if (cp < 0x80) output += String.fromCharCode(cp);
+  const basic = output.length;
+  let handled = basic;
+  if (basic > 0) output += '-';
+
+  let n = INITIAL_N;
+  let delta = 0;
+  let bias = INITIAL_BIAS;
+  while (handled < codePoints.length) {
+    let m = MAX_INT;
+    for (const cp of codePoints) if (cp >= n && cp < m) m = cp;
+    if (m - n > Math.floor((MAX_INT - delta) / (handled + 1))) return null;
+    delta += (m - n) * (handled + 1);
+    n = m;
+    for (const cp of codePoints) {
+      if (cp < n) {
+        delta += 1;
+        if (delta > MAX_INT) return null;
+      }
+      if (cp === n) {
+        let q = delta;
+        for (let k = BASE; ; k += BASE) {
+          const t = k <= bias ? T_MIN : k >= bias + T_MAX ? T_MAX : k - bias;
+          if (q < t) break;
+          output += digitChar(t + ((q - t) % (BASE - t)));
+          q = Math.floor((q - t) / (BASE - t));
+        }
+        output += digitChar(q);
+        bias = adapt(delta, handled + 1, handled === basic);
+        delta = 0;
+        handled += 1;
+      }
+    }
+    delta += 1;
+    n += 1;
+  }
+  return output;
+}
+
+/** RFC 3492 section 6.2 decoding of one label (without `xn--`). */
+function decodePunycode(input: string): string | null {
+  const output: number[] = [];
+  const basic = Math.max(input.lastIndexOf('-'), 0);
+  for (let j = 0; j < basic; j++) {
+    const code = input.charCodeAt(j);
+    if (code >= 0x80) return null;
+    output.push(code);
+  }
+
+  let n = INITIAL_N;
+  let bias = INITIAL_BIAS;
+  let i = 0;
+  for (let index = basic > 0 ? basic + 1 : 0; index < input.length;) {
+    const oldI = i;
+    for (let w = 1, k = BASE; ; k += BASE) {
+      if (index >= input.length) return null;
+      const digit = digitValue(input.charCodeAt(index++));
+      if (digit >= BASE || digit > Math.floor((MAX_INT - i) / w)) return null;
+      i += digit * w;
+      const t = k <= bias ? T_MIN : k >= bias + T_MAX ? T_MAX : k - bias;
+      if (digit < t) break;
+      if (w > Math.floor(MAX_INT / (BASE - t))) return null;
+      w *= BASE - t;
+    }
+    const length = output.length + 1;
+    bias = adapt(i - oldI, length, oldI === 0);
+    if (Math.floor(i / length) > MAX_INT - n) return null;
+    n += Math.floor(i / length);
+    i %= length;
+    if (n > 0x10ffff || (n >= 0xd800 && n <= 0xdfff)) return null;
+    output.splice(i++, 0, n);
+  }
+  if (output.length === 0) return null;
+  return String.fromCodePoint(...output);
+}

src/lib/server-discovery.ts

@@ -9,6 +9,7 @@
 
 import { secureFetch } from './client-cert';
 import { signalWithTimeout } from './sign-in-abort';
+import { toAsciiDomain } from './idn';
 
 export const DISCOVERY_TIMEOUT_MS = 2500;
 
@@ -69,16 +70,30 @@ export function normalizeServerUrl(input: string): string | null {
   const host = value.slice(value.indexOf('://') + 3).split('/')[0];
   if (!host || /[\s@]/.test(host)) return null;
 
-  const hostname = host.split(':')[0].toLowerCase();
+  let hostname = host.split(':')[0].toLowerCase();
   if (!hostname) return null;
+  // An internationalized host goes on the wire in its ASCII form (#1100).
+  if (/[^\x00-\x7f]/.test(hostname)) {
+    const ascii = toAsciiDomain(hostname);
+    if (!ascii) return null;
+    const schemeEnd = value.indexOf('://') + 3;
+    value = value.slice(0, schemeEnd) + ascii + value.slice(schemeEnd + hostname.length);
+    hostname = ascii;
+  }
   if (!hostname.includes('.') && hostname !== 'localhost') return null;
 
   return value;
 }
 
+/**
+ * The domain of an address in its ASCII form (`jörg@bücher.de` ->
+ * `xn--bcher-kva.de`), which is what discovery probes and the server knows.
+ * Show it with toUnicodeDomain.
+ */
 export function emailDomain(email: string): string | null {
   const match = EMAIL_RE.exec(email.trim());
-  return match ? match[1].toLowerCase() : null;
+  if (!match) return null;
+  return toAsciiDomain(match[1]) ?? match[1].toLowerCase();
 }
 
 export function isEmailAddress(value: string): boolean {

src/screens/login/ServerStep.tsx

@@ -6,6 +6,7 @@ import { useColors } from '../../theme/colors';
 import { Button, Input } from '../../components';
 import LoginNotice from './LoginNotice';
 import { useLocaleStore } from '../../stores/locale-store';
+import { toUnicodeDomain } from '../../lib/idn';
 
 interface ServerStepProps {
   value: string;
@@ -53,7 +54,7 @@ export default function ServerStep({
         <Text style={styles.title}>{t('login.mobile.server_title', "What's your server address?")}</Text>
         <Text style={styles.subtitle}>
           {failedDomain
-            ? t('login.mobile.server_not_found', "We couldn't find a server for {domain}. If you self-host, the address is the one you use for webmail.", { domain: failedDomain })
+            ? t('login.mobile.server_not_found', "We couldn't find a server for {domain}. If you self-host, the address is the one you use for webmail.", { domain: toUnicodeDomain(failedDomain) })
             : t('login.mobile.server_subtitle', 'Enter the address you use to reach your webmail.')}
         </Text>
       </View>

src/stores/__tests__/auth-store-cancel.test.ts

@@ -99,3 +99,13 @@ describe('cancelling a sign-in', () => {
     expect(useAuthStore.getState().error).toBeNull();
   });
 });
+
+describe('internationalized domains (#1100)', () => {
+  it('signs in with the ASCII form of the domain, keeping the local part', async () => {
+    const pending = useAuthStore.getState().login(SERVER, 'Jörg@Bücher.de', 'pw').catch(() => undefined);
+    await connectStarted();
+    expect(jmapClient.connect).toHaveBeenCalledWith(SERVER, 'Jörg@xn--bcher-kva.de', 'pw', undefined, undefined);
+    finishConnect!({ apiUrl: `${SERVER}/jmap/` });
+    await pending;
+  });
+});

src/stores/auth-store.ts

@@ -42,6 +42,7 @@ import { reconcileServerFingerprint } from '../lib/server-change';
 import { deviceSyncSignedIn, releaseDeviceSyncBeforeSignOut } from '../device-sync/app/lifecycle';
 import { perfMark } from '../lib/perf-marks';
 import { abortable, isSignInCancelled, throwIfCancelled } from '../lib/sign-in-abort';
+import { toAsciiEmail } from '../lib/idn';
 
 // Persist middleware hydrates asynchronously on cold start. Without this
 // guard, restoreSession() can read the account-store before AsyncStorage has
@@ -553,8 +554,11 @@ export const useAuthStore = create<AuthState>((set, get) => ({
   client: null,
   pendingTotpLogin: null,
 
-  login: async (serverUrl, username, password, opts) => {
+  login: async (serverUrl, typedUsername, password, opts) => {
     set({ isLoading: true, error: null });
+    // An address on an internationalized domain signs in with the domain's
+    // ASCII form, the one Stalwart stores (#1100); the local part stays.
+    const username = toAsciiEmail(typedUsername);
     try {
       // A new account with no room left fails here, before connect swaps
       // the live client over or stores credentials for it.