fix(links): end autolinked urls at CJK punctuation in chat, terminal and mobile (#32013)

mobile/src/components/MobileMarkdown.file-links.test.ts

@@ -135,6 +135,29 @@ describe('MobileMarkdown file links', () => {
     expect(openURL).toHaveBeenCalledWith('https://example.com/a')
   })
 
+  it.each([
+    [
+      '投稿（https://github.com/stablyai/orca/pull/1078）。',
+      'https://github.com/stablyai/orca/pull/1078',
+      'https://github.com/stablyai/orca/pull/1078'
+    ],
+    ['詳細は「www.example.com/docs」、次へ', 'www.example.com/docs', 'http://www.example.com/docs'],
+    [
+      'https://ja.wikipedia.org/wiki/ジョン・スミス。',
+      'https://ja.wikipedia.org/wiki/ジョン・スミス',
+      'https://ja.wikipedia.org/wiki/%E3%82%B8%E3%83%A7%E3%83%B3%E3%83%BB%E3%82%B9%E3%83%9F%E3%82%B9'
+    ]
+  ])('ends the autolink in %s at CJK punctuation', (content, linkText, url) => {
+    pressByText(render(content), linkText)
+    expect(openURL).toHaveBeenCalledExactlyOnceWith(url)
+  })
+
+  it('renders text whose url candidate ends before its host', () => {
+    expect(flattenText(render('https://（ユーザー）@example.com').root)).toBe(
+      'https://（ユーザー）@example.com'
+    )
+  })
+
   it('opens inline-code path:line citations', () => {
     pressByText(render('fix `src/foo.ts:42` now'), 'src/foo.ts:42')
     expect(onOpenFile).toHaveBeenCalledWith('src/foo.ts:42')

mobile/src/components/markdown-linkify-prose-boundary.ts

@@ -0,0 +1,29 @@
+import type MarkdownIt from 'markdown-it'
+import { findProseUrlEnd, splitAtProseUrlTerminators } from '../../../src/shared/prose-url-boundary'
+
+type Linkify = MarkdownIt['linkify']
+
+// Why: linkify-it lets non-ascii punctuation run on in a url path, so `…/1078）。` kept `）。`;
+// matching only the text between prose punctuation lets linkify's own trim run on just the url.
+export function endLinkifyUrlsAtProsePunctuation(linkify: Linkify): void {
+  const match = linkify.match.bind(linkify)
+  const matchAtStart = linkify.matchAtStart.bind(linkify)
+
+  linkify.match = (text) => {
+    const links: NonNullable<ReturnType<Linkify['match']>> = []
+    let offset = 0
+    splitAtProseUrlTerminators(text).forEach((piece, index) => {
+      if (index % 2 === 0) {
+        for (const link of match(piece) ?? []) {
+          links.push({ ...link, index: link.index + offset, lastIndex: link.lastIndex + offset })
+        }
+      }
+      offset += piece.length
+    })
+    return links
+  }
+  linkify.matchAtStart = (text) => {
+    const end = findProseUrlEnd(text)
+    return matchAtStart(end === -1 ? text : text.slice(0, end))
+  }
+}

mobile/src/components/mobile-markdown-tree.ts

@@ -12,12 +12,14 @@ import type {
 } from 'mdast'
 import remend from 'remend'
 import { normalizeMobileMarkdownPreviewHtml } from './mobile-markdown-preview-html'
+import { endLinkifyUrlsAtProsePunctuation } from './markdown-linkify-prose-boundary'
 
 // CommonMark with GitHub's tables, strikethrough, autolinks and footnotes: the rules desktop's
 // renderer follows, read by a parser fast enough for the phone's JavaScript engine.
 const markdown = new MarkdownIt({ html: true, linkify: true }).use(footnote)
 // Every href parses as a link, file: included; routeMarkdownHref decides what a tap opens.
 markdown.validateLink = () => true
+endLinkifyUrlsAtProsePunctuation(markdown.linkify)
 
 type Token = ReturnType<MarkdownIt['parse']>[number]
 

mobile/src/terminal/document/native-document-bundle.test.ts

@@ -315,7 +315,8 @@ describe('the bundled native document', () => {
     // `terminal-grid-fit` is the one fit of that box and `terminal-cell-box` its one comparison,
     // both shared with the app. 52: `cell-metrics` and `selection-mode` split out to break cycles.
     // 53: the bottom row backdrop extends a uniform row into the fractional gap.
-    expect(inputs).toHaveLength(53)
+    // 54: `src/shared/prose-url-boundary`, so a tapped url ends where desktop's link does.
+    expect(inputs).toHaveLength(54)
     expect(script).not.toContain('__commonJS')
     // `__esm` wrappers are esbuild's answer to a cycle, and a cycle would make a module's top level
     // run at first import rather than where the bundle places it.

mobile/src/terminal/document/url-tap.ts

@@ -19,7 +19,7 @@ export function findTerminalUrlAtColumn(lineText: unknown, col: number, source:
   if (typeof lineText !== 'string' || lineText.length === 0) {
     return null
   }
-  const re = new RegExp(source, 'gi')
+  const re = new RegExp(source, 'giu')
   let match: RegExpExecArray | null
   while ((match = re.exec(lineText)) !== null) {
     const end = match.index + match[0].length

mobile/src/terminal/terminal-webview-url-tap.test.ts

@@ -157,6 +157,15 @@ describe('findUrlAtColumn', () => {
     expect(findUrlAtColumn(line, line.length - 1)).toBeNull()
   })
 
+  it('ends the tapped URL at CJK punctuation', () => {
+    const line = '投稿（https://github.com/stablyai/orca/pull/1078）。'
+
+    expect(findUrlAtColumn(line, line.indexOf('1078'))).toBe(
+      'https://github.com/stablyai/orca/pull/1078'
+    )
+    expect(findUrlAtColumn(line, line.indexOf('）'))).toBeNull()
+  })
+
   it('only matches http(s) schemes', () => {
     const line = 'ftp://example.com/file and file:///etc/hosts'
 

mobile/src/terminal/terminal-webview-url-tap.ts

@@ -1,11 +1,8 @@
+import { PROSE_HTTP_URL_SOURCE } from '../../../src/shared/prose-url-boundary'
+
 export { resolveTerminalFileUrlTap, resolveTerminalOscFileTap } from './terminal-file-url-tap'
 
-export const TERMINAL_HTTP_URL_REGEX_SOURCE =
-  String.raw`\bhttps?:\/\/[^\s"'!*(){}|\\^<>` +
-  '`' +
-  String.raw`]*[^\s"':,.!?{}|\\^~[\]` +
-  '`' +
-  String.raw`()<>]`
+export const TERMINAL_HTTP_URL_REGEX_SOURCE = String.raw`\b${PROSE_HTTP_URL_SOURCE}`
 export const TERMINAL_FILE_URL_REGEX_SOURCE =
   String.raw`\bfile:\/\/[^\s"'!*(){}|\\^<>` +
   '`' +
@@ -26,7 +23,7 @@ function findTerminalUrlAtColumn(lineText: string, col: number, source: string):
   if (typeof lineText !== 'string' || lineText.length === 0) {
     return null
   }
-  const re = new RegExp(source, 'gi')
+  const re = new RegExp(source, 'giu')
   let match: RegExpExecArray | null
   while ((match = re.exec(lineText)) !== null) {
     const start = match.index

mobile/src/test-support/webview-document-bundle-digest.ts

@@ -31,10 +31,11 @@ export function bundleDigestBuiltFrom(cwd: string, generator: string, member: st
  * Every line of a bundle that names a directory only the machine that built it has.
  *
  * Both shapes esbuild can emit when the working directory is not the one the sources live under: a
- * comment that is an absolute path, and one that climbs out with `../`.
+ * comment that is an absolute path, and one that climbs out of the repo with `../../`. A single
+ * `../` from the pinned `mobile/` root is the repo's own `src/shared`, which every machine has.
  */
 export function machinePathCommentsIn(script: string): string[] {
   return script
     .split('\n')
-    .filter((line) => /^\s*\/\/ (\/|\.\.\/)/.test(line) || line.includes('/Users/'))
+    .filter((line) => /^\s*\/\/ (\/|\.\.\/\.\.\/)/.test(line) || line.includes('/Users/'))
 }

src/renderer/src/components/dashboard-popout/preview-terminal-links.ts

@@ -2,6 +2,7 @@ import type { Terminal } from '@xterm/xterm'
 import { WebLinksAddon } from '@xterm/addon-web-links'
 import { installGuardedLinkProviderRegistration } from '@/lib/pane-manager/terminal-link-provider-guard'
 import { isTerminalHttpLinkActivation } from '@/components/terminal-pane/terminal-http-link-activation'
+import { TERMINAL_HTTP_URL_REGEX } from '@/components/terminal-pane/terminal-http-url-extraction'
 
 /**
  * Makes URLs in the preview clickable under the same Mod+click gesture a pane
@@ -14,13 +15,16 @@ export function installPreviewTerminalLinks(terminal: Terminal): void {
   // and kills the renderer — guard before any provider registers.
   installGuardedLinkProviderRegistration(terminal)
   terminal.loadAddon(
-    new WebLinksAddon((event, uri) => {
-      if (!isTerminalHttpLinkActivation(event)) {
-        return
-      }
-      event.preventDefault()
-      void window.api.shell.openUrl(uri).catch(() => undefined)
-      terminal.clearSelection()
-    })
+    new WebLinksAddon(
+      (event, uri) => {
+        if (!isTerminalHttpLinkActivation(event)) {
+          return
+        }
+        event.preventDefault()
+        void window.api.shell.openUrl(uri).catch(() => undefined)
+        terminal.clearSelection()
+      },
+      { urlRegex: TERMINAL_HTTP_URL_REGEX }
+    )
   )
 }

src/renderer/src/components/editor/markdown-preview-plugins.ts

@@ -14,6 +14,7 @@ import {
   GITHUB_CALLOUT_SANITIZE_ATTRIBUTE,
   remarkGitHubCallouts
 } from '@/lib/remark-github-callouts'
+import { remarkAutolinkLiteralProseBoundary } from '@/lib/remark-autolink-literal-prose-boundary'
 
 const markdownPreviewSanitizeSchema = {
   ...defaultSchema,
@@ -61,6 +62,7 @@ const markdownPreviewSanitizeSchema = {
 type MarkdownPluginList = NonNullable<ReactMarkdownOptions['remarkPlugins']>
 export const MARKDOWN_REMARK_PLUGINS: MarkdownPluginList = [
   remarkGfm,
+  remarkAutolinkLiteralProseBoundary,
   remarkCjkFriendly,
   remarkGitHubCallouts,
   remarkBreaks,

src/renderer/src/components/sidebar/CommentMarkdown.test.tsx

@@ -59,6 +59,30 @@ describe('CommentMarkdown', () => {
     expect(markup).toContain('href="https://github.com/another-org/other-repo/issues/42"')
   })
 
+  it('ends bare urls at CJK sentence punctuation', () => {
+    const hrefsIn = (content: string): string[] =>
+      [
+        ...renderToStaticMarkup(<CommentMarkdown content={content} />).matchAll(/href="([^"]*)"/g)
+      ].map((match) => match[1]!)
+
+    expect(hrefsIn('投稿はお願いします (https://github.com/stablyai/orca/pull/1078)。')).toEqual([
+      'https://github.com/stablyai/orca/pull/1078'
+    ])
+    expect(hrefsIn('投稿（https://github.com/stablyai/orca/pull/1078）。')).toEqual([
+      'https://github.com/stablyai/orca/pull/1078'
+    ])
+    expect(hrefsIn('「https://example.com/a」、https://example.com/b。')).toEqual([
+      'https://example.com/a',
+      'https://example.com/b'
+    ])
+    expect(hrefsIn('https://ja.wikipedia.org/wiki/ジョン・スミス。')).toEqual([
+      'https://ja.wikipedia.org/wiki/%E3%82%B8%E3%83%A7%E3%83%B3%E3%83%BB%E3%82%B9%E3%83%9F%E3%82%B9'
+    ])
+    expect(hrefsIn('https://en.wikipedia.org/wiki/Foo_(bar)')).toEqual([
+      'https://en.wikipedia.org/wiki/Foo_(bar)'
+    ])
+  })
+
   it('does not autolink GitHub issue references inside existing links or code', () => {
     const markup = renderToStaticMarkup(
       <CommentMarkdown

src/renderer/src/components/sidebar/CommentMarkdown.tsx

@@ -32,6 +32,7 @@ import {
   GITHUB_CALLOUT_SANITIZE_ATTRIBUTE,
   remarkGitHubCallouts
 } from '@/lib/remark-github-callouts'
+import { remarkAutolinkLiteralProseBoundary } from '@/lib/remark-autolink-literal-prose-boundary'
 
 export type { CommentMarkdownLinkClickHandler } from './comment-markdown-element-renderers'
 
@@ -80,7 +81,12 @@ const commentMarkdownFileUriUrlTransform: UrlTransform = (value, key, node) => {
 // plain-text renderer used whitespace-pre-wrap which preserved them. Adding
 // remark-breaks converts single newlines to <br>, keeping backward compat
 // with existing plain-text comments that rely on newline formatting.
-const remarkPlugins = [remarkGfm, remarkGitHubCallouts, remarkBreaks]
+const remarkPlugins = [
+  remarkGfm,
+  remarkAutolinkLiteralProseBoundary,
+  remarkGitHubCallouts,
+  remarkBreaks
+]
 
 const GITHUB_REFERENCE_PATTERN = /(?:\b([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+))?#([1-9][0-9]*)\b/g
 

src/renderer/src/components/terminal-pane/terminal-http-url-extraction.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest'
-import { extractTerminalHttpLinks } from './terminal-http-url-extraction'
+import { extractTerminalHttpLinks, TERMINAL_HTTP_URL_REGEX } from './terminal-http-url-extraction'
 
 const urlsIn = (line: string): string[] => extractTerminalHttpLinks(line).map((link) => link.url)
 
@@ -30,6 +30,7 @@ describe('extractTerminalHttpLinks', () => {
   it('stops at a full-width comma and full stop between a url and prose', () => {
     expect(urlsIn('https://example.com/a、次へ')).toEqual(['https://example.com/a'])
     expect(urlsIn('https://example.com/b。')).toEqual(['https://example.com/b'])
+    expect(urlsIn('詳細はhttps://example.com/docs……')).toEqual(['https://example.com/docs'])
   })
 
   it('still extracts a plain ascii url and keeps its own punctuation', () => {
@@ -41,4 +42,19 @@ describe('extractTerminalHttpLinks', () => {
   it('still trims ascii trailing punctuation', () => {
     expect(urlsIn('see https://example.com/a.')).toEqual(['https://example.com/a'])
   })
+
+  it('gives the web-links hover the same url the extractor opens', () => {
+    const webLinksUrlsIn = (line: string): string[] =>
+      [...line.matchAll(new RegExp(TERMINAL_HTTP_URL_REGEX, 'gu'))].map((match) => match[0])
+    for (const line of [
+      '投稿はお願いします（https://github.com/stablyai/orca/pull/1078）。',
+      '投稿はお願いします (https://github.com/stablyai/orca/pull/1078)。',
+      '詳細は「https://example.com/docs」、https://example.com/b、次へ',
+      'https://example.com/文档/路径。',
+      'https://ja.wikipedia.org/wiki/ジョン・スミス、https://en.wikipedia.org/wiki/Michelson–Morley_experiment。',
+      'see https://example.com/a_b-c?d=1&e=2#f.'
+    ]) {
+      expect(webLinksUrlsIn(line).map((url) => new URL(url).toString())).toEqual(urlsIn(line))
+    }
+  })
 })

src/renderer/src/components/terminal-pane/terminal-http-url-extraction.ts

@@ -1,4 +1,5 @@
 import { TERMINAL_HTTP_URL_MAX_LENGTH } from './terminal-http-link-limits'
+import { isProseUrlTerminator, PROSE_HTTP_URL_SOURCE } from '../../../../shared/prose-url-boundary'
 
 type ParsedTerminalHttpLink = {
   url: string
@@ -90,7 +91,7 @@ function trimHttpUrlTrailingPunctuation(
 
 function isHttpUrlBodyTerminator(code: number): boolean {
   return (
-    isNonAsciiUrlTerminator(code) ||
+    (code > 0x7f && isProseUrlTerminator(String.fromCharCode(code))) ||
     isAsciiWhitespace(code) ||
     code === 0x22 ||
     code === 0x27 ||
@@ -135,13 +136,7 @@ function isHttpUrlTrailingPunctuation(code: number): boolean {
   )
 }
 
-// Why not all non-ascii: a url path may legitimately carry unencoded CJK (see the wrapped-url
-// tests), so only punctuation and spaces end it - full-width brackets, an ideographic space, `\u30fb`.
-const NON_ASCII_URL_TERMINATOR = /[\p{P}\p{S}\p{Z}]/u
-
-function isNonAsciiUrlTerminator(code: number): boolean {
-  return code > 0x7e && NON_ASCII_URL_TERMINATOR.test(String.fromCharCode(code))
-}
+export const TERMINAL_HTTP_URL_REGEX = new RegExp(PROSE_HTTP_URL_SOURCE, 'iu')
 
 function isAsciiWhitespace(code: number): boolean {
   return code === 9 || code === 10 || code === 11 || code === 12 || code === 13 || code === 32

src/renderer/src/lib/pane-manager/pane-dom-creation.ts

@@ -5,6 +5,7 @@ import { WebLinksAddon } from '@xterm/addon-web-links'
 import { Terminal } from '@xterm/xterm'
 import type { ILink, ITerminalOptions } from '@xterm/xterm'
 import { createWrappedTerminalHttpLinkProvider } from '../../components/terminal-pane/wrapped-terminal-http-link-provider'
+import { TERMINAL_HTTP_URL_REGEX } from '../../components/terminal-pane/terminal-http-url-extraction'
 import type { TerminalLeafId } from '../../../../shared/stable-pane-id'
 import type { DragReorderCallbacks, DragReorderState } from './pane-drag-reorder'
 import { attachPaneDrag } from './pane-drag-pointer'
@@ -105,10 +106,10 @@ export function createPaneDOM(
   if (options.onLinkClick) {
     terminal.registerLinkProvider(createWrappedTerminalHttpLinkProvider(terminal, linkHandlers))
   }
-  const webLinksAddon = new WebLinksAddon(
-    options.onLinkClick ? linkHandlers.activate : undefined,
-    linkHandlers
-  )
+  const webLinksAddon = new WebLinksAddon(options.onLinkClick ? linkHandlers.activate : undefined, {
+    ...linkHandlers,
+    urlRegex: TERMINAL_HTTP_URL_REGEX
+  })
 
   const panePointerDownHandler = (event: PointerEvent): void => {
     onPointerDown(id, {

src/renderer/src/lib/remark-autolink-literal-prose-boundary.ts

@@ -0,0 +1,65 @@
+import type { Processor } from 'unified'
+import { splitAtProseUrlTerminators } from '../../../shared/prose-url-boundary'
+
+type MarkdownNode = {
+  type: string
+  value?: string
+  position?: { start: { offset?: number }; end: { offset?: number } }
+  children?: MarkdownNode[]
+}
+
+/** The text of a bare-url link written as itself, not `<url>` or `[url](url)`. */
+function autolinkLiteralText(node: MarkdownNode): string | null {
+  const [child, ...rest] = node.children ?? []
+  const start = node.position?.start.offset
+  const end = node.position?.end.offset
+  if (node.type !== 'link' || child?.type !== 'text' || rest.length > 0) {
+    return null
+  }
+  if (child.value === undefined || start === undefined || end === undefined) {
+    return null
+  }
+  return end - start === child.value.length ? child.value : null
+}
+
+function removePositions(node: MarkdownNode): void {
+  delete node.position
+  node.children?.forEach(removePositions)
+}
+
+// Why: GFM ends a bare url only at whitespace and trims only ASCII punctuation, so the url
+// in `(https://…/1078)。` kept `)。`; parsing each piece between prose punctuation lets GFM's
+// own trim and balanced-paren rule run on just the url.
+export function remarkAutolinkLiteralProseBoundary(this: Processor): (tree: MarkdownNode) => void {
+  const parseInline = (text: string): MarkdownNode[] => {
+    const tree: MarkdownNode = this.parse(text)
+    const [paragraph] = tree.children ?? []
+    if (paragraph?.type !== 'paragraph' || !paragraph.children) {
+      return [{ type: 'text', value: text }]
+    }
+    paragraph.children.forEach(removePositions)
+    return paragraph.children
+  }
+
+  const splitAutolinkLiterals = (node: MarkdownNode): void => {
+    if (!node.children) {
+      return
+    }
+    node.children = node.children.flatMap((child) => {
+      const literal = autolinkLiteralText(child)
+      const pieces = literal === null ? [] : splitAtProseUrlTerminators(literal)
+      if (pieces.length < 2) {
+        splitAutolinkLiterals(child)
+        return [child]
+      }
+      return pieces.flatMap((piece, index) => {
+        if (index % 2 === 1) {
+          return [{ type: 'text', value: piece }]
+        }
+        return piece ? parseInline(piece) : []
+      })
+    })
+  }
+
+  return splitAutolinkLiterals
+}

src/shared/prose-url-boundary.ts

@@ -0,0 +1,28 @@
+// Why not all non-ascii punctuation: titles keep `・`, `–` and `·` unencoded in a url
+// (`/wiki/ジョン・スミス`), so only brackets, quotes, spaces and sentence marks end one.
+const PROSE_URL_TERMINATOR_SOURCE = String.raw`(?![\0-\x7f])[\p{Ps}\p{Pe}\p{Pi}\p{Pf}\p{Z}、。，．！？：；｡､…‥—]`
+
+const NOT_PROSE_URL_TERMINATOR = `(?!${PROSE_URL_TERMINATOR_SOURCE})`
+
+/** xterm's default web-links url pattern, also ended by prose punctuation; needs the `u` flag. */
+export const PROSE_HTTP_URL_SOURCE =
+  String.raw`https?:\/\/(?:${NOT_PROSE_URL_TERMINATOR}[^\s"'!*(){}|\\^<>\x60])*` +
+  String.raw`${NOT_PROSE_URL_TERMINATOR}[^\s"':,.!?{}|\\^~\[\]\x60()<>]`
+
+const PROSE_URL_TERMINATOR = new RegExp(PROSE_URL_TERMINATOR_SOURCE, 'u')
+const PROSE_URL_END = new RegExp(String.raw`\s|${PROSE_URL_TERMINATOR_SOURCE}`, 'u')
+const PROSE_URL_TERMINATOR_RUN = new RegExp(`((?:${PROSE_URL_TERMINATOR_SOURCE})+)`, 'u')
+
+export function isProseUrlTerminator(char: string): boolean {
+  return PROSE_URL_TERMINATOR.test(char)
+}
+
+/** Index of the first whitespace or terminator, where a url starting the text ends, or -1. */
+export function findProseUrlEnd(text: string): number {
+  return text.search(PROSE_URL_END)
+}
+
+/** The text cut around each run of terminators; odd entries are the runs. */
+export function splitAtProseUrlTerminators(text: string): string[] {
+  return text.split(PROSE_URL_TERMINATOR_RUN)
+}