From ceb303aa6ccee23096342eb30978dd93aa32334d Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Sat, 18 Jul 2026 21:41:58 +0800 Subject: [PATCH] fix(terminal): prefer the OS tokenizer over jieba for CJK selection jieba's dictionary costs ~55 MB resident and ~130 ms to build, and it was built eagerly on every terminal-view creation regardless of the `smart_select` setting or whether the user ever selects CJK text. macOS already ships a Chinese lexicon in CFStringTokenizer that matches jieba on most prose, is locale-independent (identical output for current / NULL / zh_CN / en_US), and segments Japanese and Korean properly where jieba shreds them into single characters. Make the OS tokenizer the primary path and keep jieba only as the fallback for platforms with no such API: - gate the dependency behind `cfg(not(target_os = "macos"))` so the embedded dictionary isn't even linked into the macOS build - never warm eagerly; the first CJK double-click kicks the build off in the background and settles for the unsegmented run, so the UI thread never blocks on it - skip jieba for runs containing kana or hangul, where selecting the whole run beats per-character tokens Also honor `Config::smart_select` in the prompt's command editor, which did bracket pairing, CJK segmentation and mixed-script narrowing even with the Settings toggle off. --- Cargo.toml | 17 +++-- src/terminal/cmd_editor.rs | 53 ++++++++------ src/terminal/smart_select.rs | 133 ++++++++++++++++++++++++----------- src/terminal/view.rs | 13 ++-- 4 files changed, 140 insertions(+), 76 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 20b739d5..89c0ed7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,10 +24,6 @@ log.workspace = true # Smart double-click selection patterns (URL/email/path). Already in the tree # transitively, so pinning it here adds no new native code. regex = "1" -# Dictionary-based Chinese word segmentation for double-click selection -# (`terminal::smart_select`). The default dict ships deflate-compressed inside -# the binary (~1.8 MB); the table is built lazily on a background thread. -jieba-rs = "0.10" smol.workspace = true smallvec.workspace = true serde = { workspace = true } @@ -179,7 +175,8 @@ winresource = "0.1" # so the native traffic-light buttons render in the right light/dark style. [target.'cfg(target_os = "macos")'.dependencies] # CFStringTokenizer FFI for dictionary-based CJK word segmentation on -# double-click (`terminal::smart_select`). Already in the tree transitively. +# double-click (`terminal::smart_select`). Already in the tree transitively, +# and it makes jieba unnecessary here — see the non-macos section below. core-foundation = "0.10" objc2 = "0.6" objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSResponder", "NSAppearance", "NSGraphics", "NSImage"] } @@ -187,6 +184,16 @@ objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSResponder", " # `set_dock_icon_for_bare_binary` in main.rs. objc2-foundation = { version = "0.3", features = ["NSData"] } +# Dictionary-based Chinese word segmentation for double-click selection +# (`terminal::smart_select`), as a *fallback* where the OS has no tokenizer of +# its own. macOS is excluded on purpose: CFStringTokenizer segments Chinese +# about as well (and Japanese/Korean far better) at zero cost, while jieba's +# table costs ~55 MB resident once built and ~2 MB of binary for the embedded +# dictionary. Keeping the dep off macOS means that dictionary isn't even +# linked into the build that can't use it. +[target.'cfg(not(target_os = "macos"))'.dependencies] +jieba-rs = "0.10" + # x11/wayland are the Linux windowing backends; only pull them on Linux. The # Windows backend (`gpui_windows`) and macOS backend are selected by gpui_platform # itself via `cfg`, so no feature is needed for them. diff --git a/src/terminal/cmd_editor.rs b/src/terminal/cmd_editor.rs index ec551f8a..971bf46f 100644 --- a/src/terminal/cmd_editor.rs +++ b/src/terminal/cmd_editor.rs @@ -228,8 +228,15 @@ impl CmdEditor { /// selection). A separator char is its own one-char word, matching the /// grid; on whitespace the run collapses and the leftward walk snaps to /// the previous word's start. - pub fn word_bounds(&self, idx: usize, separators: &str) -> (usize, usize) { + /// + /// `smart` mirrors `Config::smart_select`: with it off this is exactly + /// [`Self::plain_word_bounds`], so the Settings toggle governs the prompt + /// editor and the grid alike. + pub fn word_bounds(&self, idx: usize, separators: &str, smart: bool) -> (usize, usize) { let idx = idx.min(self.chars.len()); + if !smart { + return self.plain_word_bounds(idx, separators, smart); + } // A bracket or quote selects through its match, same as the grid. // Checked before CJK segmentation so full-width `（）`/`“”` pair // instead of being segmented as lone punctuation tokens. Only for @@ -238,9 +245,8 @@ impl CmdEditor { if let Some((s, e)) = super::smart_select::pair_range(&self.chars, idx) { return (s, e + 1); } - // CJK prose has no separators between words: segment it (jieba for - // Chinese, the OS tokenizer for Kana/Hangul on macOS) instead of - // selecting the whole unbroken run. + // CJK prose has no separators between words: segment it with the + // platform dictionary instead of selecting the whole unbroken run. if let Some(&c) = self.chars.get(idx) && super::smart_select::is_cjk(c) { @@ -249,13 +255,15 @@ impl CmdEditor { return (s, e + 1); } } - self.plain_word_bounds(idx, separators) + self.plain_word_bounds(idx, separators, smart) } /// [`Self::word_bounds`] without the pair/segmentation smarts: the plain /// separator-walk word. Used for word-granular drags, where pair matching /// would make the selection jump around as the pointer crosses a quote. - fn plain_word_bounds(&self, idx: usize, separators: &str) -> (usize, usize) { + /// `smart` still governs the mixed-script narrowing, so a drag matches + /// what the double-click that started it selected. + fn plain_word_bounds(&self, idx: usize, separators: &str, smart: bool) -> (usize, usize) { let idx = idx.min(self.chars.len()); if let Some(&c) = self.chars.get(idx) && !c.is_whitespace() @@ -274,7 +282,7 @@ impl CmdEditor { } // Mixed-script runs (a Latin word glued to CJK text) shrink to the // clicked char's script class — same correction as the grid's. - if idx < e { + if smart && idx < e { let (ns, ne) = super::smart_select::narrow_to_script(&self.chars, idx, s, e - 1); return (ns, ne + 1); } @@ -282,8 +290,8 @@ impl CmdEditor { } /// Select the word containing char index `idx` (see [`Self::word_bounds`]). - pub fn select_word_at(&mut self, idx: usize, separators: &str) { - let (s, e) = self.word_bounds(idx, separators); + pub fn select_word_at(&mut self, idx: usize, separators: &str, smart: bool) { + let (s, e) = self.word_bounds(idx, separators, smart); self.anchor = Some(s); self.cursor = e; } @@ -299,8 +307,9 @@ impl CmdEditor { anchor_end: usize, idx: usize, separators: &str, + smart: bool, ) { - let (ws, we) = self.plain_word_bounds(idx.min(self.chars.len()), separators); + let (ws, we) = self.plain_word_bounds(idx.min(self.chars.len()), separators, smart); if we >= anchor_end { self.anchor = Some(anchor_start); self.cursor = we; @@ -629,7 +638,7 @@ mod tests { #[test] fn select_word_and_all() { let mut e = ed("git push origin", 6); - e.select_word_at(6, SEPS); // cursor on "push" + e.select_word_at(6, SEPS, true); // cursor on "push" assert_eq!(e.selected_text().as_deref(), Some("push")); e.select_all(); assert_eq!(e.selection(), Some((0, 15))); @@ -639,15 +648,15 @@ mod tests { fn select_word_stops_at_separators() { // Quotes and commas bound a word; a separator char is its own word. let mut e = ed("echo 'a,b'", 0); - e.select_word_at(6, SEPS); // on "a" + e.select_word_at(6, SEPS, true); // on "a" assert_eq!(e.selected_text().as_deref(), Some("a")); - e.select_word_at(7, SEPS); // on the comma itself + e.select_word_at(7, SEPS, true); // on the comma itself assert_eq!(e.selected_text().as_deref(), Some(",")); - e.select_word_at(5, SEPS); // on the opening quote: pairs to the close + e.select_word_at(5, SEPS, true); // on the opening quote: pairs to the close assert_eq!(e.selected_text().as_deref(), Some("'a,b'")); // `/ . - _ =` are not separators: a path stays one word. let mut e = ed("cat ./a-b/c_d.txt", 0); - e.select_word_at(8, SEPS); + e.select_word_at(8, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("./a-b/c_d.txt")); } @@ -664,20 +673,20 @@ mod tests { fn extend_word_to_grows_by_whole_words_both_directions() { // Double-click "push" (chars 4..8), then drag over later/earlier words. let mut e = ed("git push origin main", 4); - e.select_word_at(6, SEPS); + e.select_word_at(6, SEPS, true); let (s, a) = e.selection().unwrap(); // (4, 8) == "push" assert_eq!((s, a), (4, 8)); // Drag forward into "origin": selection reaches that word's far edge. - e.extend_word_to(s, a, 10, SEPS); + e.extend_word_to(s, a, 10, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("push origin")); // Drag on into "main": grows to its end. - e.extend_word_to(s, a, 18, SEPS); + e.extend_word_to(s, a, 18, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("push origin main")); // Drag backward before the anchor word into "git": anchor flips to the // word's far edge, selection covers "git push". - e.extend_word_to(s, a, 1, SEPS); + e.extend_word_to(s, a, 1, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("git push")); } @@ -742,15 +751,15 @@ mod tests { // that word — the same left-scan that makes a double-click at the end // of the line select the last word. let mut e = ed("ab cd", 0); - e.select_word_at(2, SEPS); // the space between the words + e.select_word_at(2, SEPS, true); // the space between the words assert_eq!(e.selected_text().as_deref(), Some("ab")); // Index at/past the end selects the trailing word, clamped. - e.select_word_at(99, SEPS); + e.select_word_at(99, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("cd")); // On a gap wider than one cell there is no adjacent word to the left of // the clicked cell: the empty range collapses to no selection. let mut e = ed("ab cd", 0); - e.select_word_at(3, SEPS); // second space: both neighbours are whitespace + e.select_word_at(3, SEPS, true); // second space: both neighbours are whitespace assert_eq!(e.selection(), None); } diff --git a/src/terminal/smart_select.rs b/src/terminal/smart_select.rs index 14968390..291ba194 100644 --- a/src/terminal/smart_select.rs +++ b/src/terminal/smart_select.rs @@ -101,8 +101,8 @@ pub(super) fn grid_smart_range( } // 3) CJK prose has no separators to walk — the whole clause is one run — - // so segment it with the OS tokenizer (dictionary-based on macOS) - // instead of selecting the entire unbroken run. + // so segment it with a dictionary instead of selecting the entire + // unbroken run. No segmenter available means the run stands as-is. if is_cjk(chars[click_idx]) && let Some((s, e)) = cjk_word_range(&text, click_idx) { @@ -129,29 +129,34 @@ pub(super) fn is_cjk(c: char) -> bool { ) } -/// Whether the char routes to jieba: Han ideographs and CJK punctuation, -/// where jieba's Chinese dictionary beats the system tokenizer. Kana and -/// Hangul stay with the platform tokenizer (jieba has no Japanese/Korean -/// dictionary and would return the whole run). -fn prefers_jieba(c: char) -> bool { +/// Kana or Hangul — the scripts jieba has no dictionary for. A run holding +/// either is left unsegmented rather than handed to jieba, which shreds it +/// into single characters (`です` → `で` `す`); selecting the whole run is the +/// friendlier failure. +#[cfg(not(target_os = "macos"))] +fn is_kana_or_hangul(c: char) -> bool { matches!( u32::from(c), - 0x3400..=0x9FFF // Han ideographs (unified + ext A) - | 0xF900..=0xFAFF // compatibility ideographs - | 0x20000..=0x3134F // ideograph extensions - | 0x3000..=0x303F // CJK punctuation - | 0xFF00..=0xFFEF // full-width forms + 0x1100..=0x11FF // Hangul Jamo + | 0x3040..=0x30FF // Hiragana + Katakana + | 0x31F0..=0x31FF // Katakana phonetic extensions + | 0xA960..=0xA97F // Hangul Jamo Extended-A + | 0xAC00..=0xD7FF // Hangul syllables + Jamo Extended-B + | 0xFF66..=0xFF9F // half-width Katakana ) } -/// The jieba segmenter, built once. Building the 350k-entry table takes a -/// few hundred ms, hence [`warm`] to move that off the first double-click. -static JIEBA: std::sync::OnceLock = std::sync::OnceLock::new(); +/// The jieba segmenter, built once on a background thread. The table costs +/// ~55 MB resident and ~130 ms to build, so it is constructed only if a CJK +/// double-click actually happens — see [`jieba_word_range`]. +#[cfg(not(target_os = "macos"))] +static JIEBA: OnceLock = OnceLock::new(); /// Kick off dictionary construction on a background thread (idempotent). -/// Called when a terminal view is created, so the table is ready long before -/// the first CJK double-click; a click that races it just blocks briefly. -pub(crate) fn warm() { +/// Never called eagerly: the first CJK double-click triggers it and settles +/// for the unsegmented run, so the UI thread never blocks on the build. +#[cfg(not(target_os = "macos"))] +fn warm() { static ONCE: std::sync::Once = std::sync::Once::new(); ONCE.call_once(|| { std::thread::spawn(|| { @@ -160,28 +165,30 @@ pub(crate) fn warm() { }); } -/// Dictionary-based word bounds for CJK text: the inclusive char range of -/// the word containing char index `click`. Chinese goes through jieba (full -/// dictionary, all platforms); Kana/Hangul fall back to the platform -/// tokenizer (CFStringTokenizer on macOS), and elsewhere the caller keeps -/// the whole run. +/// Dictionary-based word bounds for CJK text: the inclusive char range of the +/// word containing char index `click`, or `None` to keep the whole run. +/// +/// The OS tokenizer wins wherever there is one. macOS's CFStringTokenizer +/// carries a Chinese lexicon that matches jieba on most prose, is locale- +/// independent, handles Japanese and Korean properly, and costs nothing — +/// jieba is only worth its ~55 MB on platforms with no such API. pub(super) fn cjk_word_range(text: &str, click: usize) -> Option<(usize, usize)> { - let chars: Vec = text.chars().collect(); - let c = *chars.get(click)?; - if prefers_jieba(c) - && let Some(r) = jieba_word_range(&chars, click) - { - return Some(r); - } #[cfg(target_os = "macos")] - if let Some(r) = tokenizer::word_range(text, click) { - return Some(r); + { + tokenizer::word_range(text, click) + } + #[cfg(not(target_os = "macos"))] + { + let chars: Vec = text.chars().collect(); + chars.get(click)?; + jieba_word_range(&chars, click) } - None } /// Segment the contiguous CJK run around `click` with jieba and return the -/// token containing it. +/// token containing it. `None` — meaning "select the whole run" — when the +/// dictionary isn't built yet or the run isn't Chinese. +#[cfg(not(target_os = "macos"))] fn jieba_word_range(chars: &[char], click: usize) -> Option<(usize, usize)> { let mut rs = click; while rs > 0 && is_cjk(chars[rs - 1]) { @@ -191,8 +198,19 @@ fn jieba_word_range(chars: &[char], click: usize) -> Option<(usize, usize)> { while re + 1 < chars.len() && is_cjk(chars[re + 1]) { re += 1; } + // Japanese/Korean: jieba's Chinese dictionary would cut the run into + // single characters, which is worse than not segmenting at all. + if chars[rs..=re].iter().copied().any(is_kana_or_hangul) { + return None; + } + // Building the table takes ~130 ms — far too long to hold the UI thread + // on a click. Start it in the background and let this one click select + // the whole run; every later click finds the table ready. + let Some(jieba) = JIEBA.get() else { + warm(); + return None; + }; let run: String = chars[rs..=re].iter().collect(); - let jieba = JIEBA.get_or_init(jieba_rs::Jieba::new); // Token start/end are Unicode char offsets into `run`. let rel = click - rs; jieba @@ -588,6 +606,14 @@ mod tests { /// alacritty's stock separator set, which is also the config default. const SEPS: &str = ",│`|:\"' ()[]{}<>\t"; + /// In production the jieba table builds lazily off-thread and the racing + /// click settles for the whole run; tests want it ready up front. No-op on + /// macOS, where CFStringTokenizer needs no warm-up. + fn ensure_segmenter() { + #[cfg(not(target_os = "macos"))] + let _ = JIEBA.get_or_init(jieba_rs::Jieba::new); + } + fn range(text: &str, click: usize) -> Option<(usize, usize)> { let chars: Vec = text.chars().collect(); smart_range(text, &chars, click, SEPS) @@ -767,6 +793,7 @@ mod tests { #[test] fn cjk_segmentation_selects_a_dictionary_word_not_the_whole_run() { + ensure_segmenter(); let text = "run 北京欢迎你 done"; let chars: Vec = text.chars().collect(); let click = chars.iter().position(|&c| c == '京').unwrap(); @@ -777,17 +804,22 @@ mod tests { #[test] fn cjk_segmentation_survives_surrogate_pairs_before_the_click() { - // The emoji before the run must not skew the char↔offset mapping. - let text = "🙂 你好世界"; - let chars: Vec = text.chars().collect(); - let click = chars.iter().position(|&c| c == '好').unwrap(); - let (s, e) = cjk_word_range(text, click).expect("segmented range"); - let sel: String = chars[s..=e].iter().collect(); - assert_eq!(sel, "你好"); + // The emoji is two UTF-16 units: a tokenizer offset table that counted + // chars instead would shift every index after it. Both backends agree + // on `世界`, so a skewed mapping shows up as a different token. + ensure_segmenter(); + for text in ["你好世界", "🙂 你好世界", "🙂🙂🙂 你好世界"] { + let chars: Vec = text.chars().collect(); + let click = chars.iter().position(|&c| c == '世').unwrap(); + let (s, e) = cjk_word_range(text, click).expect("segmented range"); + let sel: String = chars[s..=e].iter().collect(); + assert_eq!(sel, "世界", "{text:?} segmented wrong"); + } } #[test] fn cjk_punctuation_is_its_own_token() { + ensure_segmenter(); let text = "比赛，天气"; let chars: Vec = text.chars().collect(); let click = chars.iter().position(|&c| c == '，').unwrap(); @@ -796,6 +828,23 @@ mod tests { assert_eq!(sel, "，"); } + /// Japanese must not be run through jieba's Chinese dictionary — it cuts + /// kana into single characters, which is worse than leaving the run whole. + /// macOS hands it to CFStringTokenizer, which segments it properly. + #[test] + fn japanese_is_not_shredded_into_single_kana() { + ensure_segmenter(); + let text = "日本語の文章です"; + let chars: Vec = text.chars().collect(); + let click = chars.iter().position(|&c| c == 'で').unwrap(); + // macOS yields a real token, never a lone kana; elsewhere the run + // comes back unsegmented and the caller selects all of it. + if let Some((s, e)) = cjk_word_range(text, click) { + let sel: String = chars[s..=e].iter().collect(); + assert_eq!(sel, "です"); + } + } + #[test] fn is_cjk_covers_han_kana_hangul_fullwidth() { for c in ['中', 'あ', 'ア', '한', '，', '（'] { diff --git a/src/terminal/view.rs b/src/terminal/view.rs index 8e047cd5..2fc21a5f 100644 --- a/src/terminal/view.rs +++ b/src/terminal/view.rs @@ -711,9 +711,6 @@ impl TerminalView { window: &mut Window, cx: &mut Context, ) -> anyhow::Result { - // Build the CJK segmentation dictionary off-thread now, so the first - // double-click on Chinese text doesn't pay the ~0.5s table build. - super::smart_select::warm(); // Provisional size; corrected on the first prepaint once we can measure. // The PTY lives in the daemon now. On session restore (`restore_pane`), // re-`attach` to the still-running pane so its process + scrollback come @@ -2852,8 +2849,9 @@ impl TerminalView { self.editor_drag_word = None; } 2 => { - let seps = &cx.global::().word_separators; - self.cmd.select_word_at(idx, seps); + let cfg = cx.global::(); + let (seps, smart) = (cfg.word_separators.clone(), cfg.smart_select); + self.cmd.select_word_at(idx, &seps, smart); // Drag now grows the selection by whole words around this one. self.editor_selecting = true; self.editor_drag_word = self.cmd.selection(); @@ -2884,8 +2882,9 @@ impl TerminalView { }; // A drag begun on a double-click extends by whole words; otherwise by char. if let Some((s, e)) = self.editor_drag_word { - let seps = &cx.global::().word_separators; - self.cmd.extend_word_to(s, e, idx, seps); + let cfg = cx.global::(); + let (seps, smart) = (cfg.word_separators.clone(), cfg.smart_select); + self.cmd.extend_word_to(s, e, idx, &seps, smart); } else { self.cmd.extend_to(idx); }