diff --git a/Cargo.toml b/Cargo.toml index 20b739d5..89c0ed7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,10 +24,6 @@ log.workspace = true # Smart double-click selection patterns (URL/email/path). Already in the tree # transitively, so pinning it here adds no new native code. regex = "1" -# Dictionary-based Chinese word segmentation for double-click selection -# (`terminal::smart_select`). The default dict ships deflate-compressed inside -# the binary (~1.8 MB); the table is built lazily on a background thread. -jieba-rs = "0.10" smol.workspace = true smallvec.workspace = true serde = { workspace = true } @@ -179,7 +175,8 @@ winresource = "0.1" # so the native traffic-light buttons render in the right light/dark style. [target.'cfg(target_os = "macos")'.dependencies] # CFStringTokenizer FFI for dictionary-based CJK word segmentation on -# double-click (`terminal::smart_select`). Already in the tree transitively. +# double-click (`terminal::smart_select`). Already in the tree transitively, +# and it makes jieba unnecessary here — see the non-macos section below. core-foundation = "0.10" objc2 = "0.6" objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSResponder", "NSAppearance", "NSGraphics", "NSImage"] } @@ -187,6 +184,16 @@ objc2-app-kit = { version = "0.3", features = ["NSApplication", "NSResponder", " # `set_dock_icon_for_bare_binary` in main.rs. objc2-foundation = { version = "0.3", features = ["NSData"] } +# Dictionary-based Chinese word segmentation for double-click selection +# (`terminal::smart_select`), as a *fallback* where the OS has no tokenizer of +# its own. macOS is excluded on purpose: CFStringTokenizer segments Chinese +# about as well (and Japanese/Korean far better) at zero cost, while jieba's +# table costs ~55 MB resident once built and ~2 MB of binary for the embedded +# dictionary. Keeping the dep off macOS means that dictionary isn't even +# linked into the build that can't use it. +[target.'cfg(not(target_os = "macos"))'.dependencies] +jieba-rs = "0.10" + # x11/wayland are the Linux windowing backends; only pull them on Linux. The # Windows backend (`gpui_windows`) and macOS backend are selected by gpui_platform # itself via `cfg`, so no feature is needed for them. diff --git a/src/terminal/cmd_editor.rs b/src/terminal/cmd_editor.rs index ec551f8a..971bf46f 100644 --- a/src/terminal/cmd_editor.rs +++ b/src/terminal/cmd_editor.rs @@ -228,8 +228,15 @@ impl CmdEditor { /// selection). A separator char is its own one-char word, matching the /// grid; on whitespace the run collapses and the leftward walk snaps to /// the previous word's start. - pub fn word_bounds(&self, idx: usize, separators: &str) -> (usize, usize) { + /// + /// `smart` mirrors `Config::smart_select`: with it off this is exactly + /// [`Self::plain_word_bounds`], so the Settings toggle governs the prompt + /// editor and the grid alike. + pub fn word_bounds(&self, idx: usize, separators: &str, smart: bool) -> (usize, usize) { let idx = idx.min(self.chars.len()); + if !smart { + return self.plain_word_bounds(idx, separators, smart); + } // A bracket or quote selects through its match, same as the grid. // Checked before CJK segmentation so full-width `（）`/`“”` pair // instead of being segmented as lone punctuation tokens. Only for @@ -238,9 +245,8 @@ impl CmdEditor { if let Some((s, e)) = super::smart_select::pair_range(&self.chars, idx) { return (s, e + 1); } - // CJK prose has no separators between words: segment it (jieba for - // Chinese, the OS tokenizer for Kana/Hangul on macOS) instead of - // selecting the whole unbroken run. + // CJK prose has no separators between words: segment it with the + // platform dictionary instead of selecting the whole unbroken run. if let Some(&c) = self.chars.get(idx) && super::smart_select::is_cjk(c) { @@ -249,13 +255,15 @@ impl CmdEditor { return (s, e + 1); } } - self.plain_word_bounds(idx, separators) + self.plain_word_bounds(idx, separators, smart) } /// [`Self::word_bounds`] without the pair/segmentation smarts: the plain /// separator-walk word. Used for word-granular drags, where pair matching /// would make the selection jump around as the pointer crosses a quote. - fn plain_word_bounds(&self, idx: usize, separators: &str) -> (usize, usize) { + /// `smart` still governs the mixed-script narrowing, so a drag matches + /// what the double-click that started it selected. + fn plain_word_bounds(&self, idx: usize, separators: &str, smart: bool) -> (usize, usize) { let idx = idx.min(self.chars.len()); if let Some(&c) = self.chars.get(idx) && !c.is_whitespace() @@ -274,7 +282,7 @@ impl CmdEditor { } // Mixed-script runs (a Latin word glued to CJK text) shrink to the // clicked char's script class — same correction as the grid's. - if idx < e { + if smart && idx < e { let (ns, ne) = super::smart_select::narrow_to_script(&self.chars, idx, s, e - 1); return (ns, ne + 1); } @@ -282,8 +290,8 @@ impl CmdEditor { } /// Select the word containing char index `idx` (see [`Self::word_bounds`]). - pub fn select_word_at(&mut self, idx: usize, separators: &str) { - let (s, e) = self.word_bounds(idx, separators); + pub fn select_word_at(&mut self, idx: usize, separators: &str, smart: bool) { + let (s, e) = self.word_bounds(idx, separators, smart); self.anchor = Some(s); self.cursor = e; } @@ -299,8 +307,9 @@ impl CmdEditor { anchor_end: usize, idx: usize, separators: &str, + smart: bool, ) { - let (ws, we) = self.plain_word_bounds(idx.min(self.chars.len()), separators); + let (ws, we) = self.plain_word_bounds(idx.min(self.chars.len()), separators, smart); if we >= anchor_end { self.anchor = Some(anchor_start); self.cursor = we; @@ -629,7 +638,7 @@ mod tests { #[test] fn select_word_and_all() { let mut e = ed("git push origin", 6); - e.select_word_at(6, SEPS); // cursor on "push" + e.select_word_at(6, SEPS, true); // cursor on "push" assert_eq!(e.selected_text().as_deref(), Some("push")); e.select_all(); assert_eq!(e.selection(), Some((0, 15))); @@ -639,15 +648,15 @@ mod tests { fn select_word_stops_at_separators() { // Quotes and commas bound a word; a separator char is its own word. let mut e = ed("echo 'a,b'", 0); - e.select_word_at(6, SEPS); // on "a" + e.select_word_at(6, SEPS, true); // on "a" assert_eq!(e.selected_text().as_deref(), Some("a")); - e.select_word_at(7, SEPS); // on the comma itself + e.select_word_at(7, SEPS, true); // on the comma itself assert_eq!(e.selected_text().as_deref(), Some(",")); - e.select_word_at(5, SEPS); // on the opening quote: pairs to the close + e.select_word_at(5, SEPS, true); // on the opening quote: pairs to the close assert_eq!(e.selected_text().as_deref(), Some("'a,b'")); // `/ . - _ =` are not separators: a path stays one word. let mut e = ed("cat ./a-b/c_d.txt", 0); - e.select_word_at(8, SEPS); + e.select_word_at(8, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("./a-b/c_d.txt")); } @@ -664,20 +673,20 @@ mod tests { fn extend_word_to_grows_by_whole_words_both_directions() { // Double-click "push" (chars 4..8), then drag over later/earlier words. let mut e = ed("git push origin main", 4); - e.select_word_at(6, SEPS); + e.select_word_at(6, SEPS, true); let (s, a) = e.selection().unwrap(); // (4, 8) == "push" assert_eq!((s, a), (4, 8)); // Drag forward into "origin": selection reaches that word's far edge. - e.extend_word_to(s, a, 10, SEPS); + e.extend_word_to(s, a, 10, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("push origin")); // Drag on into "main": grows to its end. - e.extend_word_to(s, a, 18, SEPS); + e.extend_word_to(s, a, 18, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("push origin main")); // Drag backward before the anchor word into "git": anchor flips to the // word's far edge, selection covers "git push". - e.extend_word_to(s, a, 1, SEPS); + e.extend_word_to(s, a, 1, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("git push")); } @@ -742,15 +751,15 @@ mod tests { // that word — the same left-scan that makes a double-click at the end // of the line select the last word. let mut e = ed("ab cd", 0); - e.select_word_at(2, SEPS); // the space between the words + e.select_word_at(2, SEPS, true); // the space between the words assert_eq!(e.selected_text().as_deref(), Some("ab")); // Index at/past the end selects the trailing word, clamped. - e.select_word_at(99, SEPS); + e.select_word_at(99, SEPS, true); assert_eq!(e.selected_text().as_deref(), Some("cd")); // On a gap wider than one cell there is no adjacent word to the left of // the clicked cell: the empty range collapses to no selection. let mut e = ed("ab cd", 0); - e.select_word_at(3, SEPS); // second space: both neighbours are whitespace + e.select_word_at(3, SEPS, true); // second space: both neighbours are whitespace assert_eq!(e.selection(), None); } diff --git a/src/terminal/smart_select.rs b/src/terminal/smart_select.rs index 14968390..291ba194 100644 --- a/src/terminal/smart_select.rs +++ b/src/terminal/smart_select.rs @@ -101,8 +101,8 @@ pub(super) fn grid_smart_range( } // 3) CJK prose has no separators to walk — the whole clause is one run — - // so segment it with the OS tokenizer (dictionary-based on macOS) - // instead of selecting the entire unbroken run. + // so segment it with a dictionary instead of selecting the entire + // unbroken run. No segmenter available means the run stands as-is. if is_cjk(chars[click_idx]) && let Some((s, e)) = cjk_word_range(&text, click_idx) { @@ -129,29 +129,34 @@ pub(super) fn is_cjk(c: char) -> bool { ) } -/// Whether the char routes to jieba: Han ideographs and CJK punctuation, -/// where jieba's Chinese dictionary beats the system tokenizer. Kana and -/// Hangul stay with the platform tokenizer (jieba has no Japanese/Korean -/// dictionary and would return the whole run). -fn prefers_jieba(c: char) -> bool { +/// Kana or Hangul — the scripts jieba has no dictionary for. A run holding +/// either is left unsegmented rather than handed to jieba, which shreds it +/// into single characters (`です` → `で` `す`); selecting the whole run is the +/// friendlier failure. +#[cfg(not(target_os = "macos"))] +fn is_kana_or_hangul(c: char) -> bool { matches!( u32::from(c), - 0x3400..=0x9FFF // Han ideographs (unified + ext A) - | 0xF900..=0xFAFF // compatibility ideographs - | 0x20000..=0x3134F // ideograph extensions - | 0x3000..=0x303F // CJK punctuation - | 0xFF00..=0xFFEF // full-width forms + 0x1100..=0x11FF // Hangul Jamo + | 0x3040..=0x30FF // Hiragana + Katakana + | 0x31F0..=0x31FF // Katakana phonetic extensions + | 0xA960..=0xA97F // Hangul Jamo Extended-A + | 0xAC00..=0xD7FF // Hangul syllables + Jamo Extended-B + | 0xFF66..=0xFF9F // half-width Katakana ) } -/// The jieba segmenter, built once. Building the 350k-entry table takes a -/// few hundred ms, hence [`warm`] to move that off the first double-click. -static JIEBA: std::sync::OnceLock = std::sync::OnceLock::new(); +/// The jieba segmenter, built once on a background thread. The table costs +/// ~55 MB resident and ~130 ms to build, so it is constructed only if a CJK +/// double-click actually happens — see [`jieba_word_range`]. +#[cfg(not(target_os = "macos"))] +static JIEBA: OnceLock = OnceLock::new(); /// Kick off dictionary construction on a background thread (idempotent). -/// Called when a terminal view is created, so the table is ready long before -/// the first CJK double-click; a click that races it just blocks briefly. -pub(crate) fn warm() { +/// Never called eagerly: the first CJK double-click triggers it and settles +/// for the unsegmented run, so the UI thread never blocks on the build. +#[cfg(not(target_os = "macos"))] +fn warm() { static ONCE: std::sync::Once = std::sync::Once::new(); ONCE.call_once(|| { std::thread::spawn(|| { @@ -160,28 +165,30 @@ pub(crate) fn warm() { }); } -/// Dictionary-based word bounds for CJK text: the inclusive char range of -/// the word containing char index `click`. Chinese goes through jieba (full -/// dictionary, all platforms); Kana/Hangul fall back to the platform -/// tokenizer (CFStringTokenizer on macOS), and elsewhere the caller keeps -/// the whole run. +/// Dictionary-based word bounds for CJK text: the inclusive char range of the +/// word containing char index `click`, or `None` to keep the whole run. +/// +/// The OS tokenizer wins wherever there is one. macOS's CFStringTokenizer +/// carries a Chinese lexicon that matches jieba on most prose, is locale- +/// independent, handles Japanese and Korean properly, and costs nothing — +/// jieba is only worth its ~55 MB on platforms with no such API. pub(super) fn cjk_word_range(text: &str, click: usize) -> Option<(usize, usize)> { - let chars: Vec = text.chars().collect(); - let c = *chars.get(click)?; - if prefers_jieba(c) - && let Some(r) = jieba_word_range(&chars, click) - { - return Some(r); - } #[cfg(target_os = "macos")] - if let Some(r) = tokenizer::word_range(text, click) { - return Some(r); + { + tokenizer::word_range(text, click) + } + #[cfg(not(target_os = "macos"))] + { + let chars: Vec = text.chars().collect(); + chars.get(click)?; + jieba_word_range(&chars, click) } - None } /// Segment the contiguous CJK run around `click` with jieba and return the -/// token containing it. +/// token containing it. `None` — meaning "select the whole run" — when the +/// dictionary isn't built yet or the run isn't Chinese. +#[cfg(not(target_os = "macos"))] fn jieba_word_range(chars: &[char], click: usize) -> Option<(usize, usize)> { let mut rs = click; while rs > 0 && is_cjk(chars[rs - 1]) { @@ -191,8 +198,19 @@ fn jieba_word_range(chars: &[char], click: usize) -> Option<(usize, usize)> { while re + 1 < chars.len() && is_cjk(chars[re + 1]) { re += 1; } + // Japanese/Korean: jieba's Chinese dictionary would cut the run into + // single characters, which is worse than not segmenting at all. + if chars[rs..=re].iter().copied().any(is_kana_or_hangul) { + return None; + } + // Building the table takes ~130 ms — far too long to hold the UI thread + // on a click. Start it in the background and let this one click select + // the whole run; every later click finds the table ready. + let Some(jieba) = JIEBA.get() else { + warm(); + return None; + }; let run: String = chars[rs..=re].iter().collect(); - let jieba = JIEBA.get_or_init(jieba_rs::Jieba::new); // Token start/end are Unicode char offsets into `run`. let rel = click - rs; jieba @@ -588,6 +606,14 @@ mod tests { /// alacritty's stock separator set, which is also the config default. const SEPS: &str = ",│`|:\"' ()[]{}<>\t"; + /// In production the jieba table builds lazily off-thread and the racing + /// click settles for the whole run; tests want it ready up front. No-op on + /// macOS, where CFStringTokenizer needs no warm-up. + fn ensure_segmenter() { + #[cfg(not(target_os = "macos"))] + let _ = JIEBA.get_or_init(jieba_rs::Jieba::new); + } + fn range(text: &str, click: usize) -> Option<(usize, usize)> { let chars: Vec = text.chars().collect(); smart_range(text, &chars, click, SEPS) @@ -767,6 +793,7 @@ mod tests { #[test] fn cjk_segmentation_selects_a_dictionary_word_not_the_whole_run() { + ensure_segmenter(); let text = "run 北京欢迎你 done"; let chars: Vec = text.chars().collect(); let click = chars.iter().position(|&c| c == '京').unwrap(); @@ -777,17 +804,22 @@ mod tests { #[test] fn cjk_segmentation_survives_surrogate_pairs_before_the_click() { - // The emoji before the run must not skew the char↔offset mapping. - let text = "🙂 你好世界"; - let chars: Vec = text.chars().collect(); - let click = chars.iter().position(|&c| c == '好').unwrap(); - let (s, e) = cjk_word_range(text, click).expect("segmented range"); - let sel: String = chars[s..=e].iter().collect(); - assert_eq!(sel, "你好"); + // The emoji is two UTF-16 units: a tokenizer offset table that counted + // chars instead would shift every index after it. Both backends agree + // on `世界`, so a skewed mapping shows up as a different token. + ensure_segmenter(); + for text in ["你好世界", "🙂 你好世界", "🙂🙂🙂 你好世界"] { + let chars: Vec = text.chars().collect(); + let click = chars.iter().position(|&c| c == '世').unwrap(); + let (s, e) = cjk_word_range(text, click).expect("segmented range"); + let sel: String = chars[s..=e].iter().collect(); + assert_eq!(sel, "世界", "{text:?} segmented wrong"); + } } #[test] fn cjk_punctuation_is_its_own_token() { + ensure_segmenter(); let text = "比赛，天气"; let chars: Vec = text.chars().collect(); let click = chars.iter().position(|&c| c == '，').unwrap(); @@ -796,6 +828,23 @@ mod tests { assert_eq!(sel, "，"); } + /// Japanese must not be run through jieba's Chinese dictionary — it cuts + /// kana into single characters, which is worse than leaving the run whole. + /// macOS hands it to CFStringTokenizer, which segments it properly. + #[test] + fn japanese_is_not_shredded_into_single_kana() { + ensure_segmenter(); + let text = "日本語の文章です"; + let chars: Vec = text.chars().collect(); + let click = chars.iter().position(|&c| c == 'で').unwrap(); + // macOS yields a real token, never a lone kana; elsewhere the run + // comes back unsegmented and the caller selects all of it. + if let Some((s, e)) = cjk_word_range(text, click) { + let sel: String = chars[s..=e].iter().collect(); + assert_eq!(sel, "です"); + } + } + #[test] fn is_cjk_covers_han_kana_hangul_fullwidth() { for c in ['中', 'あ', 'ア', '한', '，', '（'] { diff --git a/src/terminal/view.rs b/src/terminal/view.rs index 8e047cd5..2fc21a5f 100644 --- a/src/terminal/view.rs +++ b/src/terminal/view.rs @@ -711,9 +711,6 @@ impl TerminalView { window: &mut Window, cx: &mut Context, ) -> anyhow::Result { - // Build the CJK segmentation dictionary off-thread now, so the first - // double-click on Chinese text doesn't pay the ~0.5s table build. - super::smart_select::warm(); // Provisional size; corrected on the first prepaint once we can measure. // The PTY lives in the daemon now. On session restore (`restore_pane`), // re-`attach` to the still-running pane so its process + scrollback come @@ -2852,8 +2849,9 @@ impl TerminalView { self.editor_drag_word = None; } 2 => { - let seps = &cx.global::().word_separators; - self.cmd.select_word_at(idx, seps); + let cfg = cx.global::(); + let (seps, smart) = (cfg.word_separators.clone(), cfg.smart_select); + self.cmd.select_word_at(idx, &seps, smart); // Drag now grows the selection by whole words around this one. self.editor_selecting = true; self.editor_drag_word = self.cmd.selection(); @@ -2884,8 +2882,9 @@ impl TerminalView { }; // A drag begun on a double-click extends by whole words; otherwise by char. if let Some((s, e)) = self.editor_drag_word { - let seps = &cx.global::().word_separators; - self.cmd.extend_word_to(s, e, idx, seps); + let cfg = cx.global::(); + let (seps, smart) = (cfg.word_separators.clone(), cfg.smart_select); + self.cmd.extend_word_to(s, e, idx, &seps, smart); } else { self.cmd.extend_to(idx); }