diff --git a/config/scripts/mobile-file-ranking-benchmark.mjs b/config/scripts/mobile-file-ranking-benchmark.mjs index 68ac5b9d977..27bf99f8ecd 100644 --- a/config/scripts/mobile-file-ranking-benchmark.mjs +++ b/config/scripts/mobile-file-ranking-benchmark.mjs @@ -1,53 +1,141 @@ import assert from 'node:assert/strict' import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' -import { stripTypeScriptTypes } from 'node:module' import { performance } from 'node:perf_hooks' +import { transform } from 'esbuild' +import { buildCounterbalancedSchedule } from './counterbalanced-benchmark-schedule.mjs' +import { summarizeBenchmarkSamples } from './benchmark-sample-summary.mjs' const baseline = process.argv[2] if (!baseline) { - throw new Error('Usage: node config/scripts/mobile-file-ranking-benchmark.mjs ') + throw new Error( + 'Usage: node config/scripts/mobile-file-ranking-benchmark.mjs ' + ) } +// git show :mobile/src/session/mobile-native-chat-autocomplete.ts | node config/scripts/mobile-file-ranking-benchmark.mjs --autocomplete-stdin +const autocompleteSource = baseline === '--autocomplete-stdin' ? readFileSync(0, 'utf8') : null async function load(source) { - const js = stripTypeScriptTypes(source, { mode: 'transform' }) - return await import(`data:text/javascript;base64,${Buffer.from(js).toString('base64')}`) -} -function measure(fn, paths, query) { - for (let warmup = 0; warmup < 10; warmup++) { - fn(paths, query, 16) - } - const samples = [] - for (let i = 0; i < 9; i++) { - const start = performance.now() - fn(paths, query, 16) - samples.push(performance.now() - start) - } - return samples.sort((a, b) => a - b)[4] + const { code } = await transform(source, { loader: 'ts', format: 'esm' }) + return await import(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`) } const results = [] +let differentialCases = 0 for (const [file, name] of [ ['src/main/runtime/runtime-mobile-file-path-search.ts', 'rankRuntimeMobileFilePaths'], - ['mobile/src/session/mobile-native-chat-autocomplete.ts', 'rankSuggestions'] + ['mobile/src/session/mobile-native-chat-autocomplete.ts', 'rankSuggestions'], + ['mobile/src/session/mobile-native-chat-autocomplete.ts', 'rankSlashCommandSuggestions'] ]) { + if (autocompleteSource !== null && name === 'rankRuntimeMobileFilePaths') { + continue + } const before = ( - await load(execFileSync('git', ['show', `${baseline}:${file}`], { encoding: 'utf8' })) + await load( + autocompleteSource ?? + execFileSync('git', ['show', `${baseline}:${file}`], { encoding: 'utf8' }) + ) )[name] const after = (await load(readFileSync(file, 'utf8')))[name] - for (const count of [100, 100000]) { - const paths = Array.from( - { length: count }, - (_, i) => `src/components/workspace/group-${i % 100}/file-${i}.tsx` + const slash = name === 'rankSlashCommandSuggestions' + const toCandidates = (names) => + slash ? names.map((name, index) => ({ name, description: `Command ${index}` })) : names + if (name !== 'rankRuntimeMobileFilePaths') { + let seed = 42 + const random = (max) => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0 + return seed % max + } + const tokens = ['', 'app', 'src/', 'APP', 'zapp', '🙂', '한', '\ud800', '\u0130', ' '] + const limits = [ + undefined, + 0, + -0, + -1, + -0.5, + -Infinity, + Number.NaN, + 0.5, + 1.5, + 2.5, + 8, + 16, + Infinity + ] + for (let index = 0; index < 3000; index += 1) { + const candidates = toCandidates( + Array.from( + { length: random(100) }, + () => tokens[random(tokens.length)] + tokens[random(tokens.length)] + ) + ) + const query = tokens[random(tokens.length)] + const limit = limits[random(limits.length)] + assert.deepEqual(after(candidates, query, limit), before(candidates, query, limit)) + differentialCases += 1 + } + } + for (const count of slash ? [16, 100, 1000] : [16, 100, 10_000, 50_000, 100_000]) { + const names = Array.from({ length: count }, (_, index) => + slash + ? `team-review-${index}` + : `src/components/workspace/group-${index % 100}/file-${index}.tsx` ) - for (const query of ['file-9', 'missing', 'workspace']) { - assert.deepEqual(after(paths, query, 16), before(paths, query, 16)) + const limit = slash ? 12 : 16 + const substringQuery = slash ? 'review' : 'workspace' + const workloads = [ + { name: 'empty-query', names, query: '' }, + { name: 'substring', names, query: substringQuery }, + { name: 'no-match', names, query: 'missing' }, + { name: 'early-prefix', names, query: slash ? 'team' : 'file' }, + { + name: 'late-prefix', + names: [...names, ...Array.from({ length: 4 }, (_, index) => `${substringQuery}-${index}`)], + query: substringQuery + } + ] + for (const workload of workloads) { + const candidates = toCandidates(workload.names) + const expected = before(candidates, workload.query, limit) + assert.deepEqual(after(candidates, workload.query, limit), expected) + const implementations = { before, after } + const iterations = Math.max(10, Math.floor(100_000 / count)) + for (let warmup = 0; warmup < 100; warmup += 1) { + before(candidates, workload.query, limit) + after(candidates, workload.query, limit) + } + /** @type {{ before: number[], after: number[] }} */ + const samples = { before: [], after: [] } + for (const pair of buildCounterbalancedSchedule(8, 'before', 'after')) { + for (const arm of pair) { + let actual + const start = performance.now() + for (let repeat = 0; repeat < iterations; repeat += 1) { + actual = implementations[arm](candidates, workload.query, limit) + } + samples[arm].push(performance.now() - start) + assert.deepEqual(actual, expected) + } + } results.push({ function: name, - paths: count, - query, - beforeMs: measure(before, paths, query), - afterMs: measure(after, paths, query) + candidates: candidates.length, + workload: workload.name, + iterations, + meanMicrosecondsPerCall: Object.fromEntries( + Object.entries(samples).map(([arm, values]) => [ + arm, + (values.reduce((sum, ms) => sum + ms, 0) * 1000) / values.length / iterations + ]) + ), + before: summarizeBenchmarkSamples(samples.before), + after: summarizeBenchmarkSamples(samples.after) }) } } } -console.log(JSON.stringify({ node: process.version, platform: process.platform, results }, null, 2)) +console.log( + JSON.stringify( + { node: process.version, platform: process.platform, differentialCases, results }, + null, + 2 + ) +) diff --git a/mobile/src/session/mobile-native-chat-autocomplete.test.ts b/mobile/src/session/mobile-native-chat-autocomplete.test.ts index 608917449d4..7fac03e7354 100644 --- a/mobile/src/session/mobile-native-chat-autocomplete.test.ts +++ b/mobile/src/session/mobile-native-chat-autocomplete.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { applyAutocomplete, detectAutocompleteTrigger, @@ -90,4 +90,77 @@ describe('rankSlashCommandSuggestions', () => { expect(rankSlashCommandSuggestions(commands, 'CLE').map((c) => c.name)).toEqual(['clear']) expect(rankSlashCommandSuggestions(commands, 'zzz')).toEqual([]) }) + + it('preserves command identity and metadata for duplicate names', () => { + const catalog = [ + { name: 'team-review', description: 'First' }, + { name: 'team-review', argumentHint: '' }, + { name: 'review', kindUnspecified: true as const } + ] + const result = rankSlashCommandSuggestions(catalog, 'review', 3) + expect(result[0]).toBe(catalog[2]) + expect(result[1]).toBe(catalog[0]) + expect(result[2]).toBe(catalog[1]) + }) +}) + +describe.each([ + { name: 'file', rank: rankSuggestions }, + { + name: 'slash', + rank: (names: readonly string[], query: string, limit: number): string[] => + rankSlashCommandSuggestions( + names.map((name) => ({ name })), + query, + limit + ).map((command) => command.name) + } +])('$name suggestion bounds', ({ rank }) => { + it('keeps later prefixes ahead of the earliest substring matches', () => { + const candidates = ['team-review', 'team-review', 'pre-review', 'review-a', 'REVIEW-b'] + expect(rank(candidates, 'REVIEW', 4)).toEqual([ + 'review-a', + 'REVIEW-b', + 'team-review', + 'team-review' + ]) + }) + + it('stops substring matching once enough fallback suggestions are retained', () => { + const candidates = Array.from({ length: 10_000 }, (_, index) => `team-review-${index}`) + candidates.push('review-last', 'review-final') + const includes = String.prototype.includes + let substringChecks = 0 + const spy = vi.spyOn(String.prototype, 'includes').mockImplementation(function ( + this: string, + search: string, + position?: number + ) { + substringChecks += 1 + return includes.call(this, search, position) + }) + let result: string[] + try { + result = rank(candidates, 'review', 8) + } finally { + spy.mockRestore() + } + expect(result).toEqual(['review-last', 'review-final', ...candidates.slice(0, 6)]) + expect(substringChecks).toBeLessThanOrEqual(8) + }) + + it.each([0, -0, -1, -0.5, -Infinity, Number.NaN])( + 'preserves empty results for limit %s', + (limit) => { + expect(rank(['team-review', 'review-a', 'review-b'], 'review', limit)).toEqual([]) + } + ) + + it.each([0.5, 1.5, 2.5, Infinity])('preserves slice truncation for limit %s', (limit) => { + const candidates = ['team-review', 'pre-review', 'review-a', 'review-b'] + expect(rank(candidates, 'review', limit)).toEqual( + ['review-a', 'review-b', 'team-review', 'pre-review'].slice(0, limit) + ) + expect(rank(candidates, '', limit)).toEqual(candidates.slice(0, limit)) + }) }) diff --git a/mobile/src/session/mobile-native-chat-autocomplete.ts b/mobile/src/session/mobile-native-chat-autocomplete.ts index 8de107c9fb0..5bbeb8a5d65 100644 --- a/mobile/src/session/mobile-native-chat-autocomplete.ts +++ b/mobile/src/session/mobile-native-chat-autocomplete.ts @@ -84,7 +84,7 @@ export function rankSuggestions(candidates: readonly string[], query: string, li const base = lower.slice(lower.lastIndexOf('/') + 1) if (lower.startsWith(q) || base.startsWith(q)) { prefix.push(candidate) - } else if (lower.includes(q)) { + } else if (substring.length < limit && lower.includes(q)) { substring.push(candidate) } if (prefix.length >= limit) { @@ -112,7 +112,7 @@ export function rankSlashCommandSuggestions( const lower = command.name.toLowerCase() if (lower.startsWith(q)) { prefix.push(command) - } else if (lower.includes(q)) { + } else if (substring.length < limit && lower.includes(q)) { substring.push(command) } if (prefix.length >= limit) {