mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 16:02:32 +00:00
* feat(ssh): stream fs.readFile to lift 10MB SSH preview cap (#1095)
Replaces the single-shot fs.readFile path on the SSH relay with a
push-style stream protocol modeled on VS Code's readFileStream.
Wire shape:
- fs.readFileStream request returns metadata (streamId, totalSize,
isBinary, mimeType, chunkEncoding, resultEncoding, optional empty)
- Relay pumps fs.streamChunk notifications (256 KB base64 chunks) and
ends with fs.streamEnd or fs.streamError
- Client cancels via fs.cancelStream notification
Invariants:
- Max 16 concurrent streams per FsHandler (TooManyStreams)
- Client clamps totalSize against caps before allocating
- Sequence-number defense against out-of-order/missing chunks
- Subscribe-before-await with frame queueing until streamId is known
- Pump cleans up registry+handle in finally; disposeAll aborts before
release so in-flight reads exit cleanly instead of EBADF
- Empty files short-circuit (no streamId, no handle open)
Compat:
- New client tries fs.readFileStream first, falls back to legacy
fs.readFile on JSON-RPC -32601 (with once-per-session warn log)
- Bumps MAX_PREVIEWABLE_BINARY_SIZE 10 MB to 50 MB to match local
Tests: 91 streaming tests across relay, client, mux, integration.
Co-authored-by: Orca <help@stably.ai>
* test(ssh): wait for streamEnd instead of fixed flush() in stream test
Why: the binary-streaming test relied on 5 setImmediate ticks to drain
the pump, which is racy on slower CI runners (each handle.read is async
I/O). Swap to a deadline-bounded waitFor(streamEnd) so the test is
deterministic regardless of scheduler latency.
Co-authored-by: Orca <help@stably.ai>
* fix(ssh): preserve small binary detection in streamed reads
Co-authored-by: Orca <help@stably.ai>
* fix(ssh): rebind file watcher when connection id hydrates
Co-authored-by: Orca <help@stably.ai>
* fix(ssh): refresh explorer for update-only file creates
Co-authored-by: Orca <help@stably.ai>
* fix(ssh): recompute file watches when repo connection changes
Co-authored-by: Orca <help@stably.ai>
* Revert "fix(ssh): refresh explorer for update-only file creates"
This reverts commit 7c3c683cd0.
* fix(ssh): install relay watcher dependency
Co-authored-by: Orca <help@stably.ai>
---------
Co-authored-by: Orca <help@stably.ai>
193 lines
6.4 KiB
TypeScript
193 lines
6.4 KiB
TypeScript
/**
|
|
* Pure helpers and child-process search utilities extracted from fs-handler.ts.
|
|
*
|
|
* Why: oxlint max-lines requires .ts files to stay under 300 lines.
|
|
* These functions depend only on their arguments (plus `rg` being on PATH),
|
|
* so they are straightforward to test independently.
|
|
*/
|
|
import { spawn, execFile } from 'child_process'
|
|
import { open } from 'fs/promises'
|
|
import {
|
|
buildRgArgs,
|
|
createAccumulator,
|
|
finalize,
|
|
ingestRgJsonLine,
|
|
SEARCH_TIMEOUT_MS as SHARED_SEARCH_TIMEOUT_MS
|
|
} from '../shared/text-search'
|
|
import type { SearchResult as SharedSearchResult } from '../shared/types'
|
|
|
|
// ─── Constants ───────────────────────────────────────────────────────
|
|
|
|
// Why: remote reads still travel through bounded JSON-RPC frames, but matching
|
|
// the old 5MB search cap would block common JSON/log files before Monaco's
|
|
// large-file optimizations can handle them.
|
|
export const MAX_TEXT_FILE_SIZE = 10 * 1024 * 1024
|
|
// Why: matches the local cap (src/main/ipc/filesystem.ts MAX_PREVIEWABLE_BINARY_SIZE).
|
|
// Reads above the legacy 16MB single-frame budget go through fs.readFileStream,
|
|
// which chunks at STREAM_CHUNK_SIZE; see docs/relay-file-stream-design.md.
|
|
export const MAX_PREVIEWABLE_BINARY_SIZE = 50 * 1024 * 1024
|
|
export const BINARY_PROBE_BYTES = 8192
|
|
export const SEARCH_TIMEOUT_MS = SHARED_SEARCH_TIMEOUT_MS
|
|
export const DEFAULT_MAX_RESULTS = 2000
|
|
|
|
export const IMAGE_MIME_TYPES: Record<string, string> = {
|
|
'.png': 'image/png',
|
|
'.jpg': 'image/jpeg',
|
|
'.jpeg': 'image/jpeg',
|
|
'.gif': 'image/gif',
|
|
'.svg': 'image/svg+xml',
|
|
'.webp': 'image/webp',
|
|
'.bmp': 'image/bmp',
|
|
'.ico': 'image/x-icon',
|
|
'.pdf': 'application/pdf'
|
|
}
|
|
|
|
// ─── Binary detection ────────────────────────────────────────────────
|
|
|
|
export function isBinaryBuffer(buffer: Buffer): boolean {
|
|
const len = Math.min(buffer.length, 8192)
|
|
for (let i = 0; i < len; i++) {
|
|
if (buffer[i] === 0) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
export async function isBinaryFilePrefix(filePath: string): Promise<boolean> {
|
|
const handle = await open(filePath, 'r')
|
|
try {
|
|
const probe = Buffer.alloc(BINARY_PROBE_BYTES)
|
|
const { bytesRead } = await handle.read(probe, 0, probe.length, 0)
|
|
return isBinaryBuffer(probe.subarray(0, bytesRead))
|
|
} finally {
|
|
await handle.close()
|
|
}
|
|
}
|
|
|
|
// ─── Search types ────────────────────────────────────────────────────
|
|
|
|
export type SearchOptions = {
|
|
caseSensitive?: boolean
|
|
wholeWord?: boolean
|
|
useRegex?: boolean
|
|
includePattern?: string
|
|
excludePattern?: string
|
|
maxResults: number
|
|
}
|
|
|
|
export type SearchResult = SharedSearchResult
|
|
|
|
// ─── rg-based search ─────────────────────────────────────────────────
|
|
|
|
/**
|
|
* Run ripgrep (`rg`) with JSON output to collect text matches.
|
|
*
|
|
* Why `spawn` and not `execFile`: `execFile` buffers stdout internally and
|
|
* kills the child when `maxBuffer` is exceeded, even when 'data' listeners
|
|
* are attached. Under rg's verbose `--json` output, a 50MB buffer fills
|
|
* well before the match cap in large folders, and `execFile`'s silent
|
|
* buffer-exceeded error resolves the result as `truncated: false` despite
|
|
* dropping matches. See docs/design/share-text-search.md.
|
|
*/
|
|
export function searchWithRg(
|
|
rootPath: string,
|
|
query: string,
|
|
opts: SearchOptions
|
|
): Promise<SearchResult> {
|
|
return new Promise((resolve) => {
|
|
const rgArgs = buildRgArgs(query, rootPath, opts)
|
|
const acc = createAccumulator()
|
|
let buffer = ''
|
|
let resolved = false
|
|
|
|
// Why: spawn can throw synchronously on invalid options (e.g. bad cwd),
|
|
// which would leak out of the `new Promise` executor and leave the
|
|
// promise forever pending. Treat a synchronous throw as a clean
|
|
// "no results" fallback, the same way an async 'error' event is handled.
|
|
let child: ReturnType<typeof spawn>
|
|
try {
|
|
child = spawn('rg', rgArgs, {
|
|
cwd: rootPath,
|
|
stdio: ['ignore', 'pipe', 'pipe']
|
|
})
|
|
} catch {
|
|
resolve(finalize(acc))
|
|
return
|
|
}
|
|
|
|
const resolveOnce = (): void => {
|
|
if (resolved) {
|
|
return
|
|
}
|
|
resolved = true
|
|
clearTimeout(killTimeout)
|
|
resolve(finalize(acc))
|
|
}
|
|
|
|
const processLine = (line: string): void => {
|
|
const verdict = ingestRgJsonLine(line, rootPath, acc, opts.maxResults)
|
|
if (verdict === 'stop') {
|
|
child.kill()
|
|
}
|
|
}
|
|
|
|
child.stdout!.setEncoding('utf-8')
|
|
child.stdout!.on('data', (chunk: string) => {
|
|
buffer += chunk
|
|
const lines = buffer.split('\n')
|
|
buffer = lines.pop() ?? ''
|
|
for (const line of lines) {
|
|
processLine(line)
|
|
}
|
|
})
|
|
child.stderr!.on('data', () => {
|
|
/* drain */
|
|
})
|
|
child.once('error', () => resolveOnce())
|
|
child.once('close', () => {
|
|
if (buffer) {
|
|
processLine(buffer)
|
|
}
|
|
resolveOnce()
|
|
})
|
|
|
|
const killTimeout = setTimeout(() => {
|
|
acc.truncated = true
|
|
child.kill()
|
|
}, SEARCH_TIMEOUT_MS)
|
|
})
|
|
}
|
|
|
|
// ─── rg availability check ──────────────────────────────────────────
|
|
|
|
// Why no cache: `rg --version` is a sub-10ms local spawn, and caching the
|
|
// result caused a footgun — a negative cache persisted across rg installs
|
|
// (forcing a relay restart), while a positive cache could mask an rg that
|
|
// was uninstalled or broken mid-session. The `settled` flag below closes
|
|
// the original race between 'error' and 'close' that the cache was added
|
|
// to paper over, so re-checking per call is both simpler and safer.
|
|
export function checkRgAvailable(): Promise<boolean> {
|
|
return new Promise((resolve) => {
|
|
let settled = false
|
|
const child = execFile('rg', ['--version'])
|
|
child.once('error', () => {
|
|
if (settled) {
|
|
return
|
|
}
|
|
settled = true
|
|
resolve(false)
|
|
})
|
|
child.once('close', (code) => {
|
|
if (settled) {
|
|
return
|
|
}
|
|
settled = true
|
|
resolve(code === 0)
|
|
})
|
|
})
|
|
}
|
|
|
|
// Moved to fs-handler-list-files.ts to keep this file under 300 lines (oxlint)
|
|
export { listFilesWithRg, LIST_FILES_TIMEOUT_MS } from './fs-handler-list-files'
|