starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-11-01 09:37:33 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
3d812eb9bf54c39e80098bb6ce55f8498a3d3c8c
orca/config/patches/xterm-src/@xterm__addon-image@0.10.0-beta.301.src.patch
T
Neil 90849ed0c6 Enable Claude plugin images and bounded terminal file transport (#28200) 
* fix: render Kitty Unicode placeholder images with bounded ownership

* Enable managed Claude plugin images and bounded host file transport

* Preserve isolated Claude evidence and paired restart diagnostics

* Resolve Windows workspace aliases before WSL image reads

* Handle split remote terminal identities and improve WSL file-read diagnostics

* Reuse native drive paths for WSL image files and preserve guest temporary roots

* Apply native Windows path restrictions to mapped drive image sources

* Preserve WSL participation evidence across image checks

* Keep WSL drive directory containment case-sensitive after canonicalization

* Negotiate managed Claude image support with the launching renderer

* Make image repaint checks independent of display profile and shell output

* Capture repaint diagnostics only after the pixel assertion fails

* Allow floating terminal image reads from host temporary folders

* Keep image stack independent of unrelated CI runner and mobile lint settings

* Reuse worktree argument builder for terminal viewer context

Remove duplicated mapping and keep the RPC method table within its existing line limit.
2026-10-15 21:43:48 -07:00

1521 lines
64 KiB
Diff
Raw Blame History

diff --git a/src/IIPHandler.ts b/src/IIPHandler.ts
index 559b907416eb38318f439d060d7f89311ed34c7e..73cedbe99f831bffd202697cc8ba80a46a39cb99 100644
--- a/src/IIPHandler.ts
+++ b/src/IIPHandler.ts
@@ -13,8 +13,10 @@ import { imageType, UNSUPPORTED_TYPE } from './IIPMetrics';
// Local const enum mirror - esbuild can't inline const enums from external packages
const enum DecoderConst {
- // Limit held memory in base64 decoder (encoded bytes).
- KEEP_DATA = 4194304,
+ // Held memory in base64/QOI decoders between images. Zero because each kept
+ // decoder pins a wasm memory, and V8 caps those per process (~124 in a
+ // sandboxed renderer), so idle terminals must not hold one.
+ KEEP_DATA = 0,
// Initial buffer allocation for the decoder.
INITIAL_DATA = 1048576,
// Local mirror of const enum (esbuild can't inline const enums from external packages)
@@ -34,6 +36,7 @@ const DEFAULT_HEADER: IHeaderFields = {
export class IIPHandler implements IOscHandler, IResetHandler {
+ private _generation = 0;
private _aborted = false;
private _hp = new HeaderParser();
private _header: IHeaderFields = DEFAULT_HEADER;
@@ -55,6 +58,7 @@ export class IIPHandler implements IOscHandler, IResetHandler {
}
public reset(): void {
+ this._generation++;
this._hp.reset();
this._dec.release();
this._qoiDec.release();
@@ -92,7 +96,10 @@ export class IIPHandler implements IOscHandler, IResetHandler {
this._aborted = true;
return;
}
- this._dec.init();
+ if (!this._initDecoder()) {
+ this._aborted = true;
+ return;
+ }
} else if (this._abortMulti) {
this._aborted = true;
return;
@@ -135,7 +142,9 @@ export class IIPHandler implements IOscHandler, IResetHandler {
this._isMultipart = true;
this._abortMulti = false;
this._dec.release();
- this._dec.init();
+ if (!this._initDecoder()) {
+ this._abortMulti = true;
+ }
return true;
}
@@ -179,7 +188,15 @@ export class IIPHandler implements IOscHandler, IResetHandler {
let blob: Blob | ImageData;
if (metrics.mime === 'image/qoi') {
- const data = this._qoiDec.decode(this._dec.data8);
+ let data: Uint8Array<ArrayBuffer>;
+ try {
+ data = this._qoiDec.decode(this._dec.data8);
+ } catch (e) {
+ console.warn('IIP: could not decode QOI image', e);
+ this._dec.release();
+ this._qoiDec.release();
+ return true;
+ }
blob = new ImageData(
new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength),
this._qoiDec.width,
@@ -198,8 +215,13 @@ export class IIPHandler implements IOscHandler, IResetHandler {
blob = new Blob([this._dec.data8], { type: metrics.mime });
}
this._dec.release();
+ const generation = this._generation;
return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h })
.then(bm => {
+ if (generation !== this._generation) {
+ bm.close();
+ return true;
+ }
this._storage.addImage(bm);
return true;
})
@@ -209,6 +231,18 @@ export class IIPHandler implements IOscHandler, IResetHandler {
});
}
+ // Why: wasm memory exhaustion must drop this image, not throw out of the parser and wedge the write queue.
+ private _initDecoder(): boolean {
+ try {
+ this._dec.init();
+ return true;
+ } catch (e) {
+ console.warn('IIP: could not allocate decoder', e);
+ this._dec.release();
+ return false;
+ }
+ }
+
private _resize(w: number, h: number): [number, number] {
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
diff --git a/src/ImageAddon.ts b/src/ImageAddon.ts
index 8fd39543118cd420e36c1614c1af370b6c7bbfbb..fcabb492ab1e21853f7c473f8c4d085e1ab12414 100644
--- a/src/ImageAddon.ts
+++ b/src/ImageAddon.ts
@@ -6,6 +6,7 @@
import type { ITerminalAddon, IDisposable } from '@xterm/xterm';
import type { ImageAddon as IImageApi } from '@xterm/addon-image';
import { Emitter, type IEvent } from 'common/Event';
+import { MicrotaskTimer } from 'common/Async';
import { IIPHandler } from './IIPHandler';
import { ImageRenderer } from './ImageRenderer';
import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage';
@@ -113,6 +114,7 @@ export class ImageAddon implements ITerminalAddon, IImageApi {
}
public dispose(): void {
+ for (const handler of this._handlers.values()) handler.reset();
for (const obj of this._disposables) {
obj.dispose();
}
@@ -196,14 +198,21 @@ export class ImageAddon implements ITerminalAddon, IImageApi {
// Kitty graphics handler
if (this._opts.kittySupport) {
- const kittyStorage = new KittyImageStorage(this._storage!);
+ const kittyStorage = new KittyImageStorage(this._storage!, this._renderer!);
const kittyHandler = new KittyGraphicsHandler(this._opts, this._renderer!, kittyStorage, terminal);
this._handlers.set('kitty', kittyHandler);
+ const scrollMask = new MicrotaskTimer();
+ const hide = (): void => kittyStorage.hidePlaceholderGlyphs(terminal);
this._disposeLater(
kittyStorage,
kittyHandler,
+ scrollMask,
+ terminal.onWriteParsed(() => { scrollMask.cancel(); hide(); }),
+ terminal.onScroll(() => scrollMask.set(hide)),
+ terminal.onResize(hide),
terminal._core._inputHandler._parser.registerApcHandler({ final: 'G' }, kittyHandler)
);
+ hide();
}
}
diff --git a/src/ImageRenderer.ts b/src/ImageRenderer.ts
index 5854efaec1fdf9dfcb886023542998a563b6d2f2..85448368ec3d65e4f718ac4991a34d6a3fa0ca7d 100644
--- a/src/ImageRenderer.ts
+++ b/src/ImageRenderer.ts
@@ -140,6 +140,10 @@ export class ImageRenderer extends Disposable implements IDisposable {
};
}
+ public refreshAll(): void {
+ this._renderService?.refreshRows(0, this._terminal.rows - 1);
+ }
+
/**
* Clear a region of the image layer canvas.
*/
@@ -186,16 +190,17 @@ export class ImageRenderer extends Disposable implements IDisposable {
this._rescaleImage(imgSpec, width, height);
const img = imgSpec.actual!;
- const cols = Math.ceil(img.width / width);
+ const { width: sourceWidth, height: sourceHeight } = imgSpec.actualCellSize;
+ const cols = Math.ceil(img.width / sourceWidth);
- const sx = (tileId % cols) * width;
- const sy = Math.floor(tileId / cols) * height;
+ const sx = (tileId % cols) * sourceWidth;
+ const sy = Math.floor(tileId / cols) * sourceHeight;
const dx = col * width;
const dy = row * height;
// safari bug: never access image source out of bounds
- const finalWidth = count * width + sx > img.width ? img.width - sx : count * width;
- const finalHeight = sy + height > img.height ? img.height - sy : height;
+ const finalWidth = count * sourceWidth + sx > img.width ? img.width - sx : count * sourceWidth;
+ const finalHeight = sy + sourceHeight > img.height ? img.height - sy : sourceHeight;
// Floor all pixel offsets to get stable tile mapping without any overflows.
// Note: For not pixel perfect aligned cells like in the DOM renderer
@@ -204,10 +209,72 @@ export class ImageRenderer extends Disposable implements IDisposable {
ctx.drawImage(
img,
Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight),
- Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth), Math.ceil(finalHeight)
+ Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth * width / sourceWidth), Math.ceil(finalHeight * height / sourceHeight)
);
}
+ // Fit the original source into the cell grid without distorting its aspect ratio.
+ public drawCellImage(imgSpec: IImageSpec, boxCols: number, boxRows: number, gridRow: number, gridCol: number, count: number, col: number, row: number): void {
+ const ctx = this._layers.get(imgSpec.layer);
+ if (!ctx) {
+ return;
+ }
+ const { width, height } = this.cellSize;
+ // Don't try to draw anything, if we cannot get valid renderer metrics.
+ if (width === -1 || height === -1) {
+ return;
+ }
+ const img = imgSpec.orig;
+ if (!img) {
+ return;
+ }
+
+ const boxWidth = boxCols * width;
+ const boxHeight = boxRows * height;
+ const fitWidth = img.width * boxHeight > img.height * boxWidth;
+ const scale = fitWidth
+ ? boxWidth / Math.max(1, img.width)
+ : boxHeight / Math.max(1, img.height);
+ const xOffset = fitWidth ? 0 : (boxWidth - img.width * scale) / 2;
+ const yOffset = fitWidth ? (boxHeight - img.height * scale) / 2 : 0;
+
+ // Clip source and destination together so edge tiles are not stretched.
+ let dx = col * width;
+ let dy = row * height;
+ let dw = count * width;
+ let dh = height;
+ let sx = (gridCol * width - xOffset) / scale;
+ let sy = (gridRow * height - yOffset) / scale;
+ let sw = dw / scale;
+ let sh = dh / scale;
+ if (sx < 0) {
+ const clipped = -sx * scale;
+ dx += clipped;
+ dw -= clipped;
+ sw += sx;
+ sx = 0;
+ }
+ if (sy < 0) {
+ const clipped = -sy * scale;
+ dy += clipped;
+ dh -= clipped;
+ sh += sy;
+ sy = 0;
+ }
+ if (sx + sw > img.width) {
+ sw = img.width - sx;
+ dw = sw * scale;
+ }
+ if (sy + sh > img.height) {
+ sh = img.height - sy;
+ dh = sh * scale;
+ }
+ if (sw <= 0 || sh <= 0 || dw <= 0 || dh <= 0) {
+ return;
+ }
+ ctx.drawImage(img, sx, sy, sw, sh, dx, dy, dw, dh);
+ }
+
/**
* Extract a single tile from an image.
*/
@@ -219,19 +286,20 @@ export class ImageRenderer extends Disposable implements IDisposable {
}
this._rescaleImage(imgSpec, width, height);
const img = imgSpec.actual!;
- const cols = Math.ceil(img.width / width);
- const sx = (tileId % cols) * width;
- const sy = Math.floor(tileId / cols) * height;
- const finalWidth = width + sx > img.width ? img.width - sx : width;
- const finalHeight = sy + height > img.height ? img.height - sy : height;
-
- const canvas = ImageRenderer.createCanvas(this.document, finalWidth, finalHeight);
+ const { width: sourceWidth, height: sourceHeight } = imgSpec.actualCellSize;
+ const cols = Math.ceil(img.width / sourceWidth);
+ const sx = (tileId % cols) * sourceWidth;
+ const sy = Math.floor(tileId / cols) * sourceHeight;
+ const finalWidth = sourceWidth + sx > img.width ? img.width - sx : sourceWidth;
+ const finalHeight = sy + sourceHeight > img.height ? img.height - sy : sourceHeight;
+
+ const canvas = ImageRenderer.createCanvas(this.document, Math.ceil(finalWidth * width / sourceWidth), Math.ceil(finalHeight * height / sourceHeight));
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.drawImage(
img,
Math.floor(sx), Math.floor(sy), Math.floor(finalWidth), Math.floor(finalHeight),
- 0, 0, Math.floor(finalWidth), Math.floor(finalHeight)
+ 0, 0, canvas.width, canvas.height
);
return canvas;
}
@@ -299,11 +367,16 @@ export class ImageRenderer extends Disposable implements IDisposable {
spec.actualCellSize.height = originalHeight;
return;
}
- const canvas = ImageRenderer.createCanvas(
- this.document,
- Math.ceil(spec.orig!.width * currentWidth / originalWidth),
- Math.ceil(spec.orig!.height * currentHeight / originalHeight)
- );
+ const scaledWidth = Math.ceil(spec.orig!.width * currentWidth / originalWidth);
+ const scaledHeight = Math.ceil(spec.orig!.height * currentHeight / originalHeight);
+ // Upscale visible tiles directly; a full zoomed copy can dwarf the image budget.
+ if (scaledWidth * scaledHeight > spec.orig!.width * spec.orig!.height) {
+ spec.actual = spec.orig;
+ spec.actualCellSize.width = originalWidth;
+ spec.actualCellSize.height = originalHeight;
+ return;
+ }
+ const canvas = ImageRenderer.createCanvas(this.document, scaledWidth, scaledHeight);
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.drawImage(spec.orig!, 0, 0, canvas.width, canvas.height);
@@ -415,7 +488,11 @@ export class ImageRenderer extends Disposable implements IDisposable {
for (let i = 0; i < width; i += bWidth) {
ctx2.drawImage(blueprint, i, 0);
}
- ImageRenderer.createImageBitmap(this._placeholder).then(bitmap => this._placeholderBitmap = bitmap);
+ const placeholder = this._placeholder;
+ ImageRenderer.createImageBitmap(placeholder).then(bitmap => {
+ if (this._placeholder !== placeholder) bitmap?.close();
+ else this._placeholderBitmap = bitmap;
+ }).catch(() => {});
}
public get document(): Document | undefined {
diff --git a/src/ImageStorage.ts b/src/ImageStorage.ts
index 5dfca3f69fb5fcb5104ded9853b29111f926fe76..189288f0cab7a50ffc2b255c0c8f4e18d3065147 100644
--- a/src/ImageStorage.ts
+++ b/src/ImageStorage.ts
@@ -49,6 +49,8 @@ export class ImageStorage implements IDisposable {
private _viewportMetrics: { cols: number, rows: number };
public onImageAdded: (() => void) | undefined;
public onImageDeleted: ((storageId: number) => void) | undefined;
+ // Placeholder cells carry image references in text and colors, not tile attributes.
+ public onRenderLine: ((line: IBufferLine, row: number, cols: number) => void) | undefined;
constructor(
private _terminal: ITerminalExt,
@@ -74,12 +76,9 @@ export class ImageStorage implements IDisposable {
}
public reset(): void {
- for (const spec of this._images.values()) {
- spec.marker?.dispose();
+ for (const id of [...this._images.keys()]) {
+ this.deleteImage(id);
}
- // NOTE: marker.dispose above already calls ImageBitmap.close
- // therefore we can just wipe the map here
- this._images.clear();
this._renderer.clearAll();
}
@@ -121,6 +120,7 @@ export class ImageStorage implements IDisposable {
spec.orig.close();
}
this.onImageDeleted?.(id);
+ this._renderer.refreshAll();
}
/**
@@ -155,6 +155,36 @@ export class ImageStorage implements IDisposable {
}
}
+ public addUnplacedImage(img: HTMLCanvasElement | ImageBitmap, opts: IAddImageOpts): number {
+ // never allow storage to exceed memory limit
+ this._evictOldest(img.width * img.height);
+
+ let cellSize = this._renderer.cellSize;
+ if (cellSize.width === -1 || cellSize.height === -1) {
+ cellSize = CELL_SIZE_DEFAULT;
+ }
+
+ const imageId = ++this._lastId;
+ this._images.set(imageId, {
+ orig: img,
+ origCellSize: cellSize,
+ actual: img,
+ actualCellSize: { ...cellSize },
+ marker: undefined,
+ tileCount: 0,
+ virtual: true,
+ bufferType: this._terminal.buffer.active.type,
+ layer: opts.layer,
+ zIndex: opts.zIndex
+ });
+ this.onImageAdded?.();
+ return imageId;
+ }
+
+ public getImageSpec(id: number): IImageSpec | undefined {
+ return this._images.get(id);
+ }
+
/**
* Method to add an image to the storage.
* @param img - The image to add (canvas or bitmap).
@@ -226,7 +256,7 @@ export class ImageStorage implements IDisposable {
// deleted images with zero tile count
const zero = [];
for (const [id, spec] of this._images.entries()) {
- if (spec.tileCount < 1) {
+ if (spec.tileCount < 1 && !spec.virtual) {
spec.marker?.dispose();
zero.push(id);
}
@@ -349,6 +379,7 @@ export class ImageStorage implements IDisposable {
for (let row = start; row <= end; ++row) {
const line = buffer.lines.get(row + buffer.ydisp);
if (!line) return;
+ this.onRenderLine?.(line, row, cols);
for (let col = 0; col < cols; ++col) {
const e = line.getExtended(col)?.payload;
if (e instanceof ImageTileInfo) {
@@ -571,7 +602,7 @@ export class ImageStorage implements IDisposable {
// deleted images with zero tile count
const zero = [];
for (const [id, spec] of this._images.entries()) {
- if (spec.bufferType === 'alternate' && !spec.tileCount) {
+ if (spec.bufferType === 'alternate' && !spec.tileCount && !spec.virtual) {
spec.marker?.dispose();
zero.push(id);
}
diff --git a/src/SixelHandler.ts b/src/SixelHandler.ts
index 1af2d85bcdd541ed60b1e707f6186a91f1bbf1b2..0711a122ea43d5212a2851add9e744b65550ec85 100644
--- a/src/SixelHandler.ts
+++ b/src/SixelHandler.ts
@@ -10,6 +10,7 @@ import { RGBA8888 } from 'sixel/lib/Types';
import { ImageRenderer } from './ImageRenderer';
import { DecoderAsync, Decoder } from 'sixel/lib/Decoder';
+import { LIMITS } from 'sixel/lib/wasm';
// always free decoder ressources after decoding if it exceeds this limit
const MEM_PERMA_LIMIT = 4194304; // 1024 pixels * 1024 pixels * 4 channels = 4MB
@@ -18,48 +19,88 @@ const MEM_PERMA_LIMIT = 4194304; // 1024 pixels * 1024 pixels * 4 channels = 4MB
const DEFAULT_PALETTE = PALETTE_ANSI_256;
DEFAULT_PALETTE.set(PALETTE_VT340_COLOR);
+// Why pooled: every decoder owns a wasm memory, and V8 caps live wasm memories
+// per process (~124 in a sandboxed renderer). Terminals borrow a decoder only
+// while a SIXEL sequence is open, so idle terminals hold none.
+const MAX_IDLE_DECODERS = 2;
+const idleDecoders = new Map<number, Decoder[]>();
+let poolPrimed = false;
+
+function primeDecoderPool(memoryLimit: number): void {
+ if (poolPrimed) return;
+ poolPrimed = true;
+ // Async compile once, off the parser's hot path; later decoders reuse the cached module.
+ DecoderAsync({ memoryLimit, palette: DEFAULT_PALETTE }).then(
+ d => releaseDecoder(d, memoryLimit),
+ () => { poolPrimed = false; }
+ );
+}
+
+function acquireDecoder(memoryLimit: number): Decoder {
+ return idleDecoders.get(memoryLimit)?.pop() ?? new Decoder({ memoryLimit, palette: DEFAULT_PALETTE });
+}
+
+function releaseDecoder(dec: Decoder, memoryLimit: number): void {
+ if (dec.memoryUsage > MEM_PERMA_LIMIT) {
+ dec.release();
+ }
+ const idle = idleDecoders.get(memoryLimit) ?? [];
+ if (idle.length < MAX_IDLE_DECODERS) {
+ idle.push(dec);
+ idleDecoders.set(memoryLimit, idle);
+ }
+}
+
export class SixelHandler implements IDcsHandler, IResetHandler {
private _size = 0;
private _aborted = false;
private _dec: Decoder | undefined;
+ private _decMemoryLimit = 0;
+ // Color registers outlive a single image, so they live here rather than in a pooled decoder.
+ private readonly _palette = new Uint32Array(LIMITS.PALETTE_SIZE);
constructor(
private readonly _opts: IImageAddonOptions,
private readonly _storage: SixelImageStorage,
private readonly _coreTerminal: ITerminalExt
) {
- DecoderAsync({
- memoryLimit: this._opts.pixelLimit * 4,
- palette: DEFAULT_PALETTE,
- paletteLimit: this._opts.sixelPaletteLimit
- }).then(d => this._dec = d);
+ this._palette.set(DEFAULT_PALETTE);
+ primeDecoderPool(this._opts.pixelLimit * 4);
}
public reset(): void {
- /**
- * reset sixel decoder to defaults:
- * - release all memory
- * - nullify palette (4096)
- * - apply default palette (256)
- */
- if (this._dec) {
- this._dec.release();
- // FIXME: missing interface on decoder to nullify full palette
- (this._dec as any)._palette.fill(0);
- this._dec.init(0, DEFAULT_PALETTE, this._opts.sixelPaletteLimit);
- }
+ this._returnDecoder();
+ this._palette.fill(0);
+ this._palette.set(DEFAULT_PALETTE);
}
public hook(params: IParams): void {
this._size = 0;
this._aborted = false;
- if (this._dec) {
- const fillColor = params.params[1] === 1 ? 0 : extractActiveBg(
- this._coreTerminal._core._inputHandler._curAttrData,
- this._coreTerminal._core._themeService?.colors);
- this._dec.init(fillColor, null, this._opts.sixelPaletteLimit);
+ this._returnDecoder();
+ const memoryLimit = this._opts.pixelLimit * 4;
+ try {
+ this._dec = acquireDecoder(memoryLimit);
+ } catch (e) {
+ // Why: exhausting wasm memory must drop this image, not throw out of the parser and wedge the write queue.
+ console.warn(`SIXEL: could not allocate decoder - ${e}`);
+ this._aborted = true;
+ return;
}
+ this._decMemoryLimit = memoryLimit;
+ const fillColor = params.params[1] === 1 ? 0 : extractActiveBg(
+ this._coreTerminal._core._inputHandler._curAttrData,
+ this._coreTerminal._core._themeService?.colors);
+ this._dec.init(fillColor, this._palette, this._opts.sixelPaletteLimit);
+ }
+
+ private _returnDecoder(): void {
+ const dec = this._dec;
+ if (!dec) return;
+ this._dec = undefined;
+ this._palette.set(dec.palette);
+ releaseDecoder(dec, this._decMemoryLimit);
}
public put(data: Uint32Array, start: number, end: number): void {
@@ -83,6 +124,14 @@ export class SixelHandler implements IDcsHandler, IResetHandler {
}
public unhook(success: boolean): boolean | Promise<boolean> {
+ try {
+ return this._unhook(success);
+ } finally {
+ this._returnDecoder();
+ }
+ }
+
+ private _unhook(success: boolean): boolean {
if (this._aborted || !success || !this._dec) {
return true;
}
@@ -100,9 +149,6 @@ export class SixelHandler implements IDcsHandler, IResetHandler {
const canvas = ImageRenderer.createCanvas(undefined, width, height);
canvas.getContext('2d')?.putImageData(new ImageData(this._dec.data8 as Uint8ClampedArray<ArrayBuffer>, width, height), 0, 0);
- if (this._dec.memoryUsage > MEM_PERMA_LIMIT) {
- this._dec.release();
- }
this._storage.addImage(canvas);
return true;
}
diff --git a/src/Types.ts b/src/Types.ts
index 04328420c2bae02010bb4c9adb3fd341aed56c0c..8dc120fac57656d2b378e1cc89f4edcf17dd51a2 100644
--- a/src/Types.ts
+++ b/src/Types.ts
@@ -40,6 +40,7 @@ export interface IImageAddonOptions {
iipSizeLimit: number;
kittySupport: boolean;
kittySizeLimit: number;
+ readKittyFile?: (request: { path: string; offset: number; length?: number; signal: AbortSignal }) => Promise<Uint8Array>;
}
export interface IResetHandler {
@@ -103,6 +104,7 @@ export interface IImageSpec {
actualCellSize: ICellSize;
marker: IMarker | undefined;
tileCount: number;
+ virtual?: boolean;
bufferType: 'alternate' | 'normal';
layer: ImageLayer;
zIndex: number;
diff --git a/src/kitty/KittyGraphicsHandler.ts b/src/kitty/KittyGraphicsHandler.ts
index de889dfff75d9ecc8ab47a025e6989ffe75bb202..b69362705ac18c26997d4a07563e3bf512fdf00d 100644
--- a/src/kitty/KittyGraphicsHandler.ts
+++ b/src/kitty/KittyGraphicsHandler.ts
@@ -7,6 +7,7 @@ import { IDisposable } from '@xterm/xterm';
import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt, ImageLayer } from '../Types';
import { ImageRenderer } from '../ImageRenderer';
import { CELL_SIZE_DEFAULT } from '../ImageStorage';
+import { imageType } from '../IIPMetrics';
import { KittyImageStorage } from './KittyImageStorage';
import Base64Decoder, { type DecodeStatus } from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm';
import {
@@ -37,6 +38,8 @@ const DECODER_OK = Constants.DECODER_OK as unknown as DecodeStatus.OK;
// Kitty graphics protocol handler with streaming base64 decoding.
export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDisposable {
private _aborted = false;
+ private _generation = 0;
+ private _fileRead: AbortController | undefined;
private _decodeError = false;
private _activeDecoder: Base64Decoder | null = null;
@@ -80,6 +83,8 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
public reset(): void {
+ this._generation++;
+ this._fileRead?.abort();
this._cleanupAllPending();
if (this._activeDecoder) {
this._activeDecoder.release();
@@ -155,6 +160,12 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
return;
}
+ if (this._parsedCommand.transmission === 'f' && this._parsedCommand.more === 1) {
+ this._sendResponse(this._parsedCommand.id ?? 0, 'EINVAL:chunked file path', this._parsedCommand.quiet ?? 0);
+ this._aborted = true;
+ return;
+ }
+
// Delete action doesn't need payload - skip streaming
if (this._parsedCommand.action === KittyAction.DELETE) {
return;
@@ -200,8 +211,38 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
this._activeDecoder = pending.decoder;
}
if (!this._activeDecoder) {
- this._activeDecoder = new Base64Decoder(Constants.DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes);
- this._activeDecoder.init();
+ // Budget WASM capacity, including one page of decoder state and rounding.
+ const decoderCapacity = this._maxEncodedBytes + 131072;
+ if (decoderCapacity > this._opts.storageLimit * 1000000) {
+ this._aborted = true;
+ if (this._parsedCommand?.id !== undefined) {
+ this._sendResponse(this._parsedCommand.id, 'ENOMEM:pending image budget exceeded', this._parsedCommand.quiet ?? 0);
+ }
+ return;
+ }
+ const maxPending = Math.max(1, Math.floor(this._opts.storageLimit * 1000000 / decoderCapacity));
+ while (this._pendingTransmissions.size >= maxPending) {
+ const oldest = this._pendingTransmissions.entries().next().value;
+ if (!oldest) break;
+ oldest[1].decoder.release();
+ this._removePendingEntry(oldest[0]);
+ if (oldest[1].cmd.id !== undefined) {
+ this._sendResponse(oldest[1].cmd.id, 'ENOMEM:pending image budget exceeded', oldest[1].cmd.quiet ?? 0);
+ }
+ }
+ const decoder = new Base64Decoder(Constants.DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes);
+ try {
+ decoder.init();
+ } catch (e) {
+ // Why: wasm memory exhaustion must drop this image, not throw out of the parser and wedge the write queue.
+ console.warn('KITTY: could not allocate decoder', e);
+ this._aborted = true;
+ if (this._parsedCommand?.id !== undefined) {
+ this._sendResponse(this._parsedCommand.id, 'ENOMEM:could not allocate decoder', this._parsedCommand.quiet ?? 0);
+ }
+ return;
+ }
+ this._activeDecoder = decoder;
}
if (this._activeDecoder.put(data.subarray(start, end)) !== DECODER_OK) {
@@ -295,6 +336,42 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
return result;
}
+ private async _handleFile(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): Promise<boolean> {
+ const reader = this._opts.readKittyFile;
+ const action = cmd.action ?? 't';
+ if (!reader) {
+ this._sendResponse(cmd.id ?? 0, 'ENOTSUP:file transmission unavailable', cmd.quiet ?? 0);
+ return true;
+ }
+ const offset = cmd.fileOffset ?? 0;
+ const length = cmd.fileSize;
+ if (decodeError || bytes.length === 0 || bytes.length > 4096 || cmd.more === 1
+ || !['t', 'T', 'q'].includes(action) || !Number.isSafeInteger(offset) || offset < 0
+ || (length !== undefined && (!Number.isSafeInteger(length) || length <= 0 || length > this._opts.kittySizeLimit))) {
+ this._sendResponse(cmd.id ?? 0, 'EINVAL:invalid file transmission', cmd.quiet ?? 0);
+ return true;
+ }
+ const generation = this._generation;
+ const controller = new AbortController();
+ this._fileRead = controller;
+ try {
+ const path = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+ if (path.includes('\0')) throw new Error('invalid file path');
+ const data = await reader({ path, offset, length, signal: controller.signal });
+ if (generation !== this._generation || controller.signal.aborted) return true;
+ if (data.byteLength === 0 || data.byteLength > Math.min(this._opts.kittySizeLimit, this._opts.storageLimit * 1000000)
+ || (length !== undefined && data.byteLength !== length)) throw new Error('invalid file extent');
+ return await this._handleCommandWithBytesAndCmd({ ...cmd, transmission: 'd' }, data, false);
+ } catch {
+ if (generation === this._generation && !controller.signal.aborted) {
+ this._sendResponse(cmd.id ?? 0, 'EINVAL:file transmission failed', cmd.quiet ?? 0);
+ }
+ return true;
+ } finally {
+ if (this._fileRead === controller) this._fileRead = undefined;
+ }
+ }
+
// Command handling
private _parseControlDataString(): string {
@@ -320,8 +397,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
case KittyAction.DELETE:
return this._handleDelete(cmd);
case KittyAction.QUERY:
- this._sendResponse(cmd.id ?? 0, 'OK', cmd.quiet ?? 0);
- return true;
+ return this._handleQuery(cmd, new Uint8Array(0), false);
case KittyAction.PLACEMENT:
return this._handlePlacement(cmd);
default:
@@ -337,6 +413,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
private _handleCommandWithBytesAndCmd(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise<boolean> {
+ if (cmd.transmission === 'f') return this._handleFile(cmd, bytes, decodeError);
const action = cmd.action ?? 't';
switch (action) {
@@ -382,24 +459,17 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
this._sendResponse(id, 'ENOENT:image not found', cmd.quiet ?? 0, cmd.placementId);
return true;
}
- const result = this._displayImage(image, cmd);
+ const result = cmd.unicodePlaceholder === 1
+ ? this._placeVirtual(image, cmd)
+ : this._displayImage(image, cmd);
return result.then(success => {
this._sendResponse(id, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0, cmd.placementId);
return true;
});
}
- private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean {
- // TODO: Support file-based transmission modes (t=f, t=t, t=s)
- // Currently only supports direct transmission (t=d, the default).
- // - t=f (file): Payload is base64-encoded file path. Terminal reads image from that path.
- // - t=t (temp file): Payload is base64-encoded path in temp directory. Terminal reads, deletes.
- // - t=s: Payload is base64-encoded POSIX shm name. Terminal reads from shared memory.
- // These modes require filesystem/IPC access not available in browsers. For Node.js/Electron:
- // 1. Check cmd.transmission (t key) before treating bytes as image data
- // 2. For t=f/t/s: decode bytes as UTF-8 string (the path/name), then read file contents
- // 3. For t=d: treat bytes as image data (current behavior)
- // When implementing, also update _handleQuery to accept these transmission mediums.
+ private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean, retainVirtual = false): boolean {
+ // File reads resolve into direct bytes before reaching storage.
const transmission = cmd.transmission ?? 'd';
if (transmission !== 'd') {
if (cmd.id !== undefined) {
@@ -416,7 +486,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
height: cmd.height ?? 0,
format: (cmd.format ?? KittyFormat.RGBA) as 24 | 32 | 100,
compression: cmd.compression ?? ''
- });
+ }, retainVirtual);
return true;
}
@@ -428,12 +498,15 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
return true;
}
- this._handleTransmit(cmd, bytes, decodeError);
+ const virtual = cmd.unicodePlaceholder === 1;
+ this._handleTransmit(cmd, bytes, decodeError, virtual);
const id = cmd.id ?? this._kittyStorage.lastImageId;
const image = this._kittyStorage.getImage(id);
if (image) {
- const result = this._displayImage(image, cmd);
+ const result = virtual
+ ? this._placeVirtual(image, cmd, true)
+ : this._displayImage(image, cmd);
if (cmd.id !== undefined) {
return result.then(success => {
this._sendResponse(id, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0);
@@ -449,9 +522,6 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
const id = cmd.id ?? 0;
const quiet = cmd.quiet ?? 0;
- // Per spec: reject unsupported transmission mediums (only t=d is supported atm)
- // TODO: When filesystem support is added (Node.js/Electron), update this to accept
- // t=f (file), t=t (temp file), and t=s (shared memory) and respond OK for queries.
const transmission = cmd.transmission ?? 'd';
if (transmission !== 'd') {
this._sendResponse(id, 'EINVAL:unsupported transmission medium', quiet);
@@ -500,28 +570,25 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
// Per spec: default delete selector is 'a' (delete all visible placements)
const selector = cmd.deleteSelector ?? 'a';
- // TODO: Distinguish lowercase (delete placements only) from uppercase
- // (delete placements + free stored image data). Currently both variants
- // free everything since we don't separate stored data from placements.
switch (selector) {
case 'a':
+ this._kittyStorage.deletePhysicalPlacements();
+ break;
case 'A':
this._cleanupAllPending();
- this._kittyStorage.deleteAll();
+ this._kittyStorage.deleteAll(true);
break;
case 'i':
case 'I':
- // TODO: When placement id tracking is implemented (see TODO in
- // KittyImageStorage), d=i with p=<pid> should delete only that
- // specific placement, while d=i without p should delete all
- // placements for the image.
if (cmd.id !== undefined) {
- const pending = this._pendingTransmissions.get(cmd.id);
- if (pending) {
- pending.decoder.release();
+ if (selector === 'I') {
+ const pending = this._pendingTransmissions.get(cmd.id);
+ pending?.decoder.release();
+ this._removePendingEntry(cmd.id);
+ this._kittyStorage.deleteById(cmd.id, cmd.placementId);
+ } else {
+ this._kittyStorage.deletePlacementsById(cmd.id, cmd.placementId);
}
- this._removePendingEntry(cmd.id);
- this._kittyStorage.deleteById(cmd.id);
}
break;
default:
@@ -550,49 +617,17 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise<void> {
+ const generation = this._generation;
let bitmap: ImageBitmap | undefined = await this._createBitmap(image);
try {
- const cropX = Math.max(0, cmd.x ?? 0);
- const cropY = Math.max(0, cmd.y ?? 0);
- const cropW = cmd.sourceWidth || (bitmap.width - cropX);
- const cropH = cmd.sourceHeight || (bitmap.height - cropY);
-
- const maxCropW = Math.max(0, bitmap.width - cropX);
- const maxCropH = Math.max(0, bitmap.height - cropY);
- const finalCropW = Math.max(0, Math.min(cropW, maxCropW));
- const finalCropH = Math.max(0, Math.min(cropH, maxCropH));
-
- if (finalCropW === 0 || finalCropH === 0) {
- throw new Error('invalid source rectangle');
- }
-
- if (cropX !== 0 || cropY !== 0 || finalCropW !== bitmap.width || finalCropH !== bitmap.height) {
- const cropped = await createImageBitmap(bitmap, cropX, cropY, finalCropW, finalCropH);
- bitmap.close();
- bitmap = cropped;
- }
+ if (generation !== this._generation) throw new Error('image decode canceled');
+ const cropped = this._cropImage(bitmap, cmd);
+ bitmap = cropped instanceof Promise ? await cropped : cropped;
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
-
- // Per spec: c/r default to image's natural cell dimensions.
- // If only one of c/r is specified, compute the other from image aspect ratio.
- let imgCols: number;
- let imgRows: number;
- if (cmd.columns !== undefined && cmd.rows !== undefined) {
- imgCols = cmd.columns;
- imgRows = cmd.rows;
- } else if (cmd.columns !== undefined) {
- imgCols = cmd.columns;
- imgRows = Math.max(1, Math.ceil((bitmap.height / bitmap.width) * (imgCols * cw) / ch));
- } else if (cmd.rows !== undefined) {
- imgRows = cmd.rows;
- imgCols = Math.max(1, Math.ceil((bitmap.width / bitmap.height) * (imgRows * ch) / cw));
- } else {
- imgCols = Math.ceil(bitmap.width / cw);
- imgRows = Math.ceil(bitmap.height / ch);
- }
+ let { cols: imgCols, rows: imgRows } = this._placementGrid(bitmap, cmd, cw, ch);
let w = bitmap.width;
let h = bitmap.height;
@@ -660,6 +695,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
}
+ if (generation !== this._generation) throw new Error('image decode canceled');
const zIndex = cmd.zIndex ?? 0;
this._kittyStorage.addImage(image.id, bitmap, true, layer, zIndex);
bitmap = undefined; // ownership transferred to storage
@@ -684,6 +720,75 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
}
+ // Keep the uncropped path synchronous to preserve its parser scheduling.
+ private _cropImage(bitmap: ImageBitmap, cmd: IKittyCommand): ImageBitmap | Promise<ImageBitmap> {
+ const cropX = Math.max(0, cmd.x ?? 0);
+ const cropY = Math.max(0, cmd.y ?? 0);
+ const cropW = cmd.sourceWidth || (bitmap.width - cropX);
+ const cropH = cmd.sourceHeight || (bitmap.height - cropY);
+
+ const maxCropW = Math.max(0, bitmap.width - cropX);
+ const maxCropH = Math.max(0, bitmap.height - cropY);
+ const finalCropW = Math.max(0, Math.min(cropW, maxCropW));
+ const finalCropH = Math.max(0, Math.min(cropH, maxCropH));
+
+ if (finalCropW === 0 || finalCropH === 0) {
+ throw new Error('invalid source rectangle');
+ }
+
+ if (cropX !== 0 || cropY !== 0 || finalCropW !== bitmap.width || finalCropH !== bitmap.height) {
+ return createImageBitmap(bitmap, cropX, cropY, finalCropW, finalCropH).then(cropped => {
+ bitmap.close();
+ return cropped;
+ });
+ }
+ return bitmap;
+ }
+
+ // An omitted grid dimension follows the source aspect ratio.
+ private _placementGrid(bitmap: ImageBitmap, cmd: IKittyCommand, cw: number, ch: number): { cols: number; rows: number } {
+ if (cmd.columns !== undefined && cmd.rows !== undefined) {
+ return { cols: cmd.columns, rows: cmd.rows };
+ }
+ if (cmd.columns !== undefined) {
+ return { cols: cmd.columns, rows: Math.max(1, Math.ceil((bitmap.height / bitmap.width) * (cmd.columns * cw) / ch)) };
+ }
+ if (cmd.rows !== undefined) {
+ return { cols: Math.max(1, Math.ceil((bitmap.width / bitmap.height) * (cmd.rows * ch) / cw)), rows: cmd.rows };
+ }
+ return { cols: Math.ceil(bitmap.width / cw), rows: Math.ceil(bitmap.height / ch) };
+ }
+
+ private async _placeVirtual(image: IKittyImageData, cmd: IKittyCommand, replaces = false): Promise<boolean> {
+ const generation = this._generation;
+ const isCurrent = (): boolean => generation === this._generation && this._kittyStorage.getImage(image.id) === image;
+ let bitmap: ImageBitmap | undefined;
+ try {
+ bitmap = await this._createBitmap(image);
+ if (!isCurrent()) throw new Error('image decode canceled');
+ const cropped = this._cropImage(bitmap, cmd);
+ bitmap = cropped instanceof Promise ? await cropped : cropped;
+
+ const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
+ const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
+ const { cols, rows } = this._placementGrid(bitmap, cmd, cw, ch);
+
+ if (bitmap.width * bitmap.height > this._opts.pixelLimit) {
+ throw new Error('image exceeds pixel limit');
+ }
+
+ if (!isCurrent()) throw new Error('image decode canceled');
+ this._kittyStorage.addVirtualPlacement(image.id, cmd.placementId ?? 0, bitmap, cols, rows);
+ bitmap = undefined; // ownership transferred to storage
+ if (replaces) this._kittyStorage.discardVirtualPlacements(image.id, cmd.placementId ?? 0);
+ return true;
+ } catch {
+ bitmap?.close();
+ if (replaces && isCurrent()) this._kittyStorage.discardVirtualPlacements(image.id);
+ return false;
+ }
+ }
+
// Create ImageBitmap from already-decoded image data.
private async _createBitmap(image: IKittyImageData): Promise<ImageBitmap> {
let bytes: Uint8Array = new Uint8Array(await image.data.arrayBuffer());
@@ -693,6 +798,12 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
}
if (image.format === KittyFormat.PNG) {
+ const metrics = imageType(bytes);
+ // IHDR dimensions are parsed with signed shifts, so a value >= 0x80000000 comes
+ // back negative and a bare `>` pixel-limit test passes it; require positive.
+ if (metrics.mime !== 'image/png' || !(metrics.width > 0) || !(metrics.height > 0) || metrics.width * metrics.height > this._opts.pixelLimit) {
+ throw new RangeError('PNG exceeds pixel limit or has invalid dimensions');
+ }
const blob = new Blob([bytes as BlobPart], { type: 'image/png' });
if (!window.createImageBitmap) {
const url = URL.createObjectURL(blob);
@@ -775,27 +886,45 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
try {
return await this._decompress(compressed, 'deflate');
- } catch {
+ } catch (error) {
+ if (error instanceof RangeError) throw error;
return await this._decompress(compressed, 'deflate-raw');
}
}
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
- const ds = new DecompressionStream(format);
- const writer = ds.writable.getWriter();
- writer.write(compressed as BufferSource);
- writer.close();
-
+ const limit = Math.min(this._opts.kittySizeLimit, this._opts.pixelLimit * 4, this._opts.storageLimit * 1000000);
+ let offsetIn = 0;
+ // Bound inflation within one native transform before its output is budgeted.
+ const source = new ReadableStream<BufferSource>({
+ pull(controller) {
+ if (offsetIn >= compressed.length) {
+ controller.close();
+ return;
+ }
+ const end = Math.min(offsetIn + 4096, compressed.length);
+ controller.enqueue(new Uint8Array(compressed.subarray(offsetIn, end)));
+ offsetIn = end;
+ }
+ });
+ const reader = source.pipeThrough(new DecompressionStream(format)).getReader();
const chunks: Uint8Array[] = [];
- const reader = ds.readable.getReader();
-
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
- chunks.push(value);
+ let totalLength = 0;
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ totalLength += value.byteLength;
+ if (totalLength > limit) {
+ await reader.cancel().catch(() => {});
+ throw new RangeError('decompressed image exceeds byte limit');
+ }
+ chunks.push(value);
+ }
+ } finally {
+ reader.releaseLock();
}
- const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const result = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
diff --git a/src/kitty/KittyGraphicsTypes.ts b/src/kitty/KittyGraphicsTypes.ts
index 4288dd0a4180eca8f3ae28ec0805f834b2a0a629..d4230cc2b0bb7d247b0429bb4c5f918ef0b8efed 100644
--- a/src/kitty/KittyGraphicsTypes.ts
+++ b/src/kitty/KittyGraphicsTypes.ts
@@ -78,7 +78,11 @@ export const enum KittyKey {
// Delete selector (a/A=all, i/I=by id, c/C=at cursor, etc.) — only used when a=d
DELETE_SELECTOR = 'd',
// Placement ID for targeting specific placements
- PLACEMENT_ID = 'p'
+ PLACEMENT_ID = 'p',
+ // U=1 creates a prototype for placeholder cells without placing cursor tiles.
+ UNICODE_PLACEHOLDER = 'U',
+ FILE_OFFSET = 'O',
+ FILE_SIZE = 'S'
}
// Pixel format constants
@@ -109,9 +113,12 @@ export interface IKittyCommand {
cursorMovement?: number;
zIndex?: number;
transmission?: string;
+ fileOffset?: number;
+ fileSize?: number;
deleteSelector?: string;
placementId?: number;
compression?: string;
+ unicodePlaceholder?: number;
payload?: string;
}
@@ -139,6 +146,13 @@ export interface IKittyImageData {
compression?: string;
}
+export interface IVirtualPlacement {
+ placementId: number;
+ storageId: number;
+ cols: number;
+ rows: number;
+}
+
// Parses Kitty graphics control data into a command object.
export function parseKittyCommand(data: string): IKittyCommand {
const cmd: IKittyCommand = {};
@@ -170,6 +184,8 @@ export function parseKittyCommand(data: string): IKittyCommand {
}
const numValue = parseInt(value, 10);
switch (key) {
+ case KittyKey.FILE_OFFSET: cmd.fileOffset = Number(value); break;
+ case KittyKey.FILE_SIZE: cmd.fileSize = Number(value); break;
case KittyKey.FORMAT: cmd.format = numValue; break;
case KittyKey.ID: cmd.id = numValue; break;
case KittyKey.IMAGE_NUMBER: cmd.imageNumber = numValue; break;
@@ -188,8 +204,73 @@ export function parseKittyCommand(data: string): IKittyCommand {
case KittyKey.CURSOR_MOVEMENT: cmd.cursorMovement = numValue; break;
case KittyKey.Z_INDEX: cmd.zIndex = numValue; break;
case KittyKey.PLACEMENT_ID: cmd.placementId = numValue; break;
+ case KittyKey.UNICODE_PLACEHOLDER: cmd.unicodePlaceholder = numValue; break;
}
}
return cmd;
}
+
+
+// Source patches can only name files included in the published package.
+
+export const PLACEHOLDER_CHAR = 0x10EEEE;
+
+// Protocol table order; values are one-based so zero can mean an omitted diacritic.
+export const PLACEHOLDER_DIACRITICS: readonly number[] = [
+ 0x305, 0x30D, 0x30E, 0x310, 0x312, 0x33D, 0x33E, 0x33F, 0x346, 0x34A,
+ 0x34B, 0x34C, 0x350, 0x351, 0x352, 0x357, 0x35B, 0x363, 0x364, 0x365,
+ 0x366, 0x367, 0x368, 0x369, 0x36A, 0x36B, 0x36C, 0x36D, 0x36E, 0x36F,
+ 0x483, 0x484, 0x485, 0x486, 0x487, 0x592, 0x593, 0x594, 0x595, 0x597,
+ 0x598, 0x599, 0x59C, 0x59D, 0x59E, 0x59F, 0x5A0, 0x5A1, 0x5A8, 0x5A9,
+ 0x5AB, 0x5AC, 0x5AF, 0x5C4, 0x610, 0x611, 0x612, 0x613, 0x614, 0x615,
+ 0x616, 0x617, 0x657, 0x658, 0x659, 0x65A, 0x65B, 0x65D, 0x65E, 0x6D6,
+ 0x6D7, 0x6D8, 0x6D9, 0x6DA, 0x6DB, 0x6DC, 0x6DF, 0x6E0, 0x6E1, 0x6E2,
+ 0x6E4, 0x6E7, 0x6E8, 0x6EB, 0x6EC, 0x730, 0x732, 0x733, 0x735, 0x736,
+ 0x73A, 0x73D, 0x73F, 0x740, 0x741, 0x743, 0x745, 0x747, 0x749, 0x74A,
+ 0x7EB, 0x7EC, 0x7ED, 0x7EE, 0x7EF, 0x7F0, 0x7F1, 0x7F3, 0x816, 0x817,
+ 0x818, 0x819, 0x81B, 0x81C, 0x81D, 0x81E, 0x81F, 0x820, 0x821, 0x822,
+ 0x823, 0x825, 0x826, 0x827, 0x829, 0x82A, 0x82B, 0x82C, 0x82D, 0x951,
+ 0x953, 0x954, 0xF82, 0xF83, 0xF86, 0xF87, 0x135D, 0x135E, 0x135F, 0x17DD,
+ 0x193A, 0x1A17, 0x1A75, 0x1A76, 0x1A77, 0x1A78, 0x1A79, 0x1A7A, 0x1A7B, 0x1A7C,
+ 0x1B6B, 0x1B6D, 0x1B6E, 0x1B6F, 0x1B70, 0x1B71, 0x1B72, 0x1B73, 0x1CD0, 0x1CD1,
+ 0x1CD2, 0x1CDA, 0x1CDB, 0x1CE0, 0x1DC0, 0x1DC1, 0x1DC3, 0x1DC4, 0x1DC5, 0x1DC6,
+ 0x1DC7, 0x1DC8, 0x1DC9, 0x1DCB, 0x1DCC, 0x1DD1, 0x1DD2, 0x1DD3, 0x1DD4, 0x1DD5,
+ 0x1DD6, 0x1DD7, 0x1DD8, 0x1DD9, 0x1DDA, 0x1DDB, 0x1DDC, 0x1DDD, 0x1DDE, 0x1DDF,
+ 0x1DE0, 0x1DE1, 0x1DE2, 0x1DE3, 0x1DE4, 0x1DE5, 0x1DE6, 0x1DFE, 0x20D0, 0x20D1,
+ 0x20D4, 0x20D5, 0x20D6, 0x20D7, 0x20DB, 0x20DC, 0x20E1, 0x20E7, 0x20E9, 0x20F0,
+ 0x2CEF, 0x2CF0, 0x2CF1, 0x2DE0, 0x2DE1, 0x2DE2, 0x2DE3, 0x2DE4, 0x2DE5, 0x2DE6,
+ 0x2DE7, 0x2DE8, 0x2DE9, 0x2DEA, 0x2DEB, 0x2DEC, 0x2DED, 0x2DEE, 0x2DEF, 0x2DF0,
+ 0x2DF1, 0x2DF2, 0x2DF3, 0x2DF4, 0x2DF5, 0x2DF6, 0x2DF7, 0x2DF8, 0x2DF9, 0x2DFA,
+ 0x2DFB, 0x2DFC, 0x2DFD, 0x2DFE, 0x2DFF, 0xA66F, 0xA67C, 0xA67D, 0xA6F0, 0xA6F1,
+ 0xA8E0, 0xA8E1, 0xA8E2, 0xA8E3, 0xA8E4, 0xA8E5, 0xA8E6, 0xA8E7, 0xA8E8, 0xA8E9,
+ 0xA8EA, 0xA8EB, 0xA8EC, 0xA8ED, 0xA8EE, 0xA8EF, 0xA8F0, 0xA8F1, 0xAAB0, 0xAAB2,
+ 0xAAB3, 0xAAB7, 0xAAB8, 0xAABE, 0xAABF, 0xAAC1, 0xFE20, 0xFE21, 0xFE22, 0xFE23,
+ 0xFE24, 0xFE25, 0xFE26, 0x10A0F, 0x10A38, 0x1D185, 0x1D186, 0x1D187, 0x1D188, 0x1D189,
+ 0x1D1AA, 0x1D1AB, 0x1D1AC, 0x1D1AD, 0x1D242, 0x1D243, 0x1D244,
+];
+
+const PLACEHOLDER_DIACRITIC_VALUES: Map<number, number> = new Map();
+for (let i = 0; i < PLACEHOLDER_DIACRITICS.length; ++i) {
+ PLACEHOLDER_DIACRITIC_VALUES.set(PLACEHOLDER_DIACRITICS[i], i + 1);
+}
+
+// Zero values may inherit from the preceding placeholder cell.
+export interface IPlaceholderLocation {
+ row: number;
+ column: number;
+ highByte: number;
+}
+
+function placeholderDiacriticValue(mark: string | undefined): number {
+ return mark === undefined ? 0 : PLACEHOLDER_DIACRITIC_VALUES.get(mark.codePointAt(0) ?? -1) ?? -1;
+}
+
+export function decodePlaceholderLocation(chars: string): IPlaceholderLocation {
+ const [, row, column, highByte] = chars;
+ return {
+ row: placeholderDiacriticValue(row),
+ column: placeholderDiacriticValue(column),
+ highByte: placeholderDiacriticValue(highByte)
+ };
+}
diff --git a/src/kitty/KittyImageStorage.ts b/src/kitty/KittyImageStorage.ts
index 1f5c09ec9e2700f8f6dbd8436a1802217dfc99ef..2a0df51c965c4e2c4c6f1935c129052cadf12153 100644
--- a/src/kitty/KittyImageStorage.ts
+++ b/src/kitty/KittyImageStorage.ts
@@ -3,16 +3,27 @@
* @license MIT
*/
+import type { IBufferLine } from 'common/buffer/Types';
import { IDisposable } from '@xterm/xterm';
+import { CellData } from 'common/buffer/CellData';
+import { FgFlags } from 'common/buffer/Constants';
+import { ImageRenderer } from '../ImageRenderer';
import { ImageStorage } from '../ImageStorage';
-import { ImageLayer, IAddImageOpts } from '../Types';
-import { IKittyImageData } from './KittyGraphicsTypes';
+import { Attributes, ITerminalExt, ImageLayer, IAddImageOpts } from '../Types';
+import { decodePlaceholderLocation, IKittyImageData, IPlaceholderLocation, IVirtualPlacement, PLACEHOLDER_CHAR } from './KittyGraphicsTypes';
+
+interface IPlaceholderCell {
+ imageId: number;
+ placementId: number;
+ location: IPlaceholderLocation;
+}
// Kitty-specific image storage controller.
//
// Wraps shared ImageStorage with kitty protocol semantics:
// - tracks transmitted image payloads by kitty image id
// - tracks kitty image id -> shared ImageStorage id mapping for displayed images
+// - tracks virtual placements that U+10EEEE cells display
// - mirrors shared-storage evictions into kitty maps
// - applies protocol-level undisplayed-image eviction policy
export class KittyImageStorage implements IDisposable {
@@ -20,32 +31,33 @@ export class KittyImageStorage implements IDisposable {
private _nextImageId = 1;
private readonly _images: Map<number, IKittyImageData> = new Map();
- // TODO: Support multiple placements per image. The kitty spec identifies
- // placements by an (image id, placement id) pair — same i + different p
- // values should coexist, and same i + same p should replace the prior
- // placement. Currently we track only one storage entry per kitty image id,
- // so multiple placements of the same image overwrite each other. Fixing
- // this requires changing these maps to Map<number, Map<number, number>>
- // (kittyId → placementId → storageId) and updating addImage/deleteById
- // accordingly. The underlying shared ImageStorage would also need to
- // support multiple entries per logical image.
+ // TODO: Track physical placements by (image id, placement id), like virtual placements.
private readonly _kittyIdToStorageId: Map<number, number> = new Map();
private readonly _storageIdToKittyId: Map<number, number> = new Map();
+ private readonly _virtualPlacements: Map<number, Map<number, IVirtualPlacement>> = new Map();
+ private readonly _virtualStorageIds = new Map<number, { kittyId: number, placementId: number }>();
private readonly _previousOnImageDeleted: ((storageId: number) => void) | undefined;
private readonly _wrappedOnImageDeleted: (storageId: number) => void;
private readonly _handleStorageImageDeleted = (storageId: number): void => {
+ const virtual = this._virtualStorageIds.get(storageId);
+ if (virtual) {
+ this._virtualStorageIds.delete(storageId);
+ const placements = this._virtualPlacements.get(virtual.kittyId);
+ placements?.delete(virtual.placementId);
+ if (!placements?.size) this._virtualPlacements.delete(virtual.kittyId);
+ }
const kittyId = this._storageIdToKittyId.get(storageId);
if (kittyId !== undefined) {
this._kittyIdToStorageId.delete(kittyId);
this._storageIdToKittyId.delete(storageId);
- this._images.delete(kittyId);
}
};
private _addImageOpts: IAddImageOpts = { scrolling: true, layer: 'top', zIndex: 0, cursorPos: 'iip' };
constructor(
- private readonly _storage: ImageStorage
+ private readonly _storage: ImageStorage,
+ private readonly _renderer: ImageRenderer
) {
this._previousOnImageDeleted = this._storage.onImageDeleted;
this._wrappedOnImageDeleted = (storageId: number) => {
@@ -53,6 +65,27 @@ export class KittyImageStorage implements IDisposable {
this._handleStorageImageDeleted(storageId);
};
this._storage.onImageDeleted = this._wrappedOnImageDeleted;
+ this._storage.onRenderLine = this._renderPlaceholderLine;
+ }
+
+ // Reserved placeholder text stays in the buffer but never draws as a glyph.
+ public hidePlaceholderGlyphs(terminal: ITerminalExt): void {
+ const buffer = terminal._core.buffer;
+ let cell: CellData | undefined;
+ const start = terminal.buffer.active.viewportY;
+ for (let row = start; row < start + terminal.rows; ++row) {
+ const line = buffer.lines.get(row);
+ if (!line) continue;
+ for (let col = 0; col < terminal.cols; ++col) {
+ if (line.getFg(col) & FgFlags.INVISIBLE) continue;
+ if (line.getCodePoint(col) !== PLACEHOLDER_CHAR
+ && (!line.isCombined(col) || line.getString(col).codePointAt(0) !== PLACEHOLDER_CHAR)) continue;
+ cell ??= new CellData();
+ line.loadCell(col, cell);
+ cell.fg |= FgFlags.INVISIBLE;
+ line.setCell(col, cell);
+ }
+ }
}
public reset(): void {
@@ -60,6 +93,8 @@ export class KittyImageStorage implements IDisposable {
this._images.clear();
this._kittyIdToStorageId.clear();
this._storageIdToKittyId.clear();
+ this._virtualPlacements.clear();
+ this._virtualStorageIds.clear();
}
public dispose(): void {
@@ -67,11 +102,15 @@ export class KittyImageStorage implements IDisposable {
if (this._storage.onImageDeleted === this._wrappedOnImageDeleted) {
this._storage.onImageDeleted = this._previousOnImageDeleted;
}
+ if (this._storage.onRenderLine === this._renderPlaceholderLine) {
+ this._storage.onRenderLine = undefined;
+ }
}
- public storeImage(id: number | undefined, imageData: Omit<IKittyImageData, 'id'>): number {
+ public storeImage(id: number | undefined, imageData: Omit<IKittyImageData, 'id'>, retainVirtual = false): number {
const imageId = id ?? this._nextImageId++;
+ if (!retainVirtual) this._deleteVirtualPlacements(imageId);
const oldStorageId = this._kittyIdToStorageId.get(imageId);
if (oldStorageId !== undefined) {
this._storage.deleteImage(oldStorageId);
@@ -83,6 +122,25 @@ export class KittyImageStorage implements IDisposable {
this._evictUndisplayedImages();
}
+ // Encoded images awaiting placement are outside ImageStorage's pixel budget.
+ // Unplaced payloads are evicted first so a new upload cannot erase a visible
+ // image while abandoned blobs still hold budget; placed ones go only when
+ // that is not enough, because the byte cap is a hard bound. The new image is
+ // always stored, so an oversized one overshoots by at most one payload
+ // (itself bounded by kittySizeLimit) rather than being dropped after an OK ack.
+ const byteLimit = this._storage.getLimit() * 1000000;
+ this._images.delete(imageId);
+ let retainedBytes = 0;
+ for (const image of this._images.values()) retainedBytes += image.data.size;
+ for (const evictPlaced of [false, true]) {
+ for (const [oldestId, image] of this._images) {
+ if (retainedBytes + imageData.data.size <= byteLimit) break;
+ if ((this._kittyIdToStorageId.has(oldestId) || this._virtualPlacements.has(oldestId)) !== evictPlaced) continue;
+ retainedBytes -= image.data.size;
+ this.deleteById(oldestId);
+ }
+ }
+
this._images.set(imageId, {
...imageData,
id: imageId
@@ -111,23 +169,73 @@ export class KittyImageStorage implements IDisposable {
return this._images.get(kittyId);
}
- public deleteById(kittyId: number): void {
- this._images.delete(kittyId);
+ // Fit prototypes at render time so each placement holds only the source pixels.
+ public addVirtualPlacement(kittyId: number, placementId: number, image: HTMLCanvasElement | ImageBitmap, cols: number, rows: number): void {
+ const previous = this._virtualPlacements.get(kittyId)?.get(placementId);
+ if (image.width * image.height * 4 > this._storage.getLimit() * 1000000) throw new Error('virtual image exceeds storage limit');
+ if (!previous && this._virtualStorageIds.size >= KittyImageStorage._maxStoredImages) throw new Error('virtual placement limit exceeded');
+ if (previous) this._storage.deleteImage(previous.storageId);
+ const storageId = this._storage.addUnplacedImage(image, { scrolling: false, layer: 'top', zIndex: 0, cursorPos: 'iip' });
+ let placements = this._virtualPlacements.get(kittyId);
+ if (!placements) {
+ placements = new Map();
+ this._virtualPlacements.set(kittyId, placements);
+ }
+ placements.set(placementId, { placementId, storageId, cols, rows });
+ this._virtualStorageIds.set(storageId, { kittyId, placementId });
+ this._renderer.refreshAll();
+ }
+
+ private _virtualPlacementFor(kittyId: number, placementId: number): IVirtualPlacement | undefined {
+ const placements = this._virtualPlacements.get(kittyId);
+ return placementId === 0 ? placements?.values().next().value : placements?.get(placementId);
+ }
+
+ public discardVirtualPlacements(kittyId: number, keepPlacementId?: number): void {
+ this._deleteVirtualPlacements(kittyId, undefined, keepPlacementId);
+ }
+
+ private _deleteVirtualPlacements(kittyId: number, placementId?: number, keepPlacementId?: number): void {
+ const placements = this._virtualPlacements.get(kittyId);
+ if (!placements) return;
+ for (const placement of [...placements.values()]) {
+ if ((placementId === undefined || placementId === placement.placementId) && placement.placementId !== keepPlacementId) {
+ this._storage.deleteImage(placement.storageId);
+ }
+ }
+ }
+
+ public deletePlacementsById(kittyId: number, placementId?: number): void {
+ this._deleteVirtualPlacements(kittyId, placementId);
+ if (placementId !== undefined) return;
const storageId = this._kittyIdToStorageId.get(kittyId);
if (storageId !== undefined) {
- this._storage.deleteImage(storageId);
this._kittyIdToStorageId.delete(kittyId);
this._storageIdToKittyId.delete(storageId);
+ this._storage.deleteImage(storageId);
}
}
- public deleteAll(): void {
- this._images.clear();
- for (const storageId of this._kittyIdToStorageId.values()) {
+ public deleteById(kittyId: number, placementId?: number): void {
+ this.deletePlacementsById(kittyId, placementId);
+ if (!this._virtualPlacements.has(kittyId) && !this._kittyIdToStorageId.has(kittyId)) {
+ this._images.delete(kittyId);
+ }
+ }
+
+ public deletePhysicalPlacements(): void {
+ for (const [kittyId, storageId] of [...this._kittyIdToStorageId]) {
+ this._kittyIdToStorageId.delete(kittyId);
+ this._storageIdToKittyId.delete(storageId);
this._storage.deleteImage(storageId);
}
- this._kittyIdToStorageId.clear();
- this._storageIdToKittyId.clear();
+ }
+
+ public deleteAll(preserveVirtual = false): void {
+ if (preserveVirtual) this.deletePhysicalPlacements();
+ for (const kittyId of [...this._images.keys()]) {
+ if (!preserveVirtual || !this._virtualPlacements.has(kittyId)) this.deleteById(kittyId);
+ }
}
public get images(): ReadonlyMap<number, IKittyImageData> {
@@ -147,9 +255,96 @@ export class KittyImageStorage implements IDisposable {
if (this._images.size <= KittyImageStorage._maxStoredImages / 2) {
break;
}
- if (!this._kittyIdToStorageId.has(kittyId)) {
+ if (!this._kittyIdToStorageId.has(kittyId) && !this._virtualPlacements.has(kittyId)) {
this._images.delete(kittyId);
+ this._virtualPlacements.delete(kittyId);
+ }
+ }
+ }
+
+ // Omitted diacritics inherit only from a consecutive matching placeholder.
+ private readonly _renderPlaceholderLine = (line: IBufferLine, row: number, cols: number): void => {
+ if (this._virtualPlacements.size === 0) {
+ return;
+ }
+ let runLength = 0;
+ let previous: IPlaceholderCell | undefined;
+ for (let col = 0; col <= cols; ++col) {
+ const cell = col < cols ? this._placeholderCell(line, col) : undefined;
+ if (runLength > 0 && cell !== undefined && previous !== undefined
+ && cell.imageId === previous.imageId
+ && cell.placementId === previous.placementId
+ && (cell.location.row === 0 || cell.location.row === previous.location.row)
+ && (cell.location.column === 0 || cell.location.column === previous.location.column + 1)
+ && (cell.location.highByte === 0 || cell.location.highByte === previous.location.highByte)) {
+ runLength++;
+ cell.location.row = Math.max(previous.location.row, 1);
+ cell.location.column = previous.location.column + 1;
+ cell.location.highByte = Math.max(previous.location.highByte, 1);
+ } else {
+ if (runLength > 0 && previous !== undefined) {
+ this._drawPlaceholderRun(previous, col - runLength, runLength, row);
+ }
+ runLength = 0;
+ if (cell !== undefined) {
+ runLength = 1;
+ if (!cell.location.row) cell.location.row = 1;
+ if (!cell.location.column) cell.location.column = 1;
+ if (!cell.location.highByte) cell.location.highByte = 1;
+ }
}
+ previous = cell ?? previous;
+ }
+ };
+
+ private _placeholderCell(line: IBufferLine, col: number): IPlaceholderCell | undefined {
+ if (!line.isCombined(col) && line.getCodePoint(col) !== PLACEHOLDER_CHAR) {
+ return undefined;
+ }
+ const chars = line.getString(col);
+ if (chars.codePointAt(0) !== PLACEHOLDER_CHAR) {
+ return undefined;
+ }
+
+ const fg = line.getFg(col);
+ const fgMode = fg & Attributes.CM_MASK;
+ const imageId = fgMode === Attributes.CM_RGB
+ ? fg & Attributes.RGB_MASK
+ : fgMode === 0 ? 0 : fg & Attributes.PCOLOR_MASK;
+ if (fgMode === 0) {
+ return undefined;
+ }
+
+ // Placement IDs use underline color without an underline style.
+ let placementId = 0;
+ const underlineColor = line.getExtended(col)?.underlineColor ?? 0;
+ if (underlineColor > 0) {
+ const underlineMode = underlineColor & Attributes.CM_MASK;
+ placementId = underlineMode === Attributes.CM_RGB
+ ? underlineColor & Attributes.RGB_MASK
+ : underlineMode === 0 ? 0 : underlineColor & Attributes.PCOLOR_MASK;
+ }
+
+ const location = decodePlaceholderLocation(chars);
+ if (location.row < 0 || location.column < 0 || location.highByte < 0 || location.highByte > 256) return undefined;
+ return { imageId, placementId, location };
+ }
+
+ private _drawPlaceholderRun(cell: IPlaceholderCell, startCol: number, count: number, row: number): void {
+ const imageId = (cell.imageId | ((cell.location.highByte - 1) << 24)) >>> 0;
+ const placement = this._virtualPlacementFor(imageId, cell.placementId);
+ const spec = placement && this._storage.getImageSpec(placement.storageId);
+ // `column` is 1-based and holds the last cell of the run.
+ const gridRow = cell.location.row - 1;
+ const gridCol = cell.location.column - count;
+ if (!placement || !spec || gridRow < 0 || gridRow >= placement.rows || gridCol < 0) {
+ return;
+ }
+ // Cells past the placement's box show nothing.
+ const tiles = Math.min(count, placement.cols - gridCol);
+ if (tiles <= 0) {
+ return;
}
+ this._renderer.drawCellImage(spec, placement.cols, placement.rows, gridRow, gridCol, tiles, startCol, row);
}
}
Reference in New Issue View Git Blame Copy Permalink