diff --git a/src/IIPHandler.ts b/src/IIPHandler.ts index 559b907416eb38318f439d060d7f89311ed34c7e..73cedbe99f831bffd202697cc8ba80a46a39cb99 100644 --- a/src/IIPHandler.ts +++ b/src/IIPHandler.ts @@ -13,8 +13,10 @@ import { imageType, UNSUPPORTED_TYPE } from './IIPMetrics'; // Local const enum mirror - esbuild can't inline const enums from external packages const enum DecoderConst { - // Limit held memory in base64 decoder (encoded bytes). - KEEP_DATA = 4194304, + // Held memory in base64/QOI decoders between images. Zero because each kept + // decoder pins a wasm memory, and V8 caps those per process (~124 in a + // sandboxed renderer), so idle terminals must not hold one. + KEEP_DATA = 0, // Initial buffer allocation for the decoder. INITIAL_DATA = 1048576, // Local mirror of const enum (esbuild can't inline const enums from external packages) @@ -34,6 +36,7 @@ const DEFAULT_HEADER: IHeaderFields = { export class IIPHandler implements IOscHandler, IResetHandler { + private _generation = 0; private _aborted = false; private _hp = new HeaderParser(); private _header: IHeaderFields = DEFAULT_HEADER; @@ -55,6 +58,7 @@ export class IIPHandler implements IOscHandler, IResetHandler { } public reset(): void { + this._generation++; this._hp.reset(); this._dec.release(); this._qoiDec.release(); @@ -92,7 +96,10 @@ export class IIPHandler implements IOscHandler, IResetHandler { this._aborted = true; return; } - this._dec.init(); + if (!this._initDecoder()) { + this._aborted = true; + return; + } } else if (this._abortMulti) { this._aborted = true; return; @@ -135,7 +142,9 @@ export class IIPHandler implements IOscHandler, IResetHandler { this._isMultipart = true; this._abortMulti = false; this._dec.release(); - this._dec.init(); + if (!this._initDecoder()) { + this._abortMulti = true; + } return true; } @@ -179,7 +188,15 @@ export class IIPHandler implements IOscHandler, IResetHandler { let blob: Blob | ImageData; if (metrics.mime === 'image/qoi') { - const data = this._qoiDec.decode(this._dec.data8); + let data: Uint8Array; + try { + data = this._qoiDec.decode(this._dec.data8); + } catch (e) { + console.warn('IIP: could not decode QOI image', e); + this._dec.release(); + this._qoiDec.release(); + return true; + } blob = new ImageData( new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength), this._qoiDec.width, @@ -198,8 +215,13 @@ export class IIPHandler implements IOscHandler, IResetHandler { blob = new Blob([this._dec.data8], { type: metrics.mime }); } this._dec.release(); + const generation = this._generation; return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h }) .then(bm => { + if (generation !== this._generation) { + bm.close(); + return true; + } this._storage.addImage(bm); return true; }) @@ -209,6 +231,18 @@ export class IIPHandler implements IOscHandler, IResetHandler { }); } + // Why: wasm memory exhaustion must drop this image, not throw out of the parser and wedge the write queue. + private _initDecoder(): boolean { + try { + this._dec.init(); + return true; + } catch (e) { + console.warn('IIP: could not allocate decoder', e); + this._dec.release(); + return false; + } + } + private _resize(w: number, h: number): [number, number] { const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width; const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height; diff --git a/src/ImageAddon.ts b/src/ImageAddon.ts index 8fd39543118cd420e36c1614c1af370b6c7bbfbb..fcabb492ab1e21853f7c473f8c4d085e1ab12414 100644 --- a/src/ImageAddon.ts +++ b/src/ImageAddon.ts @@ -6,6 +6,7 @@ import type { ITerminalAddon, IDisposable } from '@xterm/xterm'; import type { ImageAddon as IImageApi } from '@xterm/addon-image'; import { Emitter, type IEvent } from 'common/Event'; +import { MicrotaskTimer } from 'common/Async'; import { IIPHandler } from './IIPHandler'; import { ImageRenderer } from './ImageRenderer'; import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage'; @@ -113,6 +114,7 @@ export class ImageAddon implements ITerminalAddon, IImageApi { } public dispose(): void { + for (const handler of this._handlers.values()) handler.reset(); for (const obj of this._disposables) { obj.dispose(); } @@ -196,14 +198,21 @@ export class ImageAddon implements ITerminalAddon, IImageApi { // Kitty graphics handler if (this._opts.kittySupport) { - const kittyStorage = new KittyImageStorage(this._storage!); + const kittyStorage = new KittyImageStorage(this._storage!, this._renderer!); const kittyHandler = new KittyGraphicsHandler(this._opts, this._renderer!, kittyStorage, terminal); this._handlers.set('kitty', kittyHandler); + const scrollMask = new MicrotaskTimer(); + const hide = (): void => kittyStorage.hidePlaceholderGlyphs(terminal); this._disposeLater( kittyStorage, kittyHandler, + scrollMask, + terminal.onWriteParsed(() => { scrollMask.cancel(); hide(); }), + terminal.onScroll(() => scrollMask.set(hide)), + terminal.onResize(hide), terminal._core._inputHandler._parser.registerApcHandler({ final: 'G' }, kittyHandler) ); + hide(); } } diff --git a/src/ImageRenderer.ts b/src/ImageRenderer.ts index 5854efaec1fdf9dfcb886023542998a563b6d2f2..85448368ec3d65e4f718ac4991a34d6a3fa0ca7d 100644 --- a/src/ImageRenderer.ts +++ b/src/ImageRenderer.ts @@ -140,6 +140,10 @@ export class ImageRenderer extends Disposable implements IDisposable { }; } + public refreshAll(): void { + this._renderService?.refreshRows(0, this._terminal.rows - 1); + } + /** * Clear a region of the image layer canvas. */ @@ -186,16 +190,17 @@ export class ImageRenderer extends Disposable implements IDisposable { this._rescaleImage(imgSpec, width, height); const img = imgSpec.actual!; - const cols = Math.ceil(img.width / width); + const { width: sourceWidth, height: sourceHeight } = imgSpec.actualCellSize; + const cols = Math.ceil(img.width / sourceWidth); - const sx = (tileId % cols) * width; - const sy = Math.floor(tileId / cols) * height; + const sx = (tileId % cols) * sourceWidth; + const sy = Math.floor(tileId / cols) * sourceHeight; const dx = col * width; const dy = row * height; // safari bug: never access image source out of bounds - const finalWidth = count * width + sx > img.width ? img.width - sx : count * width; - const finalHeight = sy + height > img.height ? img.height - sy : height; + const finalWidth = count * sourceWidth + sx > img.width ? img.width - sx : count * sourceWidth; + const finalHeight = sy + sourceHeight > img.height ? img.height - sy : sourceHeight; // Floor all pixel offsets to get stable tile mapping without any overflows. // Note: For not pixel perfect aligned cells like in the DOM renderer @@ -204,10 +209,72 @@ export class ImageRenderer extends Disposable implements IDisposable { ctx.drawImage( img, Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight), - Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth), Math.ceil(finalHeight) + Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth * width / sourceWidth), Math.ceil(finalHeight * height / sourceHeight) ); } + // Fit the original source into the cell grid without distorting its aspect ratio. + public drawCellImage(imgSpec: IImageSpec, boxCols: number, boxRows: number, gridRow: number, gridCol: number, count: number, col: number, row: number): void { + const ctx = this._layers.get(imgSpec.layer); + if (!ctx) { + return; + } + const { width, height } = this.cellSize; + // Don't try to draw anything, if we cannot get valid renderer metrics. + if (width === -1 || height === -1) { + return; + } + const img = imgSpec.orig; + if (!img) { + return; + } + + const boxWidth = boxCols * width; + const boxHeight = boxRows * height; + const fitWidth = img.width * boxHeight > img.height * boxWidth; + const scale = fitWidth + ? boxWidth / Math.max(1, img.width) + : boxHeight / Math.max(1, img.height); + const xOffset = fitWidth ? 0 : (boxWidth - img.width * scale) / 2; + const yOffset = fitWidth ? (boxHeight - img.height * scale) / 2 : 0; + + // Clip source and destination together so edge tiles are not stretched. + let dx = col * width; + let dy = row * height; + let dw = count * width; + let dh = height; + let sx = (gridCol * width - xOffset) / scale; + let sy = (gridRow * height - yOffset) / scale; + let sw = dw / scale; + let sh = dh / scale; + if (sx < 0) { + const clipped = -sx * scale; + dx += clipped; + dw -= clipped; + sw += sx; + sx = 0; + } + if (sy < 0) { + const clipped = -sy * scale; + dy += clipped; + dh -= clipped; + sh += sy; + sy = 0; + } + if (sx + sw > img.width) { + sw = img.width - sx; + dw = sw * scale; + } + if (sy + sh > img.height) { + sh = img.height - sy; + dh = sh * scale; + } + if (sw <= 0 || sh <= 0 || dw <= 0 || dh <= 0) { + return; + } + ctx.drawImage(img, sx, sy, sw, sh, dx, dy, dw, dh); + } + /** * Extract a single tile from an image. */ @@ -219,19 +286,20 @@ export class ImageRenderer extends Disposable implements IDisposable { } this._rescaleImage(imgSpec, width, height); const img = imgSpec.actual!; - const cols = Math.ceil(img.width / width); - const sx = (tileId % cols) * width; - const sy = Math.floor(tileId / cols) * height; - const finalWidth = width + sx > img.width ? img.width - sx : width; - const finalHeight = sy + height > img.height ? img.height - sy : height; - - const canvas = ImageRenderer.createCanvas(this.document, finalWidth, finalHeight); + const { width: sourceWidth, height: sourceHeight } = imgSpec.actualCellSize; + const cols = Math.ceil(img.width / sourceWidth); + const sx = (tileId % cols) * sourceWidth; + const sy = Math.floor(tileId / cols) * sourceHeight; + const finalWidth = sourceWidth + sx > img.width ? img.width - sx : sourceWidth; + const finalHeight = sy + sourceHeight > img.height ? img.height - sy : sourceHeight; + + const canvas = ImageRenderer.createCanvas(this.document, Math.ceil(finalWidth * width / sourceWidth), Math.ceil(finalHeight * height / sourceHeight)); const ctx = canvas.getContext('2d'); if (ctx) { ctx.drawImage( img, Math.floor(sx), Math.floor(sy), Math.floor(finalWidth), Math.floor(finalHeight), - 0, 0, Math.floor(finalWidth), Math.floor(finalHeight) + 0, 0, canvas.width, canvas.height ); return canvas; } @@ -299,11 +367,16 @@ export class ImageRenderer extends Disposable implements IDisposable { spec.actualCellSize.height = originalHeight; return; } - const canvas = ImageRenderer.createCanvas( - this.document, - Math.ceil(spec.orig!.width * currentWidth / originalWidth), - Math.ceil(spec.orig!.height * currentHeight / originalHeight) - ); + const scaledWidth = Math.ceil(spec.orig!.width * currentWidth / originalWidth); + const scaledHeight = Math.ceil(spec.orig!.height * currentHeight / originalHeight); + // Upscale visible tiles directly; a full zoomed copy can dwarf the image budget. + if (scaledWidth * scaledHeight > spec.orig!.width * spec.orig!.height) { + spec.actual = spec.orig; + spec.actualCellSize.width = originalWidth; + spec.actualCellSize.height = originalHeight; + return; + } + const canvas = ImageRenderer.createCanvas(this.document, scaledWidth, scaledHeight); const ctx = canvas.getContext('2d'); if (ctx) { ctx.drawImage(spec.orig!, 0, 0, canvas.width, canvas.height); @@ -415,7 +488,11 @@ export class ImageRenderer extends Disposable implements IDisposable { for (let i = 0; i < width; i += bWidth) { ctx2.drawImage(blueprint, i, 0); } - ImageRenderer.createImageBitmap(this._placeholder).then(bitmap => this._placeholderBitmap = bitmap); + const placeholder = this._placeholder; + ImageRenderer.createImageBitmap(placeholder).then(bitmap => { + if (this._placeholder !== placeholder) bitmap?.close(); + else this._placeholderBitmap = bitmap; + }).catch(() => {}); } public get document(): Document | undefined { diff --git a/src/ImageStorage.ts b/src/ImageStorage.ts index 5dfca3f69fb5fcb5104ded9853b29111f926fe76..189288f0cab7a50ffc2b255c0c8f4e18d3065147 100644 --- a/src/ImageStorage.ts +++ b/src/ImageStorage.ts @@ -49,6 +49,8 @@ export class ImageStorage implements IDisposable { private _viewportMetrics: { cols: number, rows: number }; public onImageAdded: (() => void) | undefined; public onImageDeleted: ((storageId: number) => void) | undefined; + // Placeholder cells carry image references in text and colors, not tile attributes. + public onRenderLine: ((line: IBufferLine, row: number, cols: number) => void) | undefined; constructor( private _terminal: ITerminalExt, @@ -74,12 +76,9 @@ export class ImageStorage implements IDisposable { } public reset(): void { - for (const spec of this._images.values()) { - spec.marker?.dispose(); + for (const id of [...this._images.keys()]) { + this.deleteImage(id); } - // NOTE: marker.dispose above already calls ImageBitmap.close - // therefore we can just wipe the map here - this._images.clear(); this._renderer.clearAll(); } @@ -121,6 +120,7 @@ export class ImageStorage implements IDisposable { spec.orig.close(); } this.onImageDeleted?.(id); + this._renderer.refreshAll(); } /** @@ -155,6 +155,36 @@ export class ImageStorage implements IDisposable { } } + public addUnplacedImage(img: HTMLCanvasElement | ImageBitmap, opts: IAddImageOpts): number { + // never allow storage to exceed memory limit + this._evictOldest(img.width * img.height); + + let cellSize = this._renderer.cellSize; + if (cellSize.width === -1 || cellSize.height === -1) { + cellSize = CELL_SIZE_DEFAULT; + } + + const imageId = ++this._lastId; + this._images.set(imageId, { + orig: img, + origCellSize: cellSize, + actual: img, + actualCellSize: { ...cellSize }, + marker: undefined, + tileCount: 0, + virtual: true, + bufferType: this._terminal.buffer.active.type, + layer: opts.layer, + zIndex: opts.zIndex + }); + this.onImageAdded?.(); + return imageId; + } + + public getImageSpec(id: number): IImageSpec | undefined { + return this._images.get(id); + } + /** * Method to add an image to the storage. * @param img - The image to add (canvas or bitmap). @@ -226,7 +256,7 @@ export class ImageStorage implements IDisposable { // deleted images with zero tile count const zero = []; for (const [id, spec] of this._images.entries()) { - if (spec.tileCount < 1) { + if (spec.tileCount < 1 && !spec.virtual) { spec.marker?.dispose(); zero.push(id); } @@ -349,6 +379,7 @@ export class ImageStorage implements IDisposable { for (let row = start; row <= end; ++row) { const line = buffer.lines.get(row + buffer.ydisp); if (!line) return; + this.onRenderLine?.(line, row, cols); for (let col = 0; col < cols; ++col) { const e = line.getExtended(col)?.payload; if (e instanceof ImageTileInfo) { @@ -571,7 +602,7 @@ export class ImageStorage implements IDisposable { // deleted images with zero tile count const zero = []; for (const [id, spec] of this._images.entries()) { - if (spec.bufferType === 'alternate' && !spec.tileCount) { + if (spec.bufferType === 'alternate' && !spec.tileCount && !spec.virtual) { spec.marker?.dispose(); zero.push(id); } diff --git a/src/SixelHandler.ts b/src/SixelHandler.ts index 1af2d85bcdd541ed60b1e707f6186a91f1bbf1b2..0711a122ea43d5212a2851add9e744b65550ec85 100644 --- a/src/SixelHandler.ts +++ b/src/SixelHandler.ts @@ -10,6 +10,7 @@ import { RGBA8888 } from 'sixel/lib/Types'; import { ImageRenderer } from './ImageRenderer'; import { DecoderAsync, Decoder } from 'sixel/lib/Decoder'; +import { LIMITS } from 'sixel/lib/wasm'; // always free decoder ressources after decoding if it exceeds this limit const MEM_PERMA_LIMIT = 4194304; // 1024 pixels * 1024 pixels * 4 channels = 4MB @@ -18,48 +19,88 @@ const MEM_PERMA_LIMIT = 4194304; // 1024 pixels * 1024 pixels * 4 channels = 4MB const DEFAULT_PALETTE = PALETTE_ANSI_256; DEFAULT_PALETTE.set(PALETTE_VT340_COLOR); +// Why pooled: every decoder owns a wasm memory, and V8 caps live wasm memories +// per process (~124 in a sandboxed renderer). Terminals borrow a decoder only +// while a SIXEL sequence is open, so idle terminals hold none. +const MAX_IDLE_DECODERS = 2; +const idleDecoders = new Map(); +let poolPrimed = false; + +function primeDecoderPool(memoryLimit: number): void { + if (poolPrimed) return; + poolPrimed = true; + // Async compile once, off the parser's hot path; later decoders reuse the cached module. + DecoderAsync({ memoryLimit, palette: DEFAULT_PALETTE }).then( + d => releaseDecoder(d, memoryLimit), + () => { poolPrimed = false; } + ); +} + +function acquireDecoder(memoryLimit: number): Decoder { + return idleDecoders.get(memoryLimit)?.pop() ?? new Decoder({ memoryLimit, palette: DEFAULT_PALETTE }); +} + +function releaseDecoder(dec: Decoder, memoryLimit: number): void { + if (dec.memoryUsage > MEM_PERMA_LIMIT) { + dec.release(); + } + const idle = idleDecoders.get(memoryLimit) ?? []; + if (idle.length < MAX_IDLE_DECODERS) { + idle.push(dec); + idleDecoders.set(memoryLimit, idle); + } +} + export class SixelHandler implements IDcsHandler, IResetHandler { private _size = 0; private _aborted = false; private _dec: Decoder | undefined; + private _decMemoryLimit = 0; + // Color registers outlive a single image, so they live here rather than in a pooled decoder. + private readonly _palette = new Uint32Array(LIMITS.PALETTE_SIZE); constructor( private readonly _opts: IImageAddonOptions, private readonly _storage: SixelImageStorage, private readonly _coreTerminal: ITerminalExt ) { - DecoderAsync({ - memoryLimit: this._opts.pixelLimit * 4, - palette: DEFAULT_PALETTE, - paletteLimit: this._opts.sixelPaletteLimit - }).then(d => this._dec = d); + this._palette.set(DEFAULT_PALETTE); + primeDecoderPool(this._opts.pixelLimit * 4); } public reset(): void { - /** - * reset sixel decoder to defaults: - * - release all memory - * - nullify palette (4096) - * - apply default palette (256) - */ - if (this._dec) { - this._dec.release(); - // FIXME: missing interface on decoder to nullify full palette - (this._dec as any)._palette.fill(0); - this._dec.init(0, DEFAULT_PALETTE, this._opts.sixelPaletteLimit); - } + this._returnDecoder(); + this._palette.fill(0); + this._palette.set(DEFAULT_PALETTE); } public hook(params: IParams): void { this._size = 0; this._aborted = false; - if (this._dec) { - const fillColor = params.params[1] === 1 ? 0 : extractActiveBg( - this._coreTerminal._core._inputHandler._curAttrData, - this._coreTerminal._core._themeService?.colors); - this._dec.init(fillColor, null, this._opts.sixelPaletteLimit); + this._returnDecoder(); + const memoryLimit = this._opts.pixelLimit * 4; + try { + this._dec = acquireDecoder(memoryLimit); + } catch (e) { + // Why: exhausting wasm memory must drop this image, not throw out of the parser and wedge the write queue. + console.warn(`SIXEL: could not allocate decoder - ${e}`); + this._aborted = true; + return; } + this._decMemoryLimit = memoryLimit; + const fillColor = params.params[1] === 1 ? 0 : extractActiveBg( + this._coreTerminal._core._inputHandler._curAttrData, + this._coreTerminal._core._themeService?.colors); + this._dec.init(fillColor, this._palette, this._opts.sixelPaletteLimit); + } + + private _returnDecoder(): void { + const dec = this._dec; + if (!dec) return; + this._dec = undefined; + this._palette.set(dec.palette); + releaseDecoder(dec, this._decMemoryLimit); } public put(data: Uint32Array, start: number, end: number): void { @@ -83,6 +124,14 @@ export class SixelHandler implements IDcsHandler, IResetHandler { } public unhook(success: boolean): boolean | Promise { + try { + return this._unhook(success); + } finally { + this._returnDecoder(); + } + } + + private _unhook(success: boolean): boolean { if (this._aborted || !success || !this._dec) { return true; } @@ -100,9 +149,6 @@ export class SixelHandler implements IDcsHandler, IResetHandler { const canvas = ImageRenderer.createCanvas(undefined, width, height); canvas.getContext('2d')?.putImageData(new ImageData(this._dec.data8 as Uint8ClampedArray, width, height), 0, 0); - if (this._dec.memoryUsage > MEM_PERMA_LIMIT) { - this._dec.release(); - } this._storage.addImage(canvas); return true; } diff --git a/src/Types.ts b/src/Types.ts index 04328420c2bae02010bb4c9adb3fd341aed56c0c..7c4929058e1cd469515d607861ef68894b1c0854 100644 --- a/src/Types.ts +++ b/src/Types.ts @@ -103,6 +103,7 @@ export interface IImageSpec { actualCellSize: ICellSize; marker: IMarker | undefined; tileCount: number; + virtual?: boolean; bufferType: 'alternate' | 'normal'; layer: ImageLayer; zIndex: number; diff --git a/src/kitty/KittyGraphicsHandler.ts b/src/kitty/KittyGraphicsHandler.ts index de889dfff75d9ecc8ab47a025e6989ffe75bb202..7de3b740021dae5e8900a51bdec51b4a738b6561 100644 --- a/src/kitty/KittyGraphicsHandler.ts +++ b/src/kitty/KittyGraphicsHandler.ts @@ -7,6 +7,7 @@ import { IDisposable } from '@xterm/xterm'; import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt, ImageLayer } from '../Types'; import { ImageRenderer } from '../ImageRenderer'; import { CELL_SIZE_DEFAULT } from '../ImageStorage'; +import { imageType } from '../IIPMetrics'; import { KittyImageStorage } from './KittyImageStorage'; import Base64Decoder, { type DecodeStatus } from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm'; import { @@ -37,6 +38,7 @@ const DECODER_OK = Constants.DECODER_OK as unknown as DecodeStatus.OK; // Kitty graphics protocol handler with streaming base64 decoding. export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDisposable { private _aborted = false; + private _generation = 0; private _decodeError = false; private _activeDecoder: Base64Decoder | null = null; @@ -80,6 +82,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos } public reset(): void { + this._generation++; this._cleanupAllPending(); if (this._activeDecoder) { this._activeDecoder.release(); @@ -200,8 +203,38 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos this._activeDecoder = pending.decoder; } if (!this._activeDecoder) { - this._activeDecoder = new Base64Decoder(Constants.DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes); - this._activeDecoder.init(); + // Budget WASM capacity, including one page of decoder state and rounding. + const decoderCapacity = this._maxEncodedBytes + 131072; + if (decoderCapacity > this._opts.storageLimit * 1000000) { + this._aborted = true; + if (this._parsedCommand?.id !== undefined) { + this._sendResponse(this._parsedCommand.id, 'ENOMEM:pending image budget exceeded', this._parsedCommand.quiet ?? 0); + } + return; + } + const maxPending = Math.max(1, Math.floor(this._opts.storageLimit * 1000000 / decoderCapacity)); + while (this._pendingTransmissions.size >= maxPending) { + const oldest = this._pendingTransmissions.entries().next().value; + if (!oldest) break; + oldest[1].decoder.release(); + this._removePendingEntry(oldest[0]); + if (oldest[1].cmd.id !== undefined) { + this._sendResponse(oldest[1].cmd.id, 'ENOMEM:pending image budget exceeded', oldest[1].cmd.quiet ?? 0); + } + } + const decoder = new Base64Decoder(Constants.DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes); + try { + decoder.init(); + } catch (e) { + // Why: wasm memory exhaustion must drop this image, not throw out of the parser and wedge the write queue. + console.warn('KITTY: could not allocate decoder', e); + this._aborted = true; + if (this._parsedCommand?.id !== undefined) { + this._sendResponse(this._parsedCommand.id, 'ENOMEM:could not allocate decoder', this._parsedCommand.quiet ?? 0); + } + return; + } + this._activeDecoder = decoder; } if (this._activeDecoder.put(data.subarray(start, end)) !== DECODER_OK) { @@ -382,14 +415,16 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos this._sendResponse(id, 'ENOENT:image not found', cmd.quiet ?? 0, cmd.placementId); return true; } - const result = this._displayImage(image, cmd); + const result = cmd.unicodePlaceholder === 1 + ? this._placeVirtual(image, cmd) + : this._displayImage(image, cmd); return result.then(success => { this._sendResponse(id, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0, cmd.placementId); return true; }); } - private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean { + private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean, retainVirtual = false): boolean { // TODO: Support file-based transmission modes (t=f, t=t, t=s) // Currently only supports direct transmission (t=d, the default). // - t=f (file): Payload is base64-encoded file path. Terminal reads image from that path. @@ -416,7 +451,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos height: cmd.height ?? 0, format: (cmd.format ?? KittyFormat.RGBA) as 24 | 32 | 100, compression: cmd.compression ?? '' - }); + }, retainVirtual); return true; } @@ -428,12 +463,15 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos return true; } - this._handleTransmit(cmd, bytes, decodeError); + const virtual = cmd.unicodePlaceholder === 1; + this._handleTransmit(cmd, bytes, decodeError, virtual); const id = cmd.id ?? this._kittyStorage.lastImageId; const image = this._kittyStorage.getImage(id); if (image) { - const result = this._displayImage(image, cmd); + const result = virtual + ? this._placeVirtual(image, cmd, true) + : this._displayImage(image, cmd); if (cmd.id !== undefined) { return result.then(success => { this._sendResponse(id, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0); @@ -500,28 +538,25 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos // Per spec: default delete selector is 'a' (delete all visible placements) const selector = cmd.deleteSelector ?? 'a'; - // TODO: Distinguish lowercase (delete placements only) from uppercase - // (delete placements + free stored image data). Currently both variants - // free everything since we don't separate stored data from placements. switch (selector) { case 'a': + this._kittyStorage.deletePhysicalPlacements(); + break; case 'A': this._cleanupAllPending(); - this._kittyStorage.deleteAll(); + this._kittyStorage.deleteAll(true); break; case 'i': case 'I': - // TODO: When placement id tracking is implemented (see TODO in - // KittyImageStorage), d=i with p= should delete only that - // specific placement, while d=i without p should delete all - // placements for the image. if (cmd.id !== undefined) { - const pending = this._pendingTransmissions.get(cmd.id); - if (pending) { - pending.decoder.release(); + if (selector === 'I') { + const pending = this._pendingTransmissions.get(cmd.id); + pending?.decoder.release(); + this._removePendingEntry(cmd.id); + this._kittyStorage.deleteById(cmd.id, cmd.placementId); + } else { + this._kittyStorage.deletePlacementsById(cmd.id, cmd.placementId); } - this._removePendingEntry(cmd.id); - this._kittyStorage.deleteById(cmd.id); } break; default: @@ -550,49 +585,17 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos } private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise { + const generation = this._generation; let bitmap: ImageBitmap | undefined = await this._createBitmap(image); try { - const cropX = Math.max(0, cmd.x ?? 0); - const cropY = Math.max(0, cmd.y ?? 0); - const cropW = cmd.sourceWidth || (bitmap.width - cropX); - const cropH = cmd.sourceHeight || (bitmap.height - cropY); - - const maxCropW = Math.max(0, bitmap.width - cropX); - const maxCropH = Math.max(0, bitmap.height - cropY); - const finalCropW = Math.max(0, Math.min(cropW, maxCropW)); - const finalCropH = Math.max(0, Math.min(cropH, maxCropH)); - - if (finalCropW === 0 || finalCropH === 0) { - throw new Error('invalid source rectangle'); - } - - if (cropX !== 0 || cropY !== 0 || finalCropW !== bitmap.width || finalCropH !== bitmap.height) { - const cropped = await createImageBitmap(bitmap, cropX, cropY, finalCropW, finalCropH); - bitmap.close(); - bitmap = cropped; - } + if (generation !== this._generation) throw new Error('image decode canceled'); + const cropped = this._cropImage(bitmap, cmd); + bitmap = cropped instanceof Promise ? await cropped : cropped; const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width; const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height; - - // Per spec: c/r default to image's natural cell dimensions. - // If only one of c/r is specified, compute the other from image aspect ratio. - let imgCols: number; - let imgRows: number; - if (cmd.columns !== undefined && cmd.rows !== undefined) { - imgCols = cmd.columns; - imgRows = cmd.rows; - } else if (cmd.columns !== undefined) { - imgCols = cmd.columns; - imgRows = Math.max(1, Math.ceil((bitmap.height / bitmap.width) * (imgCols * cw) / ch)); - } else if (cmd.rows !== undefined) { - imgRows = cmd.rows; - imgCols = Math.max(1, Math.ceil((bitmap.width / bitmap.height) * (imgRows * ch) / cw)); - } else { - imgCols = Math.ceil(bitmap.width / cw); - imgRows = Math.ceil(bitmap.height / ch); - } + let { cols: imgCols, rows: imgRows } = this._placementGrid(bitmap, cmd, cw, ch); let w = bitmap.width; let h = bitmap.height; @@ -660,6 +663,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos } } + if (generation !== this._generation) throw new Error('image decode canceled'); const zIndex = cmd.zIndex ?? 0; this._kittyStorage.addImage(image.id, bitmap, true, layer, zIndex); bitmap = undefined; // ownership transferred to storage @@ -684,6 +688,75 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos } } + // Keep the uncropped path synchronous to preserve its parser scheduling. + private _cropImage(bitmap: ImageBitmap, cmd: IKittyCommand): ImageBitmap | Promise { + const cropX = Math.max(0, cmd.x ?? 0); + const cropY = Math.max(0, cmd.y ?? 0); + const cropW = cmd.sourceWidth || (bitmap.width - cropX); + const cropH = cmd.sourceHeight || (bitmap.height - cropY); + + const maxCropW = Math.max(0, bitmap.width - cropX); + const maxCropH = Math.max(0, bitmap.height - cropY); + const finalCropW = Math.max(0, Math.min(cropW, maxCropW)); + const finalCropH = Math.max(0, Math.min(cropH, maxCropH)); + + if (finalCropW === 0 || finalCropH === 0) { + throw new Error('invalid source rectangle'); + } + + if (cropX !== 0 || cropY !== 0 || finalCropW !== bitmap.width || finalCropH !== bitmap.height) { + return createImageBitmap(bitmap, cropX, cropY, finalCropW, finalCropH).then(cropped => { + bitmap.close(); + return cropped; + }); + } + return bitmap; + } + + // An omitted grid dimension follows the source aspect ratio. + private _placementGrid(bitmap: ImageBitmap, cmd: IKittyCommand, cw: number, ch: number): { cols: number; rows: number } { + if (cmd.columns !== undefined && cmd.rows !== undefined) { + return { cols: cmd.columns, rows: cmd.rows }; + } + if (cmd.columns !== undefined) { + return { cols: cmd.columns, rows: Math.max(1, Math.ceil((bitmap.height / bitmap.width) * (cmd.columns * cw) / ch)) }; + } + if (cmd.rows !== undefined) { + return { cols: Math.max(1, Math.ceil((bitmap.width / bitmap.height) * (cmd.rows * ch) / cw)), rows: cmd.rows }; + } + return { cols: Math.ceil(bitmap.width / cw), rows: Math.ceil(bitmap.height / ch) }; + } + + private async _placeVirtual(image: IKittyImageData, cmd: IKittyCommand, replaces = false): Promise { + const generation = this._generation; + const isCurrent = (): boolean => generation === this._generation && this._kittyStorage.getImage(image.id) === image; + let bitmap: ImageBitmap | undefined; + try { + bitmap = await this._createBitmap(image); + if (!isCurrent()) throw new Error('image decode canceled'); + const cropped = this._cropImage(bitmap, cmd); + bitmap = cropped instanceof Promise ? await cropped : cropped; + + const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width; + const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height; + const { cols, rows } = this._placementGrid(bitmap, cmd, cw, ch); + + if (bitmap.width * bitmap.height > this._opts.pixelLimit) { + throw new Error('image exceeds pixel limit'); + } + + if (!isCurrent()) throw new Error('image decode canceled'); + this._kittyStorage.addVirtualPlacement(image.id, cmd.placementId ?? 0, bitmap, cols, rows); + bitmap = undefined; // ownership transferred to storage + if (replaces) this._kittyStorage.discardVirtualPlacements(image.id, cmd.placementId ?? 0); + return true; + } catch { + bitmap?.close(); + if (replaces && isCurrent()) this._kittyStorage.discardVirtualPlacements(image.id); + return false; + } + } + // Create ImageBitmap from already-decoded image data. private async _createBitmap(image: IKittyImageData): Promise { let bytes: Uint8Array = new Uint8Array(await image.data.arrayBuffer()); @@ -693,6 +766,12 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos } if (image.format === KittyFormat.PNG) { + const metrics = imageType(bytes); + // IHDR dimensions are parsed with signed shifts, so a value >= 0x80000000 comes + // back negative and a bare `>` pixel-limit test passes it; require positive. + if (metrics.mime !== 'image/png' || !(metrics.width > 0) || !(metrics.height > 0) || metrics.width * metrics.height > this._opts.pixelLimit) { + throw new RangeError('PNG exceeds pixel limit or has invalid dimensions'); + } const blob = new Blob([bytes as BlobPart], { type: 'image/png' }); if (!window.createImageBitmap) { const url = URL.createObjectURL(blob); @@ -775,27 +854,45 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos private async _decompressZlib(compressed: Uint8Array): Promise { try { return await this._decompress(compressed, 'deflate'); - } catch { + } catch (error) { + if (error instanceof RangeError) throw error; return await this._decompress(compressed, 'deflate-raw'); } } private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise { - const ds = new DecompressionStream(format); - const writer = ds.writable.getWriter(); - writer.write(compressed as BufferSource); - writer.close(); - + const limit = Math.min(this._opts.kittySizeLimit, this._opts.pixelLimit * 4, this._opts.storageLimit * 1000000); + let offsetIn = 0; + // Bound inflation within one native transform before its output is budgeted. + const source = new ReadableStream({ + pull(controller) { + if (offsetIn >= compressed.length) { + controller.close(); + return; + } + const end = Math.min(offsetIn + 4096, compressed.length); + controller.enqueue(new Uint8Array(compressed.subarray(offsetIn, end))); + offsetIn = end; + } + }); + const reader = source.pipeThrough(new DecompressionStream(format)).getReader(); const chunks: Uint8Array[] = []; - const reader = ds.readable.getReader(); - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); + let totalLength = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalLength += value.byteLength; + if (totalLength > limit) { + await reader.cancel().catch(() => {}); + throw new RangeError('decompressed image exceeds byte limit'); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); } - const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0); const result = new Uint8Array(totalLength); let offset = 0; for (const chunk of chunks) { diff --git a/src/kitty/KittyGraphicsTypes.ts b/src/kitty/KittyGraphicsTypes.ts index 4288dd0a4180eca8f3ae28ec0805f834b2a0a629..77ddbf365cd76fdc9b109e5ac497a261d2c76293 100644 --- a/src/kitty/KittyGraphicsTypes.ts +++ b/src/kitty/KittyGraphicsTypes.ts @@ -78,7 +78,9 @@ export const enum KittyKey { // Delete selector (a/A=all, i/I=by id, c/C=at cursor, etc.) — only used when a=d DELETE_SELECTOR = 'd', // Placement ID for targeting specific placements - PLACEMENT_ID = 'p' + PLACEMENT_ID = 'p', + // U=1 creates a prototype for placeholder cells without placing cursor tiles. + UNICODE_PLACEHOLDER = 'U' } // Pixel format constants @@ -112,6 +114,7 @@ export interface IKittyCommand { deleteSelector?: string; placementId?: number; compression?: string; + unicodePlaceholder?: number; payload?: string; } @@ -139,6 +142,13 @@ export interface IKittyImageData { compression?: string; } +export interface IVirtualPlacement { + placementId: number; + storageId: number; + cols: number; + rows: number; +} + // Parses Kitty graphics control data into a command object. export function parseKittyCommand(data: string): IKittyCommand { const cmd: IKittyCommand = {}; @@ -188,8 +198,73 @@ export function parseKittyCommand(data: string): IKittyCommand { case KittyKey.CURSOR_MOVEMENT: cmd.cursorMovement = numValue; break; case KittyKey.Z_INDEX: cmd.zIndex = numValue; break; case KittyKey.PLACEMENT_ID: cmd.placementId = numValue; break; + case KittyKey.UNICODE_PLACEHOLDER: cmd.unicodePlaceholder = numValue; break; } } return cmd; } + + +// Source patches can only name files included in the published package. + +export const PLACEHOLDER_CHAR = 0x10EEEE; + +// Protocol table order; values are one-based so zero can mean an omitted diacritic. +export const PLACEHOLDER_DIACRITICS: readonly number[] = [ + 0x305, 0x30D, 0x30E, 0x310, 0x312, 0x33D, 0x33E, 0x33F, 0x346, 0x34A, + 0x34B, 0x34C, 0x350, 0x351, 0x352, 0x357, 0x35B, 0x363, 0x364, 0x365, + 0x366, 0x367, 0x368, 0x369, 0x36A, 0x36B, 0x36C, 0x36D, 0x36E, 0x36F, + 0x483, 0x484, 0x485, 0x486, 0x487, 0x592, 0x593, 0x594, 0x595, 0x597, + 0x598, 0x599, 0x59C, 0x59D, 0x59E, 0x59F, 0x5A0, 0x5A1, 0x5A8, 0x5A9, + 0x5AB, 0x5AC, 0x5AF, 0x5C4, 0x610, 0x611, 0x612, 0x613, 0x614, 0x615, + 0x616, 0x617, 0x657, 0x658, 0x659, 0x65A, 0x65B, 0x65D, 0x65E, 0x6D6, + 0x6D7, 0x6D8, 0x6D9, 0x6DA, 0x6DB, 0x6DC, 0x6DF, 0x6E0, 0x6E1, 0x6E2, + 0x6E4, 0x6E7, 0x6E8, 0x6EB, 0x6EC, 0x730, 0x732, 0x733, 0x735, 0x736, + 0x73A, 0x73D, 0x73F, 0x740, 0x741, 0x743, 0x745, 0x747, 0x749, 0x74A, + 0x7EB, 0x7EC, 0x7ED, 0x7EE, 0x7EF, 0x7F0, 0x7F1, 0x7F3, 0x816, 0x817, + 0x818, 0x819, 0x81B, 0x81C, 0x81D, 0x81E, 0x81F, 0x820, 0x821, 0x822, + 0x823, 0x825, 0x826, 0x827, 0x829, 0x82A, 0x82B, 0x82C, 0x82D, 0x951, + 0x953, 0x954, 0xF82, 0xF83, 0xF86, 0xF87, 0x135D, 0x135E, 0x135F, 0x17DD, + 0x193A, 0x1A17, 0x1A75, 0x1A76, 0x1A77, 0x1A78, 0x1A79, 0x1A7A, 0x1A7B, 0x1A7C, + 0x1B6B, 0x1B6D, 0x1B6E, 0x1B6F, 0x1B70, 0x1B71, 0x1B72, 0x1B73, 0x1CD0, 0x1CD1, + 0x1CD2, 0x1CDA, 0x1CDB, 0x1CE0, 0x1DC0, 0x1DC1, 0x1DC3, 0x1DC4, 0x1DC5, 0x1DC6, + 0x1DC7, 0x1DC8, 0x1DC9, 0x1DCB, 0x1DCC, 0x1DD1, 0x1DD2, 0x1DD3, 0x1DD4, 0x1DD5, + 0x1DD6, 0x1DD7, 0x1DD8, 0x1DD9, 0x1DDA, 0x1DDB, 0x1DDC, 0x1DDD, 0x1DDE, 0x1DDF, + 0x1DE0, 0x1DE1, 0x1DE2, 0x1DE3, 0x1DE4, 0x1DE5, 0x1DE6, 0x1DFE, 0x20D0, 0x20D1, + 0x20D4, 0x20D5, 0x20D6, 0x20D7, 0x20DB, 0x20DC, 0x20E1, 0x20E7, 0x20E9, 0x20F0, + 0x2CEF, 0x2CF0, 0x2CF1, 0x2DE0, 0x2DE1, 0x2DE2, 0x2DE3, 0x2DE4, 0x2DE5, 0x2DE6, + 0x2DE7, 0x2DE8, 0x2DE9, 0x2DEA, 0x2DEB, 0x2DEC, 0x2DED, 0x2DEE, 0x2DEF, 0x2DF0, + 0x2DF1, 0x2DF2, 0x2DF3, 0x2DF4, 0x2DF5, 0x2DF6, 0x2DF7, 0x2DF8, 0x2DF9, 0x2DFA, + 0x2DFB, 0x2DFC, 0x2DFD, 0x2DFE, 0x2DFF, 0xA66F, 0xA67C, 0xA67D, 0xA6F0, 0xA6F1, + 0xA8E0, 0xA8E1, 0xA8E2, 0xA8E3, 0xA8E4, 0xA8E5, 0xA8E6, 0xA8E7, 0xA8E8, 0xA8E9, + 0xA8EA, 0xA8EB, 0xA8EC, 0xA8ED, 0xA8EE, 0xA8EF, 0xA8F0, 0xA8F1, 0xAAB0, 0xAAB2, + 0xAAB3, 0xAAB7, 0xAAB8, 0xAABE, 0xAABF, 0xAAC1, 0xFE20, 0xFE21, 0xFE22, 0xFE23, + 0xFE24, 0xFE25, 0xFE26, 0x10A0F, 0x10A38, 0x1D185, 0x1D186, 0x1D187, 0x1D188, 0x1D189, + 0x1D1AA, 0x1D1AB, 0x1D1AC, 0x1D1AD, 0x1D242, 0x1D243, 0x1D244, +]; + +const PLACEHOLDER_DIACRITIC_VALUES: Map = new Map(); +for (let i = 0; i < PLACEHOLDER_DIACRITICS.length; ++i) { + PLACEHOLDER_DIACRITIC_VALUES.set(PLACEHOLDER_DIACRITICS[i], i + 1); +} + +// Zero values may inherit from the preceding placeholder cell. +export interface IPlaceholderLocation { + row: number; + column: number; + highByte: number; +} + +function placeholderDiacriticValue(mark: string | undefined): number { + return mark === undefined ? 0 : PLACEHOLDER_DIACRITIC_VALUES.get(mark.codePointAt(0) ?? -1) ?? -1; +} + +export function decodePlaceholderLocation(chars: string): IPlaceholderLocation { + const [, row, column, highByte] = chars; + return { + row: placeholderDiacriticValue(row), + column: placeholderDiacriticValue(column), + highByte: placeholderDiacriticValue(highByte) + }; +} diff --git a/src/kitty/KittyImageStorage.ts b/src/kitty/KittyImageStorage.ts index 1f5c09ec9e2700f8f6dbd8436a1802217dfc99ef..2a0df51c965c4e2c4c6f1935c129052cadf12153 100644 --- a/src/kitty/KittyImageStorage.ts +++ b/src/kitty/KittyImageStorage.ts @@ -3,16 +3,27 @@ * @license MIT */ +import type { IBufferLine } from 'common/buffer/Types'; import { IDisposable } from '@xterm/xterm'; +import { CellData } from 'common/buffer/CellData'; +import { FgFlags } from 'common/buffer/Constants'; +import { ImageRenderer } from '../ImageRenderer'; import { ImageStorage } from '../ImageStorage'; -import { ImageLayer, IAddImageOpts } from '../Types'; -import { IKittyImageData } from './KittyGraphicsTypes'; +import { Attributes, ITerminalExt, ImageLayer, IAddImageOpts } from '../Types'; +import { decodePlaceholderLocation, IKittyImageData, IPlaceholderLocation, IVirtualPlacement, PLACEHOLDER_CHAR } from './KittyGraphicsTypes'; + +interface IPlaceholderCell { + imageId: number; + placementId: number; + location: IPlaceholderLocation; +} // Kitty-specific image storage controller. // // Wraps shared ImageStorage with kitty protocol semantics: // - tracks transmitted image payloads by kitty image id // - tracks kitty image id -> shared ImageStorage id mapping for displayed images +// - tracks virtual placements that U+10EEEE cells display // - mirrors shared-storage evictions into kitty maps // - applies protocol-level undisplayed-image eviction policy export class KittyImageStorage implements IDisposable { @@ -20,32 +31,33 @@ export class KittyImageStorage implements IDisposable { private _nextImageId = 1; private readonly _images: Map = new Map(); - // TODO: Support multiple placements per image. The kitty spec identifies - // placements by an (image id, placement id) pair — same i + different p - // values should coexist, and same i + same p should replace the prior - // placement. Currently we track only one storage entry per kitty image id, - // so multiple placements of the same image overwrite each other. Fixing - // this requires changing these maps to Map> - // (kittyId → placementId → storageId) and updating addImage/deleteById - // accordingly. The underlying shared ImageStorage would also need to - // support multiple entries per logical image. + // TODO: Track physical placements by (image id, placement id), like virtual placements. private readonly _kittyIdToStorageId: Map = new Map(); private readonly _storageIdToKittyId: Map = new Map(); + private readonly _virtualPlacements: Map> = new Map(); + private readonly _virtualStorageIds = new Map(); private readonly _previousOnImageDeleted: ((storageId: number) => void) | undefined; private readonly _wrappedOnImageDeleted: (storageId: number) => void; private readonly _handleStorageImageDeleted = (storageId: number): void => { + const virtual = this._virtualStorageIds.get(storageId); + if (virtual) { + this._virtualStorageIds.delete(storageId); + const placements = this._virtualPlacements.get(virtual.kittyId); + placements?.delete(virtual.placementId); + if (!placements?.size) this._virtualPlacements.delete(virtual.kittyId); + } const kittyId = this._storageIdToKittyId.get(storageId); if (kittyId !== undefined) { this._kittyIdToStorageId.delete(kittyId); this._storageIdToKittyId.delete(storageId); - this._images.delete(kittyId); } }; private _addImageOpts: IAddImageOpts = { scrolling: true, layer: 'top', zIndex: 0, cursorPos: 'iip' }; constructor( - private readonly _storage: ImageStorage + private readonly _storage: ImageStorage, + private readonly _renderer: ImageRenderer ) { this._previousOnImageDeleted = this._storage.onImageDeleted; this._wrappedOnImageDeleted = (storageId: number) => { @@ -53,6 +65,27 @@ export class KittyImageStorage implements IDisposable { this._handleStorageImageDeleted(storageId); }; this._storage.onImageDeleted = this._wrappedOnImageDeleted; + this._storage.onRenderLine = this._renderPlaceholderLine; + } + + // Reserved placeholder text stays in the buffer but never draws as a glyph. + public hidePlaceholderGlyphs(terminal: ITerminalExt): void { + const buffer = terminal._core.buffer; + let cell: CellData | undefined; + const start = terminal.buffer.active.viewportY; + for (let row = start; row < start + terminal.rows; ++row) { + const line = buffer.lines.get(row); + if (!line) continue; + for (let col = 0; col < terminal.cols; ++col) { + if (line.getFg(col) & FgFlags.INVISIBLE) continue; + if (line.getCodePoint(col) !== PLACEHOLDER_CHAR + && (!line.isCombined(col) || line.getString(col).codePointAt(0) !== PLACEHOLDER_CHAR)) continue; + cell ??= new CellData(); + line.loadCell(col, cell); + cell.fg |= FgFlags.INVISIBLE; + line.setCell(col, cell); + } + } } public reset(): void { @@ -60,6 +93,8 @@ export class KittyImageStorage implements IDisposable { this._images.clear(); this._kittyIdToStorageId.clear(); this._storageIdToKittyId.clear(); + this._virtualPlacements.clear(); + this._virtualStorageIds.clear(); } public dispose(): void { @@ -67,11 +102,15 @@ export class KittyImageStorage implements IDisposable { if (this._storage.onImageDeleted === this._wrappedOnImageDeleted) { this._storage.onImageDeleted = this._previousOnImageDeleted; } + if (this._storage.onRenderLine === this._renderPlaceholderLine) { + this._storage.onRenderLine = undefined; + } } - public storeImage(id: number | undefined, imageData: Omit): number { + public storeImage(id: number | undefined, imageData: Omit, retainVirtual = false): number { const imageId = id ?? this._nextImageId++; + if (!retainVirtual) this._deleteVirtualPlacements(imageId); const oldStorageId = this._kittyIdToStorageId.get(imageId); if (oldStorageId !== undefined) { this._storage.deleteImage(oldStorageId); @@ -83,6 +122,25 @@ export class KittyImageStorage implements IDisposable { this._evictUndisplayedImages(); } + // Encoded images awaiting placement are outside ImageStorage's pixel budget. + // Unplaced payloads are evicted first so a new upload cannot erase a visible + // image while abandoned blobs still hold budget; placed ones go only when + // that is not enough, because the byte cap is a hard bound. The new image is + // always stored, so an oversized one overshoots by at most one payload + // (itself bounded by kittySizeLimit) rather than being dropped after an OK ack. + const byteLimit = this._storage.getLimit() * 1000000; + this._images.delete(imageId); + let retainedBytes = 0; + for (const image of this._images.values()) retainedBytes += image.data.size; + for (const evictPlaced of [false, true]) { + for (const [oldestId, image] of this._images) { + if (retainedBytes + imageData.data.size <= byteLimit) break; + if ((this._kittyIdToStorageId.has(oldestId) || this._virtualPlacements.has(oldestId)) !== evictPlaced) continue; + retainedBytes -= image.data.size; + this.deleteById(oldestId); + } + } + this._images.set(imageId, { ...imageData, id: imageId @@ -111,23 +169,73 @@ export class KittyImageStorage implements IDisposable { return this._images.get(kittyId); } - public deleteById(kittyId: number): void { - this._images.delete(kittyId); + // Fit prototypes at render time so each placement holds only the source pixels. + public addVirtualPlacement(kittyId: number, placementId: number, image: HTMLCanvasElement | ImageBitmap, cols: number, rows: number): void { + const previous = this._virtualPlacements.get(kittyId)?.get(placementId); + if (image.width * image.height * 4 > this._storage.getLimit() * 1000000) throw new Error('virtual image exceeds storage limit'); + if (!previous && this._virtualStorageIds.size >= KittyImageStorage._maxStoredImages) throw new Error('virtual placement limit exceeded'); + if (previous) this._storage.deleteImage(previous.storageId); + const storageId = this._storage.addUnplacedImage(image, { scrolling: false, layer: 'top', zIndex: 0, cursorPos: 'iip' }); + let placements = this._virtualPlacements.get(kittyId); + if (!placements) { + placements = new Map(); + this._virtualPlacements.set(kittyId, placements); + } + placements.set(placementId, { placementId, storageId, cols, rows }); + this._virtualStorageIds.set(storageId, { kittyId, placementId }); + this._renderer.refreshAll(); + } + + private _virtualPlacementFor(kittyId: number, placementId: number): IVirtualPlacement | undefined { + const placements = this._virtualPlacements.get(kittyId); + return placementId === 0 ? placements?.values().next().value : placements?.get(placementId); + } + + public discardVirtualPlacements(kittyId: number, keepPlacementId?: number): void { + this._deleteVirtualPlacements(kittyId, undefined, keepPlacementId); + } + + private _deleteVirtualPlacements(kittyId: number, placementId?: number, keepPlacementId?: number): void { + const placements = this._virtualPlacements.get(kittyId); + if (!placements) return; + for (const placement of [...placements.values()]) { + if ((placementId === undefined || placementId === placement.placementId) && placement.placementId !== keepPlacementId) { + this._storage.deleteImage(placement.storageId); + } + } + } + + public deletePlacementsById(kittyId: number, placementId?: number): void { + this._deleteVirtualPlacements(kittyId, placementId); + if (placementId !== undefined) return; const storageId = this._kittyIdToStorageId.get(kittyId); if (storageId !== undefined) { - this._storage.deleteImage(storageId); this._kittyIdToStorageId.delete(kittyId); this._storageIdToKittyId.delete(storageId); + this._storage.deleteImage(storageId); } } - public deleteAll(): void { - this._images.clear(); - for (const storageId of this._kittyIdToStorageId.values()) { + public deleteById(kittyId: number, placementId?: number): void { + this.deletePlacementsById(kittyId, placementId); + if (!this._virtualPlacements.has(kittyId) && !this._kittyIdToStorageId.has(kittyId)) { + this._images.delete(kittyId); + } + } + + public deletePhysicalPlacements(): void { + for (const [kittyId, storageId] of [...this._kittyIdToStorageId]) { + this._kittyIdToStorageId.delete(kittyId); + this._storageIdToKittyId.delete(storageId); this._storage.deleteImage(storageId); } - this._kittyIdToStorageId.clear(); - this._storageIdToKittyId.clear(); + } + + public deleteAll(preserveVirtual = false): void { + if (preserveVirtual) this.deletePhysicalPlacements(); + for (const kittyId of [...this._images.keys()]) { + if (!preserveVirtual || !this._virtualPlacements.has(kittyId)) this.deleteById(kittyId); + } } public get images(): ReadonlyMap { @@ -147,9 +255,96 @@ export class KittyImageStorage implements IDisposable { if (this._images.size <= KittyImageStorage._maxStoredImages / 2) { break; } - if (!this._kittyIdToStorageId.has(kittyId)) { + if (!this._kittyIdToStorageId.has(kittyId) && !this._virtualPlacements.has(kittyId)) { this._images.delete(kittyId); + this._virtualPlacements.delete(kittyId); + } + } + } + + // Omitted diacritics inherit only from a consecutive matching placeholder. + private readonly _renderPlaceholderLine = (line: IBufferLine, row: number, cols: number): void => { + if (this._virtualPlacements.size === 0) { + return; + } + let runLength = 0; + let previous: IPlaceholderCell | undefined; + for (let col = 0; col <= cols; ++col) { + const cell = col < cols ? this._placeholderCell(line, col) : undefined; + if (runLength > 0 && cell !== undefined && previous !== undefined + && cell.imageId === previous.imageId + && cell.placementId === previous.placementId + && (cell.location.row === 0 || cell.location.row === previous.location.row) + && (cell.location.column === 0 || cell.location.column === previous.location.column + 1) + && (cell.location.highByte === 0 || cell.location.highByte === previous.location.highByte)) { + runLength++; + cell.location.row = Math.max(previous.location.row, 1); + cell.location.column = previous.location.column + 1; + cell.location.highByte = Math.max(previous.location.highByte, 1); + } else { + if (runLength > 0 && previous !== undefined) { + this._drawPlaceholderRun(previous, col - runLength, runLength, row); + } + runLength = 0; + if (cell !== undefined) { + runLength = 1; + if (!cell.location.row) cell.location.row = 1; + if (!cell.location.column) cell.location.column = 1; + if (!cell.location.highByte) cell.location.highByte = 1; + } } + previous = cell ?? previous; + } + }; + + private _placeholderCell(line: IBufferLine, col: number): IPlaceholderCell | undefined { + if (!line.isCombined(col) && line.getCodePoint(col) !== PLACEHOLDER_CHAR) { + return undefined; + } + const chars = line.getString(col); + if (chars.codePointAt(0) !== PLACEHOLDER_CHAR) { + return undefined; + } + + const fg = line.getFg(col); + const fgMode = fg & Attributes.CM_MASK; + const imageId = fgMode === Attributes.CM_RGB + ? fg & Attributes.RGB_MASK + : fgMode === 0 ? 0 : fg & Attributes.PCOLOR_MASK; + if (fgMode === 0) { + return undefined; + } + + // Placement IDs use underline color without an underline style. + let placementId = 0; + const underlineColor = line.getExtended(col)?.underlineColor ?? 0; + if (underlineColor > 0) { + const underlineMode = underlineColor & Attributes.CM_MASK; + placementId = underlineMode === Attributes.CM_RGB + ? underlineColor & Attributes.RGB_MASK + : underlineMode === 0 ? 0 : underlineColor & Attributes.PCOLOR_MASK; + } + + const location = decodePlaceholderLocation(chars); + if (location.row < 0 || location.column < 0 || location.highByte < 0 || location.highByte > 256) return undefined; + return { imageId, placementId, location }; + } + + private _drawPlaceholderRun(cell: IPlaceholderCell, startCol: number, count: number, row: number): void { + const imageId = (cell.imageId | ((cell.location.highByte - 1) << 24)) >>> 0; + const placement = this._virtualPlacementFor(imageId, cell.placementId); + const spec = placement && this._storage.getImageSpec(placement.storageId); + // `column` is 1-based and holds the last cell of the run. + const gridRow = cell.location.row - 1; + const gridCol = cell.location.column - count; + if (!placement || !spec || gridRow < 0 || gridRow >= placement.rows || gridCol < 0) { + return; + } + // Cells past the placement's box show nothing. + const tiles = Math.min(count, placement.cols - gridCol); + if (tiles <= 0) { + return; } + this._renderer.drawCellImage(spec, placement.cols, placement.rows, gridRow, gridCol, tiles, startCol, row); } }