From 19c53e5248aadda14dc07ce030baa75a96caaece Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:15:54 +0000 Subject: [PATCH] fix(browser): let Grab page element pick and track elements inside open shadow roots (#34265) * fix(browser): let Grab page element pick and track elements inside open shadow roots Document hit-testing and marker lookups stop at a shadow host, so Grab only ever picked the host. The overlay now descends through open shadow roots, and both the overlay and the annotation viewport bridge find the marked element and listen for scrolls inside them. Closed roots stay opaque. Co-authored-by: Felipe <5784455+fbetancourtc@users.noreply.github.com> * fix(browser): track slotted ancestors and stop re-walking shadow roots for a lost target * fix(browser): walk shadow roots once per selection and count fixed shadow hosts * fix(browser): restore grab pointer events if a page's shadowRoot throws; type the shadow fixtures * fix(browser): keep composed-tree walks safe from page-named form controls --------- Co-authored-by: Felipe <5784455+fbetancourtc@users.noreply.github.com> --- .../browser/grab-guest-overlay-follow.test.ts | 27 ++++++++++++-- src/main/browser/grab-guest-overlay-script.ts | 22 ++++++++++- src/main/browser/grab-guest-react-script.ts | 3 +- src/main/browser/grab-guest-script.test.ts | 2 +- src/main/browser/grab-guest-script.ts | 2 + .../browser-annotation-viewport-bridge.ts | 36 ++++++++++++++---- ...er-annotation-viewport-live-target.test.ts | 37 ++++++++++++++++--- src/shared/browser-composed-tree-script.ts | 20 ++++++++++ 8 files changed, 130 insertions(+), 19 deletions(-) create mode 100644 src/shared/browser-composed-tree-script.ts diff --git a/src/main/browser/grab-guest-overlay-follow.test.ts b/src/main/browser/grab-guest-overlay-follow.test.ts index 9235fe7d2d2..c38c6b8f816 100644 --- a/src/main/browser/grab-guest-overlay-follow.test.ts +++ b/src/main/browser/grab-guest-overlay-follow.test.ts @@ -11,7 +11,7 @@ afterEach(async () => { await Promise.all(guestAborts.splice(0).map((abort) => abort())) }) -async function armGuest(previousMarker?: string) { +async function armGuest(previousMarker?: string, inShadowRoot = false) { const { Window } = await import('happy-dom') const window = new Window() guestAborts.push(() => window.happyDOM.abort()) @@ -32,14 +32,20 @@ async function armGuest(previousMarker?: string) { target.setAttribute(BROWSER_GRAB_SELECTED_ATTRIBUTE, previousMarker) } scroller.appendChild(target) - document.body.appendChild(scroller) + const shadowHost = document.createElement('div') + document.body.appendChild(inShadowRoot ? shadowHost : scroller) + if (inShadowRoot) { + const pageRoot = shadowHost.attachShadow({ mode: 'open' }) + pageRoot.appendChild(scroller) + Object.defineProperty(pageRoot, 'elementFromPoint', { value: () => target }) + } let targetRect: Rect = { x: 40, y: 300, width: 120, height: 30 } const contentSize = { width: 120, height: 30 } const rootSize = { width: 1000, height: 800 } const spacing = { padding: 0, border: 0 } target.getBoundingClientRect = () => new window.DOMRect(targetRect.x, targetRect.y, targetRect.width, targetRect.height) - document.elementFromPoint = () => target + document.elementFromPoint = () => (inShadowRoot ? shadowHost : target) const frames: (() => void)[] = [] const resizeCallbacks: (() => void)[] = [] @@ -89,7 +95,7 @@ async function armGuest(previousMarker?: string) { } } const outline = (): Rect => { - const box = shadowRoots[0].firstElementChild?.firstElementChild + const box = shadowRoots.at(-1)?.firstElementChild?.firstElementChild if (!(box instanceof window.HTMLElement)) { throw new Error('outline not rendered') } @@ -187,6 +193,19 @@ describe('grab overlay after the click', () => { expect(guest.outline()).toEqual({ x: 40, y: 180, width: 120, height: 30 }) }) + it('selects an element inside an open shadow root and follows its scroll', async () => { + const guest = await armGuest(undefined, true) + expect(guest.target.getAttribute(BROWSER_GRAB_SELECTED_ATTRIBUTE)).toBe('selection-1') + expect(guest.outline()).toEqual({ x: 40, y: 300, width: 120, height: 30 }) + + // Scroll events stay inside the shadow root, so window capture never sees them. + guest.moveTarget({ x: 40, y: 180, width: 120, height: 30 }) + guest.scroller.dispatchEvent(new guest.window.Event('scroll')) + guest.flushFrames() + + expect(guest.outline()).toEqual({ x: 40, y: 180, width: 120, height: 30 }) + }) + it('re-measures on window resize and when the element changes size', async () => { const guest = await armGuest() diff --git a/src/main/browser/grab-guest-overlay-script.ts b/src/main/browser/grab-guest-overlay-script.ts index ea54d48e9c8..2b0c9f6c161 100644 --- a/src/main/browser/grab-guest-overlay-script.ts +++ b/src/main/browser/grab-guest-overlay-script.ts @@ -62,7 +62,18 @@ export const GRAB_GUEST_OVERLAY_SCRIPT = ` var host = document.createElement('d // Exclude the overlay from hit testing to find the page element underneath. host.style.pointerEvents = 'none'; var el = document.elementFromPoint(e.clientX, e.clientY); - host.style.pointerEvents = 'all'; + // Document hit-testing stops at a shadow host; closed roots stay opaque. + // Page code can shadow shadowRoot (e.g. a form control named "shadowRoot"); keep the deepest real hit. + try { + while (el && el.shadowRoot) { + var inner = el.shadowRoot.elementFromPoint(e.clientX, e.clientY); + if (!inner || inner === el) break; + el = inner; + } + } catch (err) { + } finally { + host.style.pointerEvents = 'all'; + } if (el) { requestAnimationFrame(function() { updateHighlight(el); }); } @@ -76,6 +87,7 @@ export const GRAB_GUEST_OVERLAY_SCRIPT = ` var host = document.createElement('d var previousSelectedValue = null; var frozenFrame = 0; var frozenResizeObserver = null; + var frozenShadowRoots = []; function refreshFrozenHighlight() { frozenFrame = 0; @@ -93,6 +105,10 @@ export const GRAB_GUEST_OVERLAY_SCRIPT = ` var host = document.createElement('d function stopTrackingSelection() { window.removeEventListener('scroll', scheduleFrozenHighlight, true); + frozenShadowRoots.forEach(function(root) { + root.removeEventListener('scroll', scheduleFrozenHighlight, true); + }); + frozenShadowRoots = []; window.removeEventListener('resize', scheduleFrozenHighlight, true); if (frozenResizeObserver) frozenResizeObserver.disconnect(); frozenResizeObserver = null; @@ -127,6 +143,10 @@ export const GRAB_GUEST_OVERLAY_SCRIPT = ` var host = document.createElement('d selectedEl.setAttribute(selectedAttribute, selectionId); highlightBox.style.transition = 'none'; window.addEventListener('scroll', scheduleFrozenHighlight, true); + frozenShadowRoots = shadowRootsAbove(selectedEl); + frozenShadowRoots.forEach(function(root) { + root.addEventListener('scroll', scheduleFrozenHighlight, true); + }); window.addEventListener('resize', scheduleFrozenHighlight, true); if (typeof ResizeObserver === 'function') { frozenResizeObserver = new ResizeObserver(scheduleFrozenHighlight); diff --git a/src/main/browser/grab-guest-react-script.ts b/src/main/browser/grab-guest-react-script.ts index 79d8eaab012..3790e034dd2 100644 --- a/src/main/browser/grab-guest-react-script.ts +++ b/src/main/browser/grab-guest-react-script.ts @@ -3,7 +3,8 @@ export const GRAB_GUEST_REACT_SCRIPT = ` function isElementFixed(el) { while (current && current !== document.body) { var position = window.getComputedStyle(current).position; if (position === 'fixed' || position === 'sticky') return true; - current = current.parentElement; + // Cross into the host so a fixed web component still counts for elements inside it. + current = composedParent(current); } return false; } diff --git a/src/main/browser/grab-guest-script.test.ts b/src/main/browser/grab-guest-script.test.ts index 00c760c5797..e85dbaf4bc6 100644 --- a/src/main/browser/grab-guest-script.test.ts +++ b/src/main/browser/grab-guest-script.test.ts @@ -7,7 +7,7 @@ import { clampGrabPayload } from './browser-grab-payload' describe('buildGuestOverlayScript', () => { it.each([ - ['arm', 'a721de2fc19173f5742f27a914019c5c0d5ea727f211369e22d5743bbb56c37e'], + ['arm', '6aeefde63c7ca2d90a75777f06042bf2694a2b0b5636d23d02c304c7ed739b23'], ['awaitClick', '4e7c30a9382cd18bf6286fbdbdfa70f7571cdddfc4914a9d9ffd4f9b0e430adb'], ['finalize', '91bd9836b0536c9579e0d4648d30679c0b4a5893d9a43110a70e67d6804fd291'], ['extractHover', 'cf0ee3ac61669daefa7db9389233c1abfe9f0fb9e7257300c761987aac914b02'], diff --git a/src/main/browser/grab-guest-script.ts b/src/main/browser/grab-guest-script.ts index cc291131420..0dd02a40305 100644 --- a/src/main/browser/grab-guest-script.ts +++ b/src/main/browser/grab-guest-script.ts @@ -1,3 +1,4 @@ +import { COMPOSED_TREE_SCRIPT } from '../../shared/browser-composed-tree-script' import { GRAB_GUEST_CONTENT_SCRIPT } from './grab-guest-content-script' import { GRAB_GUEST_ELEMENT_CONTEXT_SCRIPT } from './grab-guest-element-context-script' import { GRAB_GUEST_FOUNDATION_SCRIPT } from './grab-guest-foundation-script' @@ -30,6 +31,7 @@ export function buildGuestOverlayScript(action: GuestScriptAction): string { const ARM_SCRIPT = [ GRAB_GUEST_FOUNDATION_SCRIPT, + COMPOSED_TREE_SCRIPT, GRAB_GUEST_CONTENT_SCRIPT, GRAB_GUEST_ELEMENT_CONTEXT_SCRIPT, GRAB_GUEST_REACT_SCRIPT, diff --git a/src/shared/browser-annotation-viewport-bridge.ts b/src/shared/browser-annotation-viewport-bridge.ts index a1356f8647a..ee45c8fb854 100644 --- a/src/shared/browser-annotation-viewport-bridge.ts +++ b/src/shared/browser-annotation-viewport-bridge.ts @@ -1,4 +1,5 @@ import type { BrowserGrabRect } from './browser-grab-types' +import { COMPOSED_TREE_SCRIPT } from './browser-composed-tree-script' import { isValidBrowserGrabRect } from './browser-annotation-viewport-report' export type BrowserAnnotationViewportBridgeMarker = { @@ -83,6 +84,7 @@ export function buildBrowserAnnotationViewportBridgeScript({ const hostAttribute = 'data-orca-browser-annotation-overlay'; const selectedAttribute = ${JSON.stringify(BROWSER_GRAB_SELECTED_ATTRIBUTE)}; const markerSize = 24; +${COMPOSED_TREE_SCRIPT} const removeOverlay = (state) => { if (state && state.host && state.host.parentNode) { @@ -106,6 +108,7 @@ export function buildBrowserAnnotationViewportBridgeScript({ if (state.targetObserver) { state.targetObserver.disconnect(); } + state.targetShadowRoots.forEach((root) => root.removeEventListener('scroll', state.requestUpdate, true)); removeOverlay(state); }; @@ -124,23 +127,40 @@ export function buildBrowserAnnotationViewportBridgeScript({ scrollY: toNumber(window.scrollY, toNumber(window.pageYOffset, 0)) }); + const findMarked = (root) => + Array.from(root.querySelectorAll('[' + selectedAttribute + ']')).filter( + (candidate) => candidate.getAttribute(selectedAttribute) === selectionId + ); + + // Document queries stop at shadow roots, so open ones are walked when the light DOM has no match. + const findMarkedInShadowRoots = (root) => + Array.from(root.querySelectorAll('*')).flatMap((element) => + element.shadowRoot ? findMarked(element.shadowRoot).concat(findMarkedInShadowRoots(element.shadowRoot)) : [] + ); + // Why: the card must follow the element itself, not a click-time copy; the grab overlay // marks it in the shared DOM because this world cannot see the page's objects. const trackTarget = (state) => { const current = state.target; let next = null; if (emitViewportMessages && selectionId) { - for (const candidate of document.querySelectorAll('[' + selectedAttribute + ']')) { - if (candidate.getAttribute(selectedAttribute) !== selectionId) continue; - if (next) { - next = null; - break; + let found = findMarked(document); + if (found.length === 0) { + // The marker is set before the selection reaches this bridge, so one shadow walk per selection suffices. + const keep = current && current.isConnected && current.getAttribute(selectedAttribute) === selectionId; + if (keep) found = [current]; + else if (state.searchedSelectionId !== selectionId) { + state.searchedSelectionId = selectionId; + found = findMarkedInShadowRoots(document); } - next = candidate; } + next = found.length === 1 ? found[0] : null; } if (next !== current) { state.target = next; + state.targetShadowRoots.forEach((root) => root.removeEventListener('scroll', state.requestUpdate, true)); + state.targetShadowRoots = next ? shadowRootsAbove(next) : []; + state.targetShadowRoots.forEach((root) => root.addEventListener('scroll', state.requestUpdate, true)); if (state.targetObserver) { state.targetObserver.disconnect(); if (next) { @@ -272,7 +292,9 @@ export function buildBrowserAnnotationViewportBridgeScript({ requestUpdate: null, onBeforeUnload: null, target: null, - targetObserver: null + targetObserver: null, + targetShadowRoots: [], + searchedSelectionId: null }; state.requestUpdate = () => { diff --git a/src/shared/browser-annotation-viewport-live-target.test.ts b/src/shared/browser-annotation-viewport-live-target.test.ts index 72a7b0af5cd..e1681004142 100644 --- a/src/shared/browser-annotation-viewport-live-target.test.ts +++ b/src/shared/browser-annotation-viewport-live-target.test.ts @@ -19,16 +19,27 @@ afterEach(async () => { await Promise.all(guestAborts.splice(0).map((abort) => abort())) }) -async function createGuest(targetIsRoot = false) { +async function createGuest(placement: 'body' | 'root' | 'shadow' | 'slotted' = 'body') { const { Window } = await import('happy-dom') const window = new Window({ width: 1000, height: 800 }) guestAborts.push(() => window.happyDOM.abort()) const document = window.document const scroller = document.createElement('div') - const target = targetIsRoot ? document.documentElement : document.createElement('section') - if (!targetIsRoot) { + const target = placement === 'root' ? document.documentElement : document.createElement('section') + const shadowHost = document.createElement('div') + document.body.appendChild(shadowHost) + if (placement === 'body' || placement === 'shadow') { scroller.appendChild(target) - document.body.appendChild(scroller) + const container = + placement === 'shadow' ? shadowHost.attachShadow({ mode: 'open' }) : document.body + container.appendChild(scroller) + } else if (placement === 'slotted') { + // A light-DOM element slotted into a component's own scroll container. + const slot = scroller.appendChild(document.createElement('slot')) + shadowHost.attachShadow({ mode: 'open' }).appendChild(scroller) + shadowHost.appendChild(target) + // happy-dom has no slot assignment. + Object.defineProperty(target, 'assignedSlot', { value: slot }) } let targetRect: Rect = { x: 40, y: 300, width: 120, height: 30 } const contentSize = { width: 120, height: 30 } @@ -163,7 +174,7 @@ describe('annotation viewport bridge live target', () => { }) it('keeps a selected document root on outer-box observation', async () => { - const guest = await createGuest(true) + const guest = await createGuest('root') guest.target.setAttribute(BROWSER_GRAB_SELECTED_ATTRIBUTE, 'selection-1') guest.install(true) guest.flushFrames() @@ -208,6 +219,22 @@ describe('annotation viewport bridge live target', () => { expect(guest.lastReport()?.target).toEqual({ x: 40, y: 120, width: 120, height: 30 }) }) + it.each(['shadow', 'slotted'] as const)( + 'tracks a %s target through scroll inside the shadow root', + async (placement) => { + const guest = await createGuest(placement) + guest.target.setAttribute(BROWSER_GRAB_SELECTED_ATTRIBUTE, 'selection-1') + guest.install(true) + guest.flushFrames() + expect(guest.lastReport()?.target).toEqual({ x: 40, y: 300, width: 120, height: 30 }) + + guest.moveTarget({ x: 40, y: 120, width: 120, height: 30 }) + guest.scroller.dispatchEvent(new guest.window.Event('scroll')) + guest.flushFrames() + expect(guest.lastReport()?.target).toEqual({ x: 40, y: 120, width: 120, height: 30 }) + } + ) + it('ignores an earlier page-owned selection marker', async () => { const guest = await createGuest() const decoy = guest.window.document.createElement('div') diff --git a/src/shared/browser-composed-tree-script.ts b/src/shared/browser-composed-tree-script.ts new file mode 100644 index 00000000000..642b32a70a1 --- /dev/null +++ b/src/shared/browser-composed-tree-script.ts @@ -0,0 +1,20 @@ +// Why: page-named form controls (e.g.
) shadow DOM properties in the page's +// world, so each value is type-checked before the walk follows it. +export const COMPOSED_TREE_SCRIPT = ` function composedParent(element) { + var slot = element.assignedSlot; + if (slot && slot.localName === 'slot') return slot; + var parent = element.parentNode; + if (parent && parent.nodeType === 11) return parent.host || null; + return parent && parent.nodeType === 1 ? parent : null; + } + + // Scroll events don't leave a shadow root, so trackers listen on each open root above the target. + function shadowRootsAbove(element) { + var roots = []; + for (var node = element; node; node = composedParent(node)) { + var parent = node.parentNode; + if (parent && parent.nodeType === 11 && parent.host) roots.push(parent); + } + return roots; + } +`