orca/tests/e2e/plugin-startup-budget.spec.ts

// Alternate real launches with zero/twenty approved plugins; compare milestones and forbid worker entry before a trigger.

import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { expect, test, type TestInfo } from '@stablyai/playwright-test'
import { fingerprintPluginConsent } from '../../src/shared/plugins/plugin-consent-fingerprint'
import { pluginManifestSchema } from '../../src/shared/plugins/plugin-manifest'
import { createRestartSession } from './helpers/orca-restart'

const PLUGIN_COUNT = 20
const SAMPLE_COUNT = 3

type StartupSample = {
  readyToShowMs: number
  pluginDurationMs: number
  installedPlugins: number
}

function updateProfile(userDataDir: string, pluginConsents: Record<string, string>): void {
  const profilePath = join(userDataDir, 'orca-data.json')
  const profile = JSON.parse(readFileSync(profilePath, 'utf8')) as {
    settings?: Record<string, unknown>
  }
  profile.settings = {
    ...profile.settings,
    pluginSystemEnabled: true,
    pluginConsents,
    disabledPlugins: [],
    devPluginPaths: []
  }
  writeFileSync(profilePath, `${JSON.stringify(profile, null, 2)}\n`)
}

function seedPlugins(userDataDir: string, count: number): string[] {
  const pluginsDir = join(userDataDir, 'plugins')
  rmSync(pluginsDir, { recursive: true, force: true })
  mkdirSync(pluginsDir, { recursive: true })
  const pluginConsents: Record<string, string> = {}
  const markerPaths: string[] = []
  for (let index = 0; index < count; index += 1) {
    const manifest = pluginManifestSchema.parse({
      manifestVersion: 1,
      id: `startup-${index}`,
      publisher: 'budget',
      name: `Startup ${index}`,
      version: '1.0.0',
      engines: { orca: '>=1.0.0' },
      pluginApi: 1,
      main: 'main.mjs',
      contributes: { panels: [], commands: [], events: [] },
      capabilities: []
    })
    const pluginKey = `${manifest.publisher}.${manifest.id}`
    const contentHash = (index + 1).toString(16).padStart(64, '0')
    const versionDir = join(pluginsDir, pluginKey, contentHash)
    const markerPath = join(userDataDir, `plugin-startup-marker-${index}`)
    mkdirSync(versionDir, { recursive: true })
    writeFileSync(join(pluginsDir, pluginKey, 'current'), contentHash)
    writeFileSync(join(versionDir, 'orca-plugin.json'), JSON.stringify(manifest))
    writeFileSync(
      join(versionDir, 'main.mjs'),
      `import { writeFileSync } from 'node:fs'; writeFileSync(${JSON.stringify(markerPath)}, 'executed')`
    )
    pluginConsents[pluginKey] = fingerprintPluginConsent(manifest)
    markerPaths.push(markerPath)
  }
  updateProfile(userDataDir, pluginConsents)
  return markerPaths
}

function parseMetric(output: string, event: string, key: string): number | null {
  const line = output.split('\n').find((candidate) => candidate.startsWith(`[startup] ${event} `))
  const value = line?.match(new RegExp(`(?:^| )${key}=([0-9.]+)(?: |$)`))?.[1]
  return value === undefined ? null : Number(value)
}

async function launchSample(
  session: ReturnType<typeof createRestartSession>,
  expectedPlugins: number,
  testInfo: TestInfo
): Promise<StartupSample> {
  let output = ''
  const launched = await session.launch({
    onStderr: (chunk) => {
      output += chunk
    }
  })
  try {
    await expect
      .poll(
        () => ({
          ready: parseMetric(output, 'ready-to-show', 't'),
          duration: parseMetric(output, 'plugin-system-initialized', 'durationMs'),
          count: parseMetric(output, 'plugin-system-initialized', 'installedPlugins')
        }),
        { timeout: 30_000 }
      )
      .toMatchObject({
        ready: expect.any(Number),
        duration: expect.any(Number),
        count: expectedPlugins
      })
    await testInfo.attach(`plugin-startup-${expectedPlugins}-${Date.now()}.log`, {
      body: Buffer.from(output),
      contentType: 'text/plain'
    })
    return {
      readyToShowMs: parseMetric(output, 'ready-to-show', 't')!,
      pluginDurationMs: parseMetric(output, 'plugin-system-initialized', 'durationMs')!,
      installedPlugins: parseMetric(output, 'plugin-system-initialized', 'installedPlugins')!
    }
  } finally {
    await session.close(launched.app)
  }
}

function median(values: readonly number[]): number {
  const sorted = [...values].sort((left, right) => left - right)
  return sorted[Math.floor(sorted.length / 2)]!
}

// oxlint-disable-next-line no-empty-pattern -- Playwright passes fixtures before testInfo.
test('keeps real Electron launch stable with 20 approved inert plugins', async ({}, testInfo) => {
  test.setTimeout(240_000)
  const baseline: StartupSample[] = []
  const populated: StartupSample[] = []
  for (let sample = 0; sample < SAMPLE_COUNT; sample += 1) {
    for (const count of [0, PLUGIN_COUNT]) {
      // Seed each profile before its first migration into authoritative SQLite state.
      const session = createRestartSession(testInfo, { ORCA_STARTUP_DIAGNOSTICS: '1' })
      try {
        const markerPaths = seedPlugins(session.userDataDir, count)
        const samples = count === 0 ? baseline : populated
        samples.push(await launchSample(session, count, testInfo))
        expect(markerPaths.every((markerPath) => !existsSync(markerPath))).toBe(true)
      } finally {
        await session.dispose()
      }
    }
  }

  // The isolated unit gate owns ≤50ms P95; this app comparison includes overlapping startup/discovery work.
  expect(populated.every((sample) => Number.isFinite(sample.pluginDurationMs))).toBe(true)
  expect(median(populated.map((sample) => sample.readyToShowMs))).toBeLessThanOrEqual(
    median(baseline.map((sample) => sample.readyToShowMs)) + 50
  )
})