From ffa0892ef87d927eef78fe8068de4d90201fb20b Mon Sep 17 00:00:00 2001 From: Ogulcan Celik Date: Tue, 8 Sep 2026 15:58:14 +0300 Subject: [PATCH] chore: move marketplace worker to website repository --- justfile | 6 - workers/plugin-marketplace/bun.lock | 15 - workers/plugin-marketplace/package.json | 13 - workers/plugin-marketplace/r2-cors.json | 13 - workers/plugin-marketplace/src/index.test.ts | 924 ------------- workers/plugin-marketplace/src/index.ts | 1269 ------------------ workers/plugin-marketplace/wrangler.toml | 22 - 7 files changed, 2262 deletions(-) delete mode 100644 workers/plugin-marketplace/bun.lock delete mode 100644 workers/plugin-marketplace/package.json delete mode 100644 workers/plugin-marketplace/r2-cors.json delete mode 100644 workers/plugin-marketplace/src/index.test.ts delete mode 100644 workers/plugin-marketplace/src/index.ts delete mode 100644 workers/plugin-marketplace/wrangler.toml diff --git a/justfile b/justfile index f69bb38c..57f5eaba 100644 --- a/justfile +++ b/justfile @@ -9,7 +9,6 @@ test: just maintenance-test just ui-hot-path-architecture-test just integration-assets-test - just plugin-marketplace-test just docs-contract-test # Run repository maintenance contract tests @@ -41,7 +40,6 @@ ci filter='all()': lint just maintenance-test just ui-hot-path-architecture-test just integration-assets-test - just plugin-marketplace-test # Run Windows target lint from Unix/macOS to catch cfg(windows) compile and clippy failures before CI [unix] @@ -90,10 +88,6 @@ integration-assets-test: bun test src/integration/assets/opencode/herdr-agent-state.test.ts bun test src/integration/assets/opencode/herdr-tui-session.test.ts -# Run plugin marketplace Worker tests -plugin-marketplace-test: - cd workers/plugin-marketplace && bun install --frozen-lockfile && bun test - # Build the vendored libghostty-vt source dist build-libghostty-vt: scripts/build_vendored_libghostty_vt.sh diff --git a/workers/plugin-marketplace/bun.lock b/workers/plugin-marketplace/bun.lock deleted file mode 100644 index 93e6ac52..00000000 --- a/workers/plugin-marketplace/bun.lock +++ /dev/null @@ -1,15 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "herdr-plugin-marketplace-worker", - "dependencies": { - "smol-toml": "1.7.1", - }, - }, - }, - "packages": { - "smol-toml": ["smol-toml@1.7.1", "", {}, "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ=="], - } -} diff --git a/workers/plugin-marketplace/package.json b/workers/plugin-marketplace/package.json deleted file mode 100644 index a22f3179..00000000 --- a/workers/plugin-marketplace/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "herdr-plugin-marketplace-worker", - "private": true, - "type": "module", - "scripts": { - "dev": "bunx wrangler dev", - "deploy": "bunx wrangler deploy", - "test": "bun test" - }, - "dependencies": { - "smol-toml": "1.7.1" - } -} diff --git a/workers/plugin-marketplace/r2-cors.json b/workers/plugin-marketplace/r2-cors.json deleted file mode 100644 index a8c8d99f..00000000 --- a/workers/plugin-marketplace/r2-cors.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "rules": [ - { - "allowed": { - "origins": ["https://herdr.dev"], - "methods": ["GET", "HEAD"], - "headers": ["*"] - }, - "exposeHeaders": ["Content-Type", "Cache-Control", "ETag"], - "maxAgeSeconds": 3600 - } - ] -} diff --git a/workers/plugin-marketplace/src/index.test.ts b/workers/plugin-marketplace/src/index.test.ts deleted file mode 100644 index 6dacbc91..00000000 --- a/workers/plugin-marketplace/src/index.test.ts +++ /dev/null @@ -1,924 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import worker, { - normalizeRepositories, - parseManifestSummary, - refreshPlugins, - type Env, -} from "./index"; - -const HEAD_COMMIT = "a".repeat(40); -const SECOND_COMMIT = "b".repeat(40); - -type TreeFixture = { - path: string; - content?: string; - mode?: string; - type?: string; - size?: number; -}; - -class MemoryR2 { - objects = new Map(); - putCount = 0; - failPuts = false; - async put(key: string, value: string, options?: unknown): Promise { - if (this.failPuts) throw new Error("simulated R2 put failure"); - this.putCount += 1; - this.objects.set(key, { value, options }); - } - - async get(key: string): Promise<{ text(): Promise } | null> { - const object = this.objects.get(key); - return object - ? { - async text() { - return object.value; - }, - } - : null; - } -} - -class MemoryKV { - constructor(private readonly keyNames: string[]) {} - - async list(options?: { prefix?: string }): Promise<{ keys: Array<{ name: string }> }> { - return { - keys: this.keyNames - .filter((name) => !options?.prefix || name.startsWith(options.prefix)) - .map((name) => ({ name })), - }; - } -} - -function repo(overrides: Record = {}): Record { - return { - id: 1, - full_name: "ogulcancelik/herdr-plugin-example", - owner: { login: "ogulcancelik" }, - name: "herdr-plugin-example", - description: "Example plugin repository", - html_url: "https://github.com/ogulcancelik/herdr-plugin-example", - default_branch: "main", - stargazers_count: 5, - forks_count: 1, - open_issues_count: 0, - language: "TypeScript", - topics: ["herdr-plugin"], - created_at: "2026-06-01T00:00:00Z", - updated_at: "2026-06-02T00:00:00Z", - pushed_at: "2026-06-03T00:00:00Z", - archived: false, - fork: false, - disabled: false, - private: false, - visibility: "public", - ...overrides, - }; -} - -function manifest(overrides = ""): string { - return ` -id = "example.plugin" -name = "Example Plugin" -version = "0.2.0" -min_herdr_version = "0.7.0" -description = "Example manifest" -platforms = ["linux", "macos"] -${overrides}`; -} - -function env(bucket = new MemoryR2(), blacklist?: MemoryKV, backupBucket = new MemoryR2()): Env { - return { - PLUGIN_MARKETPLACE_BUCKET: bucket, - PLUGIN_MARKETPLACE_BACKUP_BUCKET: backupBucket, - PLUGIN_MARKETPLACE_BLACKLIST: blacklist, - GITHUB_TOKEN: "token", - }; -} - -function repositoryFetch(options: { - repositories: Record[]; - trees?: Record; - commits?: Record; - totalCount?: number; - incompleteResults?: boolean; - searchStatus?: number; - treeStatus?: Record; - truncatedTrees?: Set; - onRequest?: (kind: "search" | "head" | "tree" | "manifest", detail: string) => void; -}): typeof fetch { - return (async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const url = new URL(input.toString()); - if (url.pathname === "/search/repositories") { - options.onRequest?.("search", url.toString()); - if (options.searchStatus) return new Response("search failed", { status: options.searchStatus }); - return Response.json({ - total_count: options.totalCount ?? options.repositories.length, - incomplete_results: options.incompleteResults ?? false, - items: options.repositories, - }); - } - - const treeMatch = url.pathname.match(/^\/repos\/([^/]+)\/([^/]+)\/git\/trees\/([^/]+)$/); - if (treeMatch) { - const fullName = `${decodeURIComponent(treeMatch[1])}/${decodeURIComponent(treeMatch[2])}`; - options.onRequest?.("tree", fullName); - const status = options.treeStatus?.[fullName]; - if (status) return new Response("tree failed", { status }); - const fixtures = options.trees?.[fullName] ?? [ - { path: "herdr-plugin.toml", content: manifest() }, - ]; - return Response.json({ - sha: "tree-sha", - truncated: options.truncatedTrees?.has(fullName) ?? false, - tree: fixtures.map((fixture) => ({ - path: fixture.path, - mode: fixture.mode ?? "100644", - type: fixture.type ?? "blob", - size: - fixture.size ?? - new TextEncoder().encode(fixture.content ?? "").byteLength, - })), - }); - } - - if (url.pathname === "/graphql") { - const request = JSON.parse(String(init?.body ?? "{}")); - const query = String(request.query ?? ""); - const data: Record = {}; - if (query.includes("PluginMarketplaceHeads")) { - options.onRequest?.("head", query); - for (const match of query.matchAll( - /repo(\d+): repository\(owner: "([^"]+)", name: "([^"]+)"\)/g, - )) { - const [, alias, owner, name] = match; - const fullName = `${owner}/${name}`; - const repository = options.repositories.find( - (candidate) => candidate.full_name === fullName, - ); - data[`repo${alias}`] = repository - ? { - defaultBranchRef: { - name: repository.default_branch ?? "main", - target: { oid: options.commits?.[fullName] ?? HEAD_COMMIT }, - }, - } - : null; - } - } else if (query.includes("PluginMarketplaceManifests")) { - options.onRequest?.("manifest", query); - for (const match of query.matchAll( - /item(\d+): repository\(owner: "([^"]+)", name: "([^"]+)"\) \{\s+manifest: object\(expression: "[a-f0-9]+:([^"]+)"\)/g, - )) { - const [, alias, owner, name, path] = match; - const fullName = `${owner}/${name}`; - const fixture = (options.trees?.[fullName] ?? [ - { path: "herdr-plugin.toml", content: manifest() }, - ]).find((entry) => entry.path === path); - data[`item${alias}`] = fixture?.content === undefined - ? { manifest: null } - : { manifest: { text: fixture.content } }; - } - } else { - throw new Error(`unexpected GraphQL query: ${query}`); - } - return Response.json({ data }); - } - - throw new Error(`unexpected request: ${url}`); - }) as typeof fetch; -} - -describe("normalizeRepositories", () => { - test("normalizes fields while preserving repository-card ordering", () => { - const plugins = normalizeRepositories([ - repo({ - id: 2, - full_name: "other/newer", - owner: { login: "other" }, - name: "newer", - html_url: "https://github.com/other/newer", - stargazers_count: 5, - pushed_at: "2026-06-04T00:00:00Z", - }), - repo(), - ]); - - expect(plugins.map((plugin) => plugin.fullName)).toEqual([ - "other/newer", - "ogulcancelik/herdr-plugin-example", - ]); - expect(plugins[1]).toMatchObject({ - id: 1, - owner: "ogulcancelik", - name: "herdr-plugin-example", - defaultBranch: "main", - stars: 5, - }); - }); - - test("deduplicates repositories by immutable GitHub id", () => { - const plugins = normalizeRepositories([ - repo(), - repo({ full_name: "duplicate/name", owner: { login: "duplicate" }, name: "name", html_url: "https://github.com/duplicate/name" }), - ]); - expect(plugins).toHaveLength(1); - expect(plugins[0].fullName).toBe("ogulcancelik/herdr-plugin-example"); - }); - - test("drops unsafe, unavailable, and default-branch-less repositories", () => { - const plugins = normalizeRepositories([ - repo({ html_url: "https://example.com/ogulcancelik/herdr-plugin-example" }), - repo({ archived: true }), - repo({ fork: true }), - repo({ disabled: true }), - repo({ private: true }), - repo({ visibility: "private" }), - repo({ default_branch: undefined }), - repo({ id: 5 }), - ]); - expect(plugins.map((plugin) => plugin.id)).toEqual([5]); - }); -}); - -describe("parseManifestSummary", () => { - test("extracts metadata and accepts the UTF-8 BOM accepted by Herdr", () => { - expect(parseManifestSummary(`\uFEFF${manifest()}`)).toEqual({ - id: "example.plugin", - name: "Example Plugin", - version: "0.2.0", - minHerdrVersion: "0.7.0", - description: "Example manifest", - platforms: ["linux", "macos"], - }); - }); - - test("rejects malformed TOML and invalid required metadata", () => { - expect(parseManifestSummary("not = [valid")).toBeNull(); - expect(parseManifestSummary(manifest().replace("example.plugin", "bad/plugin"))).toBeNull(); - expect(parseManifestSummary(manifest().replace("0.7.0", "next"))).toBeNull(); - expect(parseManifestSummary(manifest().replace('["linux", "macos"]', "[]"))).toBeNull(); - expect( - parseManifestSummary(manifest().replace("Example Plugin", "n".repeat(121))), - ).toBeNull(); - }); -}); - -describe("refreshPlugins", () => { - test("publishes one backward-compatible repository card with multiple manifests", async () => { - const bucket = new MemoryR2(); - const fullName = "ogulcancelik/herdr-plugin-example"; - const fetch = repositoryFetch({ - repositories: [repo()], - trees: { - [fullName]: [ - { path: "herdr-plugin.toml", content: manifest() }, - { - path: "plugins/second/herdr-plugin.toml", - content: manifest().replace("example.plugin", "example.second").replace("Example Plugin", "Second Plugin"), - }, - ], - }, - }); - - const result = await refreshPlugins(env(bucket), { - fetch, - now: new Date("2026-06-20T12:00:00.000Z"), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - const snapshotObject = bucket.objects.get("plugins/index.json"); - const snapshot = JSON.parse(snapshotObject?.value ?? ""); - expect(snapshotObject?.options).toEqual({ - httpMetadata: { - contentType: "application/json; charset=utf-8", - cacheControl: "public, max-age=300, s-maxage=1800, stale-while-revalidate=3600", - }, - }); - expect(snapshot).toMatchObject({ - schemaVersion: 1, - generatedAt: "2026-06-20T12:00:00.000Z", - pluginCount: 2, - repositoryCount: 1, - source: { - missingManifestCount: 0, - invalidManifestCount: 0, - }, - }); - expect(snapshot.plugins[0]).toMatchObject({ - id: 1, - fullName, - name: "herdr-plugin-example", - headCommit: HEAD_COMMIT, - manifests: [ - { path: "herdr-plugin.toml", id: "example.plugin" }, - { path: "plugins/second/herdr-plugin.toml", id: "example.second" }, - ], - }); - expect(snapshot.plugins[0]).not.toHaveProperty("defaultBranch"); - expect(snapshot.plugins[0]).toMatchObject({ - firstSeenAt: "2026-06-20T12:00:00.000Z", - starsDelta7d: null, - starsDelta30d: null, - }); - expect(bucket.objects.has("plugins/scan-cache.json")).toBe(true); - const history = JSON.parse(bucket.objects.get("plugins/star-history.json")?.value ?? ""); - expect(history).toEqual({ - schemaVersion: 1, - entries: [ - { - repositoryId: 1, - fullName, - firstSeenAt: "2026-06-20T12:00:00.000Z", - samples: [{ date: "2026-06-20", stars: 5 }], - }, - ], - }); - }); - - test("reuses cached manifests without resolving or rescanning an unchanged repository", async () => { - const bucket = new MemoryR2(); - const requests: string[] = []; - const fetch = repositoryFetch({ - repositories: [repo()], - onRequest(kind) { - requests.push(kind); - }, - }); - - expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true); - requests.length = 0; - expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true); - - expect(requests).toEqual(["search"]); - }); - - test("rescans a repository when its pushed timestamp and head change", async () => { - const bucket = new MemoryR2(); - const repository = repo(); - const fullName = String(repository.full_name); - const commits = { [fullName]: HEAD_COMMIT }; - const trees = { [fullName]: [{ path: "herdr-plugin.toml", content: manifest() }] }; - const fetch = repositoryFetch({ repositories: [repository], commits, trees }); - - expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true); - commits[fullName] = SECOND_COMMIT; - repository.pushed_at = "2026-06-04T00:00:00Z"; - trees[fullName][0].content = manifest().replace("0.2.0", "0.3.0"); - - const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } }); - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.plugins[0].headCommit).toBe(SECOND_COMMIT); - expect(result.snapshot.plugins[0].manifests[0].version).toBe("0.3.0"); - }); - - test("counts manifests separately from repository cards and omits empty cards", async () => { - const repositories = [ - repo(), - repo({ - id: 2, - full_name: "example/empty", - owner: { login: "example" }, - name: "empty", - html_url: "https://github.com/example/empty", - }), - repo({ - id: 3, - full_name: "example/invalid", - owner: { login: "example" }, - name: "invalid", - html_url: "https://github.com/example/invalid", - }), - ]; - const result = await refreshPlugins(env(), { - fetch: repositoryFetch({ - repositories, - trees: { - "ogulcancelik/herdr-plugin-example": [ - { path: "one/herdr-plugin.toml", content: manifest() }, - { path: "two/herdr-plugin.toml", content: manifest().replace("example.plugin", "example.two") }, - ], - "example/empty": [{ path: "README.md", content: "empty" }], - "example/invalid": [{ path: "herdr-plugin.toml", content: "id = [broken" }], - }, - }), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.pluginCount).toBe(2); - expect(result.snapshot.repositoryCount).toBe(1); - expect(result.snapshot.source.missingManifestCount).toBe(1); - expect(result.snapshot.source.invalidManifestCount).toBe(1); - }); - - test("ignores manifest symlinks and indexes their regular target", async () => { - const fullName = "ogulcancelik/herdr-plugin-example"; - const result = await refreshPlugins(env(), { - fetch: repositoryFetch({ - repositories: [repo()], - trees: { - [fullName]: [ - { path: "herdr-plugin.toml", content: "plugin/herdr-plugin.toml", mode: "120000" }, - { path: "plugin/herdr-plugin.toml", content: manifest() }, - ], - }, - }), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.plugins[0].manifests.map((item) => item.path)).toEqual([ - "plugin/herdr-plugin.toml", - ]); - expect(result.snapshot.source.invalidManifestCount).toBe(0); - }); - - test("ignores test fixtures and deduplicates repeated plugin ids", async () => { - const fullName = "ogulcancelik/herdr-plugin-example"; - const result = await refreshPlugins(env(), { - fetch: repositoryFetch({ - repositories: [repo()], - trees: { - [fullName]: [ - { path: "herdr-plugin.toml", content: manifest() }, - { path: "platform/herdr-plugin.toml", content: manifest() }, - { - path: "tests/fixtures/helper/herdr-plugin.toml", - content: manifest().replace("example.plugin", "example.fixture"), - }, - ], - }, - }), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.plugins[0].manifests.map((item) => item.path)).toEqual([ - "herdr-plugin.toml", - ]); - expect(result.snapshot.source.duplicateManifestCount).toBe(1); - }); - - test("skips a truncated tree without blocking the marketplace", async () => { - const fullName = "ogulcancelik/herdr-plugin-example"; - const result = await refreshPlugins(env(), { - fetch: repositoryFetch({ - repositories: [repo()], - truncatedTrees: new Set([fullName]), - }), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.plugins).toEqual([]); - expect(result.snapshot.source.skippedRepositoryCount).toBe(1); - expect(result.snapshot.source.warnings?.[0]).toContain(fullName); - }); - - test("writes an empty snapshot when every repository is blacklisted", async () => { - const bucket = new MemoryR2(); - const result = await refreshPlugins( - env(bucket, new MemoryKV(["repo:ogulcancelik/herdr-plugin-example"])), - { fetch: repositoryFetch({ repositories: [repo()] }), logger: { error() {} } }, - ); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.plugins).toEqual([]); - expect(result.snapshot.source.blacklistedCount).toBe(1); - }); - - test("pins tree and manifest reads to the resolved commit", async () => { - const requests: Array<[string, string]> = []; - const result = await refreshPlugins(env(), { - fetch: repositoryFetch({ - repositories: [repo()], - onRequest(kind, detail) { - requests.push([kind, detail]); - }, - }), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - expect(requests.find(([kind]) => kind === "tree")?.[1]).toBe( - "ogulcancelik/herdr-plugin-example", - ); - const manifestQuery = requests.find(([kind]) => kind === "manifest")?.[1] ?? ""; - expect(manifestQuery).toContain(`${HEAD_COMMIT}:herdr-plugin.toml`); - expect(manifestQuery).not.toContain("main:herdr-plugin.toml"); - }); - - test("does not overwrite the public snapshot when scanning fails", async () => { - const bucket = new MemoryR2(); - await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); - const result = await refreshPlugins(env(bucket), { - fetch: repositoryFetch({ - repositories: [repo()], - treeStatus: { "ogulcancelik/herdr-plugin-example": 429 }, - }), - logger: { error() {} }, - }); - - expect(result.ok).toBe(false); - expect(bucket.objects.get("plugins/index.json")?.value).toBe( - '{"schemaVersion":1,"plugins":[{"id":1}]}', - ); - }); - - test("discards malformed cache state and rebuilds it", async () => { - const bucket = new MemoryR2(); - await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); - await bucket.put("plugins/scan-cache.json", "broken"); - const errors: string[] = []; - - const result = await refreshPlugins(env(bucket), { - fetch: repositoryFetch({ repositories: [repo()] }), - logger: { error(message) { errors.push(String(message)); } }, - }); - - expect(result.ok).toBe(true); - expect(errors[0]).toContain("discarding invalid plugin marketplace scan cache"); - expect(JSON.parse(bucket.objects.get("plugins/scan-cache.json")?.value ?? "").entries).toHaveLength(1); - }); - - test("rejects an implausible empty search when a healthy cache exists", async () => { - const bucket = new MemoryR2(); - const normalFetch = repositoryFetch({ repositories: [repo()] }); - expect((await refreshPlugins(env(bucket), { fetch: normalFetch, logger: { error() {} } })).ok).toBe(true); - const previous = bucket.objects.get("plugins/index.json")?.value; - - const result = await refreshPlugins(env(bucket), { - fetch: repositoryFetch({ repositories: [] }), - logger: { error() {} }, - }); - - expect(result.ok).toBe(false); - expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous); - }); - - test("publishes an empty snapshot when the initial complete search is empty", async () => { - const bucket = new MemoryR2(); - const result = await refreshPlugins(env(bucket), { - fetch: repositoryFetch({ repositories: [] }), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.plugins).toEqual([]); - expect(result.snapshot.pluginCount).toBe(0); - }); - - test("preserves the snapshot when head resolution omits a cached repository", async () => { - const bucket = new MemoryR2(); - const repository = repo(); - const baseFetch = repositoryFetch({ repositories: [repository] }); - expect((await refreshPlugins(env(bucket), { fetch: baseFetch, logger: { error() {} } })).ok).toBe(true); - const previous = bucket.objects.get("plugins/index.json")?.value; - - repository.pushed_at = "2026-06-04T00:00:00Z"; - const missingHeadFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const request = String(init?.body ?? ""); - if (new URL(input.toString()).pathname === "/graphql" && request.includes("PluginMarketplaceHeads")) { - return Response.json({ - data: { repo0: null }, - errors: [{ type: "NOT_FOUND", path: ["repo0"], message: "not found" }], - }); - } - return baseFetch(input, init); - }; - const result = await refreshPlugins(env(bucket), { - fetch: missingHeadFetch as typeof fetch, - logger: { error() {} }, - }); - - expect(result.ok).toBe(false); - expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous); - }); - - test("does not cache a transiently missing manifest response", async () => { - const bucket = new MemoryR2(); - const repository = repo(); - const fullName = String(repository.full_name); - const commits = { [fullName]: HEAD_COMMIT }; - const baseFetch = repositoryFetch({ repositories: [repository], commits }); - expect((await refreshPlugins(env(bucket), { fetch: baseFetch, logger: { error() {} } })).ok).toBe(true); - const previous = bucket.objects.get("plugins/index.json")?.value; - - repository.pushed_at = "2026-06-04T00:00:00Z"; - commits[fullName] = SECOND_COMMIT; - const missingFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const request = String(init?.body ?? ""); - if (new URL(input.toString()).pathname === "/graphql" && request.includes("PluginMarketplaceManifests")) { - return Response.json({ - data: { item0: { manifest: null } }, - errors: [{ type: "NOT_FOUND", path: ["item0"], message: "not found" }], - }); - } - return baseFetch(input, init); - }; - const result = await refreshPlugins(env(bucket), { - fetch: missingFetch as typeof fetch, - logger: { error() {} }, - }); - - expect(result.ok).toBe(false); - expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous); - }); - - test("retries budget-skipped repositories when capacity becomes available", async () => { - const repositories = Array.from({ length: 51 }, (_, index) => - repo({ - id: index + 1, - full_name: `owner/repo-${index}`, - owner: { login: "owner" }, - name: `repo-${index}`, - html_url: `https://github.com/owner/repo-${index}`, - }), - ); - const trees = Object.fromEntries( - repositories.map((repository) => [ - repository.full_name, - Array.from({ length: 100 }, (_, index) => ({ - path: `plugins/${index}/herdr-plugin.toml`, - content: manifest(), - })), - ]), - ); - - const bucket = new MemoryR2(); - const fetch = repositoryFetch({ repositories, trees }); - const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.source.skippedRepositoryCount).toBe(1); - expect(result.snapshot.source.warnings?.[0]).toContain("scan budget was exhausted"); - - repositories.shift(); - const recovered = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } }); - expect(recovered.ok).toBe(true); - if (!recovered.ok) return; - expect(recovered.snapshot.source.skippedRepositoryCount).toBe(0); - expect(recovered.snapshot.repositoryCount).toBe(50); - }); - - test("keeps one star sample per UTC day with the first observation winning", async () => { - const bucket = new MemoryR2(); - const backupBucket = new MemoryR2(); - const repository = repo({ stargazers_count: 5 }); - const logger = { error() {} }; - const run = (stars: number, now: string) => { - repository.stargazers_count = stars; - return refreshPlugins(env(bucket, undefined, backupBucket), { - fetch: repositoryFetch({ repositories: [repository] }), - now: new Date(now), - logger, - }); - }; - - expect((await run(5, "2026-06-20T00:10:00.000Z")).ok).toBe(true); - expect((await run(9, "2026-06-20T23:50:00.000Z")).ok).toBe(true); - const sameDay = JSON.parse(bucket.objects.get("plugins/star-history.json")?.value ?? ""); - expect(sameDay.entries[0].samples).toEqual([{ date: "2026-06-20", stars: 5 }]); - - const nextDay = await run(12, "2026-06-21T00:10:00.000Z"); - expect(nextDay.ok).toBe(true); - if (!nextDay.ok) return; - expect(nextDay.snapshot.plugins[0].firstSeenAt).toBe("2026-06-20T00:10:00.000Z"); - const history = JSON.parse(bucket.objects.get("plugins/star-history.json")?.value ?? ""); - expect(history.entries[0].samples).toEqual([ - { date: "2026-06-20", stars: 5 }, - { date: "2026-06-21", stars: 12 }, - ]); - - // One dated backup per UTC day, written on the first run of that day and - // never touched by later intraday runs. - expect(backupBucket.putCount).toBe(2); - expect([...backupBucket.objects.keys()]).toEqual([ - "backups/star-history/2026-06-20.json", - "backups/star-history/2026-06-21.json", - ]); - const firstBackup = JSON.parse( - backupBucket.objects.get("backups/star-history/2026-06-20.json")?.value ?? "", - ); - expect(firstBackup.entries[0].samples).toEqual([{ date: "2026-06-20", stars: 5 }]); - }); - - test("fails the refresh without touching the primary bucket when the backup put fails", async () => { - const bucket = new MemoryR2(); - await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); - const backupBucket = new MemoryR2(); - backupBucket.failPuts = true; - const errors: string[] = []; - const options = { - fetch: repositoryFetch({ repositories: [repo()] }), - now: new Date("2026-06-20T12:00:00.000Z"), - logger: { error(message: unknown) { errors.push(String(message)); } }, - }; - - const failed = await refreshPlugins(env(bucket, undefined, backupBucket), options); - expect(failed.ok).toBe(false); - expect(errors[0]).toContain("simulated R2 put failure"); - expect(bucket.objects.get("plugins/index.json")?.value).toBe( - '{"schemaVersion":1,"plugins":[{"id":1}]}', - ); - expect(bucket.objects.has("plugins/star-history.json")).toBe(false); - expect(bucket.objects.has("plugins/scan-cache.json")).toBe(false); - - backupBucket.failPuts = false; - const retried = await refreshPlugins(env(bucket, undefined, backupBucket), options); - expect(retried.ok).toBe(true); - expect(backupBucket.objects.has("backups/star-history/2026-06-20.json")).toBe(true); - expect(bucket.objects.has("plugins/star-history.json")).toBe(true); - }); - - test("computes star deltas from near-boundary baselines and rejects stale ones", async () => { - const bucket = new MemoryR2(); - await bucket.put( - "plugins/star-history.json", - JSON.stringify({ - schemaVersion: 1, - entries: [ - { - repositoryId: 1, - fullName: "ogulcancelik/herdr-plugin-example", - firstSeenAt: "2026-05-01T00:00:00.000Z", - samples: [ - { date: "2026-05-01", stars: 10 }, - { date: "2026-05-20", stars: 30 }, - { date: "2026-06-13", stars: 40 }, - { date: "2026-06-19", stars: 90 }, - ], - }, - { - repositoryId: 2, - fullName: "example/rejoined", - firstSeenAt: "2026-05-01T00:00:00.000Z", - samples: [{ date: "2026-05-01", stars: 3 }], - }, - ], - }), - ); - - const result = await refreshPlugins(env(bucket), { - fetch: repositoryFetch({ - repositories: [ - repo({ stargazers_count: 100 }), - repo({ - id: 2, - full_name: "example/rejoined", - owner: { login: "example" }, - name: "rejoined", - html_url: "https://github.com/example/rejoined", - stargazers_count: 50, - }), - ], - }), - now: new Date("2026-06-20T12:00:00.000Z"), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.snapshot.plugins[0]).toMatchObject({ - firstSeenAt: "2026-05-01T00:00:00.000Z", - starsDelta7d: 60, - starsDelta30d: 70, - }); - // The rejoined repository only has a six-week-old sample: far past both - // window boundaries, so no delta is reported instead of an inflated one. - expect(result.snapshot.plugins[1]).toMatchObject({ - fullName: "example/rejoined", - starsDelta7d: null, - starsDelta30d: null, - }); - }); - - test("prunes aged samples and keeps history for delisted repositories", async () => { - const bucket = new MemoryR2(); - await bucket.put( - "plugins/star-history.json", - JSON.stringify({ - schemaVersion: 1, - entries: [ - { - repositoryId: 1, - fullName: "ogulcancelik/herdr-plugin-example", - firstSeenAt: "2026-01-01T00:00:00.000Z", - samples: [ - { date: "2026-01-01", stars: 1 }, - { date: "2026-06-19", stars: 4 }, - ], - }, - { - repositoryId: 2, - fullName: "example/delisted-recently", - firstSeenAt: "2026-01-01T00:00:00.000Z", - samples: [ - { date: "2026-01-05", stars: 5 }, - { date: "2026-06-01", stars: 7 }, - ], - }, - { - repositoryId: 3, - fullName: "example/delisted-long-ago", - firstSeenAt: "2026-01-01T00:00:00.000Z", - samples: [{ date: "2026-01-05", stars: 3 }], - }, - ], - }), - ); - - const result = await refreshPlugins(env(bucket), { - fetch: repositoryFetch({ repositories: [repo()] }), - now: new Date("2026-06-20T12:00:00.000Z"), - logger: { error() {} }, - }); - - expect(result.ok).toBe(true); - const history = JSON.parse(bucket.objects.get("plugins/star-history.json")?.value ?? ""); - expect(history.entries.map((entry: { repositoryId: number }) => entry.repositoryId)).toEqual([ - 1, 2, - ]); - expect(history.entries[0].samples.map((sample: { date: string }) => sample.date)).toEqual([ - "2026-06-19", - "2026-06-20", - ]); - expect(history.entries[1].samples).toEqual([{ date: "2026-06-01", stars: 7 }]); - }); - - for (const { name, record } of [ - { name: "malformed JSON", record: "broken" }, - { name: "an unsupported shape", record: '{"schemaVersion":2,"entries":[]}' }, - { - name: "an invalid entry", - record: JSON.stringify({ - schemaVersion: 1, - entries: [{ repositoryId: -5, samples: "broken" }], - }), - }, - ]) { - test(`fails the refresh without overwriting anything when star history has ${name}`, async () => { - const bucket = new MemoryR2(); - await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); - await bucket.put("plugins/star-history.json", record); - const backupBucket = new MemoryR2(); - const errors: string[] = []; - - const result = await refreshPlugins(env(bucket, undefined, backupBucket), { - fetch: repositoryFetch({ repositories: [repo()] }), - now: new Date("2026-06-20T12:00:00.000Z"), - logger: { error(message: unknown) { errors.push(String(message)); } }, - }); - - expect(result.ok).toBe(false); - expect(errors[0]).toContain("invalid plugin marketplace star history"); - expect(bucket.objects.get("plugins/star-history.json")?.value).toBe(record); - expect(bucket.objects.get("plugins/index.json")?.value).toBe( - '{"schemaVersion":1,"plugins":[{"id":1}]}', - ); - expect(bucket.objects.has("plugins/scan-cache.json")).toBe(false); - // Broken history must never be preserved as a "backup". - expect(backupBucket.objects.size).toBe(0); - }); - } - - for (const { name, fetch } of [ - { - name: "GitHub failure", - fetch: repositoryFetch({ repositories: [], searchStatus: 429 }), - }, - { - name: "incomplete search results", - fetch: repositoryFetch({ repositories: [repo()], incompleteResults: true }), - }, - ]) { - test(`preserves the current snapshot on ${name}`, async () => { - const bucket = new MemoryR2(); - await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); - const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } }); - expect(result.ok).toBe(false); - expect(bucket.objects.get("plugins/index.json")?.value).toBe( - '{"schemaVersion":1,"plugins":[{"id":1}]}', - ); - }); - } -}); - -describe("fetch handler", () => { - test("does not expose a public Worker API", async () => { - const response = await worker.fetch(new Request("https://herdr.dev/api/plugins"), env()); - expect(response.status).toBe(404); - expect(response.headers.get("Cache-Control")).toBe("no-store"); - }); -}); diff --git a/workers/plugin-marketplace/src/index.ts b/workers/plugin-marketplace/src/index.ts deleted file mode 100644 index 2e5b9dfb..00000000 --- a/workers/plugin-marketplace/src/index.ts +++ /dev/null @@ -1,1269 +0,0 @@ -import { parse } from "smol-toml"; - -const SNAPSHOT_KEY = "plugins/index.json"; -const SCAN_CACHE_KEY = "plugins/scan-cache.json"; -const SNAPSHOT_CACHE_CONTROL = "public, max-age=300, s-maxage=1800, stale-while-revalidate=3600"; -const GITHUB_QUERY = "topic:herdr-plugin is:public"; -const GITHUB_API_VERSION = "2022-11-28"; -const GITHUB_API_URL = "https://api.github.com"; -const GITHUB_SEARCH_URL = `${GITHUB_API_URL}/search/repositories`; -const GITHUB_GRAPHQL_URL = `${GITHUB_API_URL}/graphql`; -const BLACKLIST_REPO_KEY_PREFIX = "repo:"; -const PLUGIN_MANIFEST_FILE = "herdr-plugin.toml"; -const PER_PAGE = 100; -const MAX_REPOS = 1000; -const GRAPHQL_BATCH_SIZE = 50; -const NETWORK_CONCURRENCY = 5; -const MANIFEST_MAX_BYTES = 32 * 1024; -const MAX_TREE_ENTRIES = 20_000; -const MAX_MANIFESTS_PER_REPOSITORY = 100; -const MAX_TOTAL_MANIFESTS = 5000; -const MAX_TOTAL_MANIFEST_BYTES = 16 * 1024 * 1024; -const PLUGIN_NAME_MAX_CHARS = 120; -const PLUGIN_VERSION_MAX_CHARS = 64; -const PLUGIN_DESCRIPTION_MAX_CHARS = 500; -const REQUEST_TIMEOUT_MS = 10_000; -const SCAN_CACHE_SCHEMA_VERSION = 1; -const STAR_HISTORY_KEY = "plugins/star-history.json"; -const STAR_HISTORY_BACKUP_KEY_PREFIX = "backups/star-history/"; -const STAR_HISTORY_SCHEMA_VERSION = 1; -const STAR_HISTORY_MAX_AGE_DAYS = 60; -const STAR_BASELINE_TOLERANCE_DAYS = 2; -const DAY_MS = 86_400_000; - -const PLUGIN_PLATFORMS = new Set(["linux", "macos", "windows"]); -const REGULAR_BLOB_MODES = new Set(["100644", "100755"]); - -type R2Bucket = { - put( - key: string, - value: string, - options?: { - httpMetadata?: { - contentType?: string; - cacheControl?: string; - }; - }, - ): Promise; - get(key: string): Promise<{ text(): Promise } | null>; -}; - -type KVNamespace = { - list(options?: { prefix?: string; cursor?: string }): Promise<{ - keys: Array<{ name: string }>; - cursor?: string; - }>; -}; - -type ExecutionContext = { - waitUntil(promise: Promise): void; -}; - -type ScheduledController = unknown; - -export type Env = { - PLUGIN_MARKETPLACE_BUCKET: R2Bucket; - PLUGIN_MARKETPLACE_BACKUP_BUCKET: R2Bucket; - PLUGIN_MARKETPLACE_BLACKLIST?: KVNamespace; - GITHUB_TOKEN?: string; -}; - -type FetchLike = typeof fetch; - -type RefreshOptions = { - fetch?: FetchLike; - now?: Date; - logger?: Pick; -}; - -type GitHubRepository = Record; - -type RepositoryListing = { - id: number; - fullName: string; - owner: string; - name: string; - description: string | null; - url: string; - stars: number; - forks: number; - openIssues: number; - language: string | null; - topics: string[]; - createdAt: string | null; - updatedAt: string | null; - pushedAt: string | null; - defaultBranch: string; -}; - -export type PluginManifestListing = { - path: string; - id: string; - name: string; - version: string; - minHerdrVersion: string; - description: string | null; - platforms: string[] | null; -}; - -export type PluginListing = Omit & { - headCommit: string; - firstSeenAt: string; - starsDelta7d: number | null; - starsDelta30d: number | null; - manifests: PluginManifestListing[]; -}; - -export type StarHistorySample = { - date: string; - stars: number; -}; - -export type StarHistoryEntry = { - repositoryId: number; - fullName: string; - firstSeenAt: string; - samples: StarHistorySample[]; -}; - -export type StarHistory = { - schemaVersion: 1; - entries: StarHistoryEntry[]; -}; - -export type PluginSnapshot = { - schemaVersion: 1; - generatedAt: string; - pluginCount: number; - repositoryCount: number; - source: { - provider: "github"; - query: string; - totalCount: number; - collectedCount: number; - blacklistedCount: number; - missingManifestCount: number; - invalidManifestCount: number; - duplicateManifestCount: number; - skippedRepositoryCount: number; - unavailableRepositoryCount: number; - truncated: boolean; - warnings?: string[]; - }; - plugins: PluginListing[]; -}; - -type ScanCacheEntry = { - repositoryId: number; - fullName: string; - defaultBranch: string; - pushedAt: string | null; - headCommit: string; - manifestFileCount: number; - invalidManifestCount: number; - duplicateManifestCount: number; - warning: string | null; - retryable: boolean; - manifests: PluginManifestListing[]; -}; - -type ScanCache = { - schemaVersion: 1; - entries: ScanCacheEntry[]; -}; - -type ResolvedRepository = { - repository: RepositoryListing; - headCommit: string; - cached: ScanCacheEntry | null; -}; - -type TreeManifestCandidate = { - repository: RepositoryListing; - headCommit: string; - path: string; - size: number; -}; - -type PendingRepositoryScan = { - resolved: ResolvedRepository; - manifestFileCount: number; - invalidManifestCount: number; - warning: string | null; - retryable: boolean; - candidates: TreeManifestCandidate[]; -}; - -type GitHubTreeEntry = { - path: string; - mode: string; - type: string; - size: number | null; -}; - -export type RefreshResult = - | { ok: true; snapshot: PluginSnapshot } - | { ok: false; error: string }; - -export default { - fetch(): Response { - return jsonResponse({ error: "Not found" }, 404, "no-store"); - }, - - scheduled(_event: ScheduledController, env: Env, ctx: ExecutionContext): void { - ctx.waitUntil(refreshPlugins(env)); - }, -}; - -export async function refreshPlugins( - env: Env, - options: RefreshOptions = {}, -): Promise { - const logger = options.logger ?? console; - try { - const token = env.GITHUB_TOKEN?.trim(); - if (!token) { - throw new Error("GITHUB_TOKEN is not configured"); - } - - const fetchFn = options.fetch ?? fetch; - const search = await fetchGitHubRepositories(fetchFn, token); - const repositories = normalizeRepositories(search.repositories); - const blockedRepositories = await readBlacklistedRepositories(env); - const candidates = repositories.filter( - (repository) => !blockedRepositories.has(repository.fullName.toLowerCase()), - ); - const blacklistedCount = repositories.length - candidates.length; - const cache = await readScanCache(env.PLUGIN_MARKETPLACE_BUCKET, logger); - if (cache.entries.length > 0 && repositories.length * 2 < cache.entries.length) { - throw new Error( - `GitHub repository inventory collapsed from ${cache.entries.length} to ${repositories.length}; remove ${SCAN_CACHE_KEY} to accept an intentional reset`, - ); - } - const cachedById = new Map(cache.entries.map((entry) => [entry.repositoryId, entry])); - - const { resolved, unavailableRepositoryCount } = await resolveRepositoryHeads( - fetchFn, - token, - candidates, - cachedById, - ); - const resolvedIds = new Set(resolved.map(({ repository }) => repository.id)); - const missingCachedRepository = candidates.find( - (repository) => cachedById.has(repository.id) && !resolvedIds.has(repository.id), - ); - if ( - missingCachedRepository || - (candidates.length > 0 && resolved.length * 2 < candidates.length) - ) { - throw new Error( - `GitHub head resolution omitted marketplace repositories${missingCachedRepository ? ` including ${missingCachedRepository.fullName}` : ""}`, - ); - } - const changed = resolved.filter( - ({ headCommit, cached }) => !cached || cached.headCommit !== headCommit || cached.retryable, - ); - const scannedById = await scanChangedRepositories(fetchFn, token, changed); - - const nextEntries = resolved.map(({ repository, headCommit, cached }) => { - const scanned = scannedById.get(repository.id); - if (scanned) return scanned; - if (!cached || cached.headCommit !== headCommit) { - throw new Error(`missing marketplace scan result for ${repository.fullName}`); - } - return { - ...cached, - fullName: repository.fullName, - defaultBranch: repository.defaultBranch, - pushedAt: repository.pushedAt, - }; - }); - const entriesById = new Map(nextEntries.map((entry) => [entry.repositoryId, entry])); - const cards = resolved.flatMap(({ repository }) => { - const entry = entriesById.get(repository.id); - if (!entry || entry.manifests.length === 0) return []; - const { defaultBranch: _defaultBranch, ...card } = repository; - return [{ ...card, headCommit: entry.headCommit, manifests: entry.manifests }]; - }); - - const now = options.now ?? new Date(); - // History updates are read-modify-write without a conditional put. - // Overlapping runs are rare (30-minute cron, sub-minute refreshes) and the - // worst case is a daily sample recording a slightly later observation, so - // the race is accepted instead of building compare-and-swap on R2. - const history = await readStarHistory(env.PLUGIN_MARKETPLACE_BUCKET); - const nextHistory = updateStarHistory(history, cards, now); - const historyById = new Map(nextHistory.entries.map((entry) => [entry.repositoryId, entry])); - const plugins: PluginListing[] = cards.map((card) => { - const entry = historyById.get(card.id); - return { - ...card, - firstSeenAt: entry?.firstSeenAt ?? now.toISOString(), - starsDelta7d: entry ? starsDeltaSince(entry, card.stars, now, 7) : null, - starsDelta30d: entry ? starsDeltaSince(entry, card.stars, now, 30) : null, - }; - }); - - const pluginCount = plugins.reduce((total, plugin) => total + plugin.manifests.length, 0); - const warnings = nextEntries.flatMap((entry) => (entry.warning ? [entry.warning] : [])); - if (search.truncated) { - warnings.unshift( - `GitHub returned ${search.totalCount} results; only the first ${search.repositories.length} were collected.`, - ); - } - const generatedAt = now.toISOString(); - const snapshot: PluginSnapshot = { - schemaVersion: 1, - generatedAt, - pluginCount, - repositoryCount: plugins.length, - source: { - provider: "github", - query: GITHUB_QUERY, - totalCount: search.totalCount, - collectedCount: search.repositories.length, - blacklistedCount, - missingManifestCount: nextEntries.filter( - (entry) => entry.manifestFileCount === 0 && !entry.warning, - ).length, - invalidManifestCount: nextEntries.reduce( - (total, entry) => total + entry.invalidManifestCount, - 0, - ), - duplicateManifestCount: nextEntries.reduce( - (total, entry) => total + entry.duplicateManifestCount, - 0, - ), - skippedRepositoryCount: nextEntries.filter((entry) => entry.warning).length, - unavailableRepositoryCount, - truncated: search.truncated, - ...(warnings.length > 0 ? { warnings } : {}), - }, - plugins, - }; - - const nextCache: ScanCache = { schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: nextEntries }; - // Star history is the only object that cannot be rebuilt, so the first run - // of each UTC day copies it to a dated key in the private backup bucket. - // The dated key's existence marks the day as backed up, which stays correct - // when history is empty or a repository joins mid-day. Backups are kept - // forever; the backup put runs before the primary puts so a failure leaves - // the primary untouched and the next run retries the same dated key. The - // get-then-put pair is not atomic; like the history read-modify-write - // above, overlapping runs are accepted because the worst case is the dated - // backup holding an observation from seconds later. - const backupKey = `${STAR_HISTORY_BACKUP_KEY_PREFIX}${isoDate(now)}.json`; - if ((await env.PLUGIN_MARKETPLACE_BACKUP_BUCKET.get(backupKey)) === null) { - await env.PLUGIN_MARKETPLACE_BACKUP_BUCKET.put(backupKey, JSON.stringify(nextHistory), { - httpMetadata: { - contentType: "application/json; charset=utf-8", - cacheControl: "no-store", - }, - }); - } - await env.PLUGIN_MARKETPLACE_BUCKET.put(STAR_HISTORY_KEY, JSON.stringify(nextHistory), { - httpMetadata: { - contentType: "application/json; charset=utf-8", - cacheControl: "no-store", - }, - }); - await env.PLUGIN_MARKETPLACE_BUCKET.put(SCAN_CACHE_KEY, JSON.stringify(nextCache), { - httpMetadata: { - contentType: "application/json; charset=utf-8", - cacheControl: "no-store", - }, - }); - await env.PLUGIN_MARKETPLACE_BUCKET.put(SNAPSHOT_KEY, JSON.stringify(snapshot), { - httpMetadata: { - contentType: "application/json; charset=utf-8", - cacheControl: SNAPSHOT_CACHE_CONTROL, - }, - }); - return { ok: true, snapshot }; - } catch (error) { - const message = error instanceof Error ? error.message : "unknown refresh error"; - logger.error(`plugin marketplace refresh failed: ${message}`); - return { ok: false, error: message }; - } -} - -async function fetchGitHubRepositories( - fetchFn: FetchLike, - token: string, - timeoutMs = REQUEST_TIMEOUT_MS, -): Promise<{ repositories: GitHubRepository[]; totalCount: number; truncated: boolean }> { - const repositories: GitHubRepository[] = []; - let totalCount = 0; - - for (let page = 1; repositories.length < MAX_REPOS; page += 1) { - const url = new URL(GITHUB_SEARCH_URL); - url.searchParams.set("q", GITHUB_QUERY); - url.searchParams.set("per_page", String(PER_PAGE)); - url.searchParams.set("page", String(page)); - url.searchParams.set("sort", "stars"); - url.searchParams.set("order", "desc"); - - const response = await fetchWithTimeout(fetchFn, url, githubRequestInit(token), timeoutMs); - if (!response.ok) { - throw new Error(`GitHub search failed with status ${response.status}`); - } - const body = await response.json(); - if (!isObject(body) || typeof body.total_count !== "number" || !Array.isArray(body.items)) { - throw new Error("GitHub search returned malformed JSON"); - } - if (body.incomplete_results === true) { - throw new Error("GitHub search returned incomplete results"); - } - - totalCount = body.total_count; - repositories.push(...body.items.slice(0, MAX_REPOS - repositories.length)); - if (repositories.length >= totalCount || body.items.length === 0) break; - } - - return { - repositories, - totalCount, - truncated: totalCount > repositories.length, - }; -} - -async function resolveRepositoryHeads( - fetchFn: FetchLike, - token: string, - repositories: RepositoryListing[], - cachedById: Map, -): Promise<{ resolved: ResolvedRepository[]; unavailableRepositoryCount: number }> { - const resolvedById = new Map(); - const needResolution: RepositoryListing[] = []; - - for (const repository of repositories) { - const cached = cachedById.get(repository.id) ?? null; - if ( - cached && - cached.fullName.toLowerCase() === repository.fullName.toLowerCase() && - cached.defaultBranch === repository.defaultBranch && - !cached.retryable && - cached.pushedAt !== null && - cached.pushedAt === repository.pushedAt - ) { - resolvedById.set(repository.id, { repository, headCommit: cached.headCommit, cached }); - } else { - needResolution.push(repository); - } - } - - const batches = chunk(needResolution, GRAPHQL_BATCH_SIZE); - const results = await mapConcurrent(batches, NETWORK_CONCURRENCY, async (batch) => { - const data = await fetchGraphqlData(fetchFn, token, repositoryHeadQuery(batch)); - return batch.flatMap((repository, index) => { - const raw = data[`repo${index}`]; - if (!isObject(raw)) return []; - const branch = isObject(raw.defaultBranchRef) ? readString(raw.defaultBranchRef.name) : null; - const target = isObject(raw.defaultBranchRef?.target) ? raw.defaultBranchRef.target : null; - const headCommit = isObject(target) ? readString(target.oid) : null; - if ( - branch !== repository.defaultBranch || - !headCommit || - !/^[0-9a-f]{40,64}$/i.test(headCommit) - ) { - return []; - } - return [{ repository, headCommit, cached: cachedById.get(repository.id) ?? null }]; - }); - }); - for (const resolved of results.flat()) { - resolvedById.set(resolved.repository.id, resolved); - } - - return { - resolved: repositories.flatMap((repository) => { - const value = resolvedById.get(repository.id); - return value ? [value] : []; - }), - unavailableRepositoryCount: repositories.length - resolvedById.size, - }; -} - -async function scanChangedRepositories( - fetchFn: FetchLike, - token: string, - repositories: ResolvedRepository[], -): Promise> { - const pending = await mapConcurrent(repositories, NETWORK_CONCURRENCY, (resolved) => - fetchRepositoryTree(fetchFn, token, resolved), - ); - const candidates: TreeManifestCandidate[] = []; - let candidateBytes = 0; - for (const scan of pending) { - const scanBytes = scan.candidates.reduce((total, candidate) => total + candidate.size, 0); - if ( - candidates.length + scan.candidates.length > MAX_TOTAL_MANIFESTS || - candidateBytes + scanBytes > MAX_TOTAL_MANIFEST_BYTES - ) { - scan.warning = `${scan.resolved.repository.fullName} was skipped because the marketplace scan budget was exhausted.`; - scan.retryable = true; - scan.candidates = []; - continue; - } - candidates.push(...scan.candidates); - candidateBytes += scanBytes; - } - const contents = await fetchManifestContents(fetchFn, token, candidates); - const entries = new Map(); - - for (const scan of pending) { - const manifests: PluginManifestListing[] = []; - let invalidManifestCount = scan.invalidManifestCount; - for (const candidate of scan.candidates) { - const content = contents.get(manifestCandidateKey(candidate)); - const metadata = typeof content === "string" ? parseManifestSummary(content) : null; - if (!metadata) { - invalidManifestCount += 1; - continue; - } - manifests.push({ path: candidate.path, ...metadata }); - } - const { manifests: uniqueManifests, duplicateCount } = deduplicateManifests(manifests); - const { repository, headCommit } = scan.resolved; - entries.set(repository.id, { - repositoryId: repository.id, - fullName: repository.fullName, - defaultBranch: repository.defaultBranch, - pushedAt: repository.pushedAt, - headCommit, - manifestFileCount: scan.manifestFileCount, - invalidManifestCount, - duplicateManifestCount: duplicateCount, - warning: scan.warning, - retryable: scan.retryable, - manifests: uniqueManifests, - }); - } - return entries; -} - -async function fetchRepositoryTree( - fetchFn: FetchLike, - token: string, - resolved: ResolvedRepository, -): Promise { - const { repository, headCommit } = resolved; - const url = new URL( - `${GITHUB_API_URL}/repos/${encodeURIComponent(repository.owner)}/${encodeURIComponent(repository.name)}/git/trees/${headCommit}`, - ); - url.searchParams.set("recursive", "1"); - const response = await fetchWithTimeout(fetchFn, url, githubRequestInit(token), REQUEST_TIMEOUT_MS); - if (!response.ok) { - throw new Error(`GitHub tree fetch failed for ${repository.fullName} with status ${response.status}`); - } - const body = await response.json(); - if (!isObject(body) || !Array.isArray(body.tree)) { - throw new Error(`GitHub tree fetch returned malformed JSON for ${repository.fullName}`); - } - if (body.truncated === true || body.tree.length > MAX_TREE_ENTRIES) { - return skippedRepositoryScan(resolved, `${repository.fullName} was skipped because its Git tree is too large.`); - } - - const manifestEntries = body.tree - .map(normalizeTreeEntry) - .filter((entry): entry is GitHubTreeEntry => entry !== null) - .filter( - (entry) => - isDiscoverableManifestPath(entry.path) && - entry.type === "blob" && - REGULAR_BLOB_MODES.has(entry.mode), - ); - if (manifestEntries.length > MAX_MANIFESTS_PER_REPOSITORY) { - return skippedRepositoryScan( - resolved, - `${repository.fullName} was skipped because it contains more than ${MAX_MANIFESTS_PER_REPOSITORY} plugin manifests.`, - ); - } - - let invalidManifestCount = 0; - const candidates: TreeManifestCandidate[] = []; - for (const entry of manifestEntries) { - if ( - entry.size === null || - entry.size < 0 || - entry.size > MANIFEST_MAX_BYTES - ) { - invalidManifestCount += 1; - continue; - } - candidates.push({ repository, headCommit, path: entry.path, size: entry.size }); - } - - return { - resolved, - manifestFileCount: manifestEntries.length, - invalidManifestCount, - warning: null, - retryable: false, - candidates, - }; -} - -function skippedRepositoryScan( - resolved: ResolvedRepository, - warning: string, -): PendingRepositoryScan { - return { - resolved, - manifestFileCount: 0, - invalidManifestCount: 0, - warning, - retryable: false, - candidates: [], - }; -} - -async function fetchManifestContents( - fetchFn: FetchLike, - token: string, - candidates: TreeManifestCandidate[], -): Promise> { - const batches = chunk(candidates, GRAPHQL_BATCH_SIZE); - const results = await mapConcurrent(batches, NETWORK_CONCURRENCY, async (batch) => { - const data = await fetchGraphqlData(fetchFn, token, manifestContentQuery(batch)); - return batch.flatMap((candidate, index) => { - const raw = data[`item${index}`]; - const manifest = isObject(raw) && isObject(raw.manifest) ? raw.manifest : null; - if (!manifest) { - throw new Error( - `GitHub manifest fetch omitted ${candidate.repository.fullName}:${candidate.path}`, - ); - } - if (manifest.isBinary === true) { - return [[manifestCandidateKey(candidate), null] as const]; - } - if (typeof manifest.text !== "string") { - throw new Error( - `GitHub manifest fetch returned malformed data for ${candidate.repository.fullName}:${candidate.path}`, - ); - } - return [[manifestCandidateKey(candidate), manifest.text] as const]; - }); - }); - return new Map(results.flat()); -} - -async function fetchGraphqlData( - fetchFn: FetchLike, - token: string, - query: string, -): Promise> { - const response = await fetchWithTimeout( - fetchFn, - new URL(GITHUB_GRAPHQL_URL), - { - ...githubRequestInit(token), - method: "POST", - body: JSON.stringify({ query }), - }, - REQUEST_TIMEOUT_MS, - ); - if (!response.ok) { - throw new Error(`GitHub marketplace query failed with status ${response.status}`); - } - const body = await response.json(); - if (!isObject(body) || !isObject(body.data)) { - throw new Error("GitHub marketplace query returned malformed JSON"); - } - if ( - Array.isArray(body.errors) && - body.errors.some((error) => !isMissingAliasGraphqlError(error)) - ) { - throw new Error("GitHub marketplace query returned GraphQL errors"); - } - return body.data; -} - -function isMissingAliasGraphqlError(error: unknown): boolean { - if (!isObject(error) || error.type !== "NOT_FOUND" || !Array.isArray(error.path)) { - return false; - } - const alias = error.path[0]; - return typeof alias === "string" && /^(repo|item)\d+$/.test(alias); -} - -function repositoryHeadQuery(repositories: RepositoryListing[]): string { - const selections = repositories.map((repository, index) => { - const owner = JSON.stringify(repository.owner); - const name = JSON.stringify(repository.name); - return `repo${index}: repository(owner: ${owner}, name: ${name}) { - defaultBranchRef { name target { ... on Commit { oid } } } - }`; - }); - return `query PluginMarketplaceHeads {\n${selections.join("\n")}\n}`; -} - -function manifestContentQuery(candidates: TreeManifestCandidate[]): string { - const selections = candidates.map((candidate, index) => { - const owner = JSON.stringify(candidate.repository.owner); - const name = JSON.stringify(candidate.repository.name); - const expression = JSON.stringify(`${candidate.headCommit}:${candidate.path}`); - return `item${index}: repository(owner: ${owner}, name: ${name}) { - manifest: object(expression: ${expression}) { ... on Blob { text isBinary } } - }`; - }); - return `query PluginMarketplaceManifests {\n${selections.join("\n")}\n}`; -} - -export function parseManifestSummary(manifestText: string): Omit | null { - try { - const manifest = parse(manifestText.replace(/^\uFEFF/, "")); - if (!isObject(manifest)) return null; - const id = readTrimmedString(manifest.id); - const name = readTrimmedString(manifest.name); - const version = readTrimmedString(manifest.version); - const minHerdrVersion = readTrimmedString(manifest.min_herdr_version); - if ( - !id || - id.length > 120 || - !/^[A-Za-z0-9:._-]+$/.test(id) || - !name || - name.length > PLUGIN_NAME_MAX_CHARS || - !version || - version.length > PLUGIN_VERSION_MAX_CHARS || - !minHerdrVersion || - minHerdrVersion.length > PLUGIN_VERSION_MAX_CHARS || - !isHerdrVersion(minHerdrVersion) - ) { - return null; - } - - let description: string | null = null; - if (manifest.description !== undefined) { - if (typeof manifest.description !== "string") return null; - description = manifest.description.trim() || null; - if (description && description.length > PLUGIN_DESCRIPTION_MAX_CHARS) return null; - } - - let platforms: string[] | null = null; - if (manifest.platforms !== undefined) { - if ( - !Array.isArray(manifest.platforms) || - manifest.platforms.length === 0 || - manifest.platforms.length > PLUGIN_PLATFORMS.size || - !manifest.platforms.every( - (platform) => typeof platform === "string" && PLUGIN_PLATFORMS.has(platform), - ) - ) { - return null; - } - platforms = [...manifest.platforms]; - if (new Set(platforms).size !== platforms.length) return null; - } - - return { id, name, version, minHerdrVersion, description, platforms }; - } catch { - return null; - } -} - -export function updateStarHistory( - history: StarHistory, - cards: Array<{ id: number; fullName: string; stars: number }>, - now: Date, -): StarHistory { - const today = isoDate(now); - const oldestKeptMs = now.getTime() - STAR_HISTORY_MAX_AGE_DAYS * DAY_MS; - const entriesById = new Map(history.entries.map((entry) => [entry.repositoryId, entry])); - const nextEntries = new Map(); - - for (const card of cards) { - const existing = entriesById.get(card.id); - const samples = (existing?.samples ?? []).filter( - (sample) => dateMs(sample.date) >= oldestKeptMs, - ); - // One sample per UTC day, first observation wins, so intraday cron runs - // keep a stable baseline for delta computation. - if (samples[samples.length - 1]?.date !== today) { - samples.push({ date: today, stars: card.stars }); - } - nextEntries.set(card.id, { - repositoryId: card.id, - fullName: card.fullName, - firstSeenAt: existing?.firstSeenAt ?? now.toISOString(), - samples, - }); - } - - // Keep history for repositories that dropped out of the snapshot (renamed, - // temporarily invalid manifest, blacklist experiments) so they rejoin with - // their record intact, until their samples age out. - for (const entry of history.entries) { - if (nextEntries.has(entry.repositoryId)) continue; - const samples = entry.samples.filter((sample) => dateMs(sample.date) >= oldestKeptMs); - if (samples.length > 0) { - nextEntries.set(entry.repositoryId, { ...entry, samples }); - } - } - - return { - schemaVersion: STAR_HISTORY_SCHEMA_VERSION, - entries: [...nextEntries.values()].sort((a, b) => a.repositoryId - b.repositoryId), - }; -} - -export function starsDeltaSince( - entry: StarHistoryEntry, - currentStars: number, - now: Date, - windowDays: number, -): number | null { - // The baseline must sit close to the window boundary. A repository that was - // delisted for weeks would otherwise report months of growth as a short - // window and dominate trending rankings. - const cutoffMs = now.getTime() - windowDays * DAY_MS; - const oldestAcceptedMs = cutoffMs - STAR_BASELINE_TOLERANCE_DAYS * DAY_MS; - let baseline: StarHistorySample | null = null; - for (const sample of entry.samples) { - const sampleMs = dateMs(sample.date); - if (sampleMs > cutoffMs) break; - if (sampleMs >= oldestAcceptedMs) baseline = sample; - } - return baseline ? currentStars - baseline.stars : null; -} - -// Unlike the scan cache, star history cannot be rebuilt from GitHub, so any -// invalid state fails the refresh instead of being discarded and overwritten. -async function readStarHistory(bucket: R2Bucket): Promise { - const object = await bucket.get(STAR_HISTORY_KEY); - if (!object) return { schemaVersion: STAR_HISTORY_SCHEMA_VERSION, entries: [] }; - const invalid = (reason: string): Error => - new Error( - `invalid plugin marketplace star history (${reason}); remove ${STAR_HISTORY_KEY} to accept losing recorded history`, - ); - let value: unknown; - try { - value = JSON.parse(await object.text()); - } catch { - throw invalid("malformed JSON"); - } - if ( - !isObject(value) || - value.schemaVersion !== STAR_HISTORY_SCHEMA_VERSION || - !Array.isArray(value.entries) - ) { - throw invalid("unsupported shape"); - } - const entries = value.entries.map(readStarHistoryEntry); - if (entries.some((entry) => entry === null)) { - throw invalid("invalid entry"); - } - return { schemaVersion: STAR_HISTORY_SCHEMA_VERSION, entries: entries as StarHistoryEntry[] }; -} - -function readStarHistoryEntry(value: unknown): StarHistoryEntry | null { - if (!isObject(value) || !Array.isArray(value.samples)) return null; - const repositoryId = readInteger(value.repositoryId); - const fullName = readString(value.fullName); - const firstSeenAt = readIsoString(value.firstSeenAt); - const samples = value.samples.map(readStarHistorySample); - if ( - repositoryId === null || - repositoryId <= 0 || - !fullName || - !firstSeenAt || - samples.some((sample) => sample === null) - ) { - return null; - } - return { - repositoryId, - fullName, - firstSeenAt, - samples: (samples as StarHistorySample[]).sort((a, b) => a.date.localeCompare(b.date)), - }; -} - -function readStarHistorySample(value: unknown): StarHistorySample | null { - if (!isObject(value)) return null; - const date = readString(value.date); - const stars = readNonNegativeIntegerOrNull(value.stars); - if (!date || !/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date)) || stars === null) { - return null; - } - return { date, stars }; -} - -function isoDate(now: Date): string { - return now.toISOString().slice(0, 10); -} - -async function readScanCache( - bucket: R2Bucket, - logger: Pick, -): Promise { - const empty = (): ScanCache => ({ schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: [] }); - const object = await bucket.get(SCAN_CACHE_KEY); - if (!object) return empty(); - try { - const value: unknown = JSON.parse(await object.text()); - if ( - !isObject(value) || - value.schemaVersion !== SCAN_CACHE_SCHEMA_VERSION || - !Array.isArray(value.entries) - ) { - throw new Error("unsupported cache shape"); - } - const entries = value.entries.map(readScanCacheEntry); - if (entries.some((entry) => entry === null)) { - throw new Error("invalid cache entry"); - } - return { schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: entries as ScanCacheEntry[] }; - } catch (error) { - const message = error instanceof Error ? error.message : "unknown cache error"; - logger.error(`discarding invalid plugin marketplace scan cache: ${message}`); - return empty(); - } -} - -function readScanCacheEntry(value: unknown): ScanCacheEntry | null { - if (!isObject(value) || !Array.isArray(value.manifests)) return null; - const repositoryId = readInteger(value.repositoryId); - const fullName = readString(value.fullName); - const defaultBranch = readString(value.defaultBranch); - const pushedAt = value.pushedAt === null ? null : readIsoString(value.pushedAt); - const headCommit = readString(value.headCommit); - const manifestFileCount = readNonNegativeIntegerOrNull(value.manifestFileCount); - const invalidManifestCount = readNonNegativeIntegerOrNull(value.invalidManifestCount); - const duplicateManifestCount = readNonNegativeIntegerOrNull(value.duplicateManifestCount); - const warning = value.warning === null ? null : readString(value.warning); - const retryable = typeof value.retryable === "boolean" ? value.retryable : null; - const manifests = value.manifests.map(readCachedManifest); - if ( - repositoryId === null || - !fullName || - !defaultBranch || - (value.pushedAt !== null && pushedAt === null) || - !headCommit || - !/^[0-9a-f]{40,64}$/i.test(headCommit) || - manifestFileCount === null || - invalidManifestCount === null || - duplicateManifestCount === null || - retryable === null || - (value.warning !== null && !warning) || - manifests.some((manifest) => manifest === null) - ) { - return null; - } - return { - repositoryId, - fullName, - defaultBranch, - pushedAt, - headCommit, - manifestFileCount, - invalidManifestCount, - duplicateManifestCount, - warning, - retryable, - manifests: manifests as PluginManifestListing[], - }; -} - -function readCachedManifest(value: unknown): PluginManifestListing | null { - if (!isObject(value)) return null; - const path = readString(value.path); - const id = readString(value.id); - const name = readString(value.name); - const version = readString(value.version); - const minHerdrVersion = readString(value.minHerdrVersion); - const description = value.description === null ? null : readString(value.description); - const platforms = value.platforms === null ? null : readStringArrayOrNull(value.platforms); - if ( - !path || - !isManifestPath(path) || - !id || - !name || - !version || - !minHerdrVersion || - (value.description !== null && !description) || - (value.platforms !== null && platforms === null) - ) { - return null; - } - return { path, id, name, version, minHerdrVersion, description, platforms }; -} - -export function normalizeRepositories(repositories: GitHubRepository[]): RepositoryListing[] { - const byId = new Map(); - for (const repository of repositories.map(normalizeRepository)) { - if (repository && !byId.has(repository.id)) byId.set(repository.id, repository); - } - return [...byId.values()].sort(compareRepositories); -} - -async function readBlacklistedRepositories(env: Env): Promise> { - const kv = env.PLUGIN_MARKETPLACE_BLACKLIST; - const blockedRepositories = new Set(); - if (!kv) return blockedRepositories; - - let cursor: string | undefined; - do { - const page = await kv.list({ prefix: BLACKLIST_REPO_KEY_PREFIX, cursor }); - for (const key of page.keys) { - const repository = key.name.slice(BLACKLIST_REPO_KEY_PREFIX.length).trim().toLowerCase(); - if (repository.includes("/")) blockedRepositories.add(repository); - } - cursor = page.cursor; - } while (cursor); - return blockedRepositories; -} - -function normalizeRepository(repo: GitHubRepository): RepositoryListing | null { - if ( - readBoolean(repo.disabled) || - readBoolean(repo.archived) || - readBoolean(repo.fork) || - readBoolean(repo.private) || - readString(repo.visibility) === "private" - ) { - return null; - } - - const fullNameParts = splitFullName(readString(repo.full_name)); - const ownerObject = isObject(repo.owner) ? repo.owner : {}; - const owner = firstString(readString(ownerObject.login), fullNameParts.owner); - const name = firstString(readString(repo.name), fullNameParts.name); - const defaultBranch = readString(repo.default_branch); - if (!owner || !name || !defaultBranch) return null; - - const fullName = readString(repo.full_name) ?? `${owner}/${name}`; - const url = readString(repo.html_url); - const id = readInteger(repo.id); - if (!url || !isValidGitHubRepoUrl(url, owner, name) || id === null || id <= 0) return null; - - return { - id, - fullName, - owner, - name, - description: readNullableString(repo.description), - url, - stars: readNonNegativeInteger(repo.stargazers_count), - forks: readNonNegativeInteger(repo.forks_count), - openIssues: readNonNegativeInteger(repo.open_issues_count), - language: readNullableString(repo.language), - topics: readStringArray(repo.topics), - createdAt: readIsoString(repo.created_at), - updatedAt: readIsoString(repo.updated_at), - pushedAt: readIsoString(repo.pushed_at), - defaultBranch, - }; -} - -function normalizeTreeEntry(value: unknown): GitHubTreeEntry | null { - if (!isObject(value)) return null; - const path = readString(value.path); - const mode = readString(value.mode); - const type = readString(value.type); - if (!path || !mode || !type) return null; - return { path, mode, type, size: readNonNegativeIntegerOrNull(value.size) }; -} - -function isManifestPath(path: string): boolean { - return path === PLUGIN_MANIFEST_FILE || path.endsWith(`/${PLUGIN_MANIFEST_FILE}`); -} - -function isDiscoverableManifestPath(path: string): boolean { - if (!isManifestPath(path)) return false; - const ignoredSegments = new Set([ - "test", - "tests", - "fixture", - "fixtures", - "testdata", - "__fixtures__", - ]); - return !path - .split("/") - .slice(0, -1) - .some((segment) => ignoredSegments.has(segment.toLowerCase())); -} - -function deduplicateManifests(manifests: PluginManifestListing[]): { - manifests: PluginManifestListing[]; - duplicateCount: number; -} { - const preferred = [...manifests].sort((a, b) => { - const depth = (path: string) => path.split("/").length; - return depth(a.path) - depth(b.path) || a.path.localeCompare(b.path); - }); - const byId = new Map(); - let duplicateCount = 0; - for (const manifest of preferred) { - if (byId.has(manifest.id)) { - duplicateCount += 1; - } else { - byId.set(manifest.id, manifest); - } - } - return { - manifests: [...byId.values()].sort((a, b) => a.path.localeCompare(b.path)), - duplicateCount, - }; -} - -function manifestCandidateKey(candidate: TreeManifestCandidate): string { - return `${candidate.repository.id}:${candidate.path}`; -} - -function compareRepositories(a: RepositoryListing, b: RepositoryListing): number { - return ( - b.stars - a.stars || - dateMs(b.pushedAt) - dateMs(a.pushedAt) || - a.fullName.localeCompare(b.fullName) - ); -} - -function githubRequestInit(token: string): RequestInit { - return { - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "User-Agent": "herdr-plugin-marketplace", - "X-GitHub-Api-Version": GITHUB_API_VERSION, - }, - }; -} - -async function fetchWithTimeout( - fetchFn: FetchLike, - url: URL, - init: RequestInit, - timeoutMs: number, -): Promise { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), timeoutMs); - try { - return await fetchFn(url, { ...init, signal: controller.signal }); - } finally { - clearTimeout(timeout); - } -} - -function chunk(values: T[], size: number): T[][] { - const chunks: T[][] = []; - for (let index = 0; index < values.length; index += size) { - chunks.push(values.slice(index, index + size)); - } - return chunks; -} - -async function mapConcurrent( - values: T[], - concurrency: number, - operation: (value: T) => Promise, -): Promise { - const results = new Array(values.length); - let nextIndex = 0; - const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => { - while (nextIndex < values.length) { - const index = nextIndex; - nextIndex += 1; - results[index] = await operation(values[index]); - } - }); - await Promise.all(workers); - return results; -} - -function jsonResponse(body: unknown, status: number, cacheControl: string): Response { - return new Response(JSON.stringify(body), { - status, - headers: { - "Content-Type": "application/json", - "Cache-Control": cacheControl, - }, - }); -} - -function isValidGitHubRepoUrl(url: string, owner: string, name: string): boolean { - try { - const parsed = new URL(url); - const segments = parsed.pathname.split("/").filter(Boolean); - return ( - parsed.protocol === "https:" && - parsed.hostname === "github.com" && - segments.length === 2 && - segments[0].toLowerCase() === owner.toLowerCase() && - segments[1].toLowerCase() === name.toLowerCase() - ); - } catch { - return false; - } -} - -function splitFullName(fullName: string | null): { owner: string | null; name: string | null } { - if (!fullName) return { owner: null, name: null }; - const [owner, name, extra] = fullName.split("/"); - return owner && name && !extra ? { owner, name } : { owner: null, name: null }; -} - -function firstString(...values: Array): string | null { - return values.find((value) => value !== null) ?? null; -} - -function readString(value: unknown): string | null { - return typeof value === "string" && value.length > 0 ? value : null; -} - -function readTrimmedString(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; -} - -function readNullableString(value: unknown): string | null { - return typeof value === "string" ? value : null; -} - -function readStringArray(value: unknown): string[] { - return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; -} - -function readStringArrayOrNull(value: unknown): string[] | null { - return Array.isArray(value) && value.every((item) => typeof item === "string") - ? value - : null; -} - -function readInteger(value: unknown): number | null { - return typeof value === "number" && Number.isInteger(value) ? value : null; -} - -function readNonNegativeInteger(value: unknown): number { - return readNonNegativeIntegerOrNull(value) ?? 0; -} - -function readNonNegativeIntegerOrNull(value: unknown): number | null { - const integer = readInteger(value); - return integer !== null && integer >= 0 ? integer : null; -} - -function readBoolean(value: unknown): boolean { - return value === true; -} - -function readIsoString(value: unknown): string | null { - return typeof value === "string" && !Number.isNaN(Date.parse(value)) ? value : null; -} - -function isHerdrVersion(value: string): boolean { - const normalized = value.startsWith("v") ? value.slice(1) : value; - const parts = normalized.split("."); - return ( - parts.length === 3 && - parts.every((part) => /^\d+$/.test(part) && Number(part) <= 0xffff_ffff) - ); -} - -function dateMs(value: string | null): number { - return value ? Date.parse(value) || 0 : 0; -} - -function isObject(value: unknown): value is Record { - return typeof value === "object" && value !== null; -} diff --git a/workers/plugin-marketplace/wrangler.toml b/workers/plugin-marketplace/wrangler.toml deleted file mode 100644 index 2367055b..00000000 --- a/workers/plugin-marketplace/wrangler.toml +++ /dev/null @@ -1,22 +0,0 @@ -name = "herdr-plugin-marketplace" -main = "src/index.ts" -compatibility_date = "2026-06-21" -workers_dev = false - -[limits] -subrequests = 3500 - -[[r2_buckets]] -binding = "PLUGIN_MARKETPLACE_BUCKET" -bucket_name = "herdr-plugin-marketplace" - -[[r2_buckets]] -binding = "PLUGIN_MARKETPLACE_BACKUP_BUCKET" -bucket_name = "herdr-plugin-marketplace-backup" - -[[kv_namespaces]] -binding = "PLUGIN_MARKETPLACE_BLACKLIST" -id = "6504b3b84171492db56e805f6aad1686" - -[triggers] -crons = ["*/30 * * * *"]