diff --git a/workers/plugin-marketplace/src/index.test.ts b/workers/plugin-marketplace/src/index.test.ts index c09d60eb..b84ed14d 100644 --- a/workers/plugin-marketplace/src/index.test.ts +++ b/workers/plugin-marketplace/src/index.test.ts @@ -9,6 +9,20 @@ class MemoryR2 { } } +class MemoryKV { + constructor(private readonly keyNames: string[]) {} + + async list(options?: { prefix?: string }): Promise<{ + keys: Array<{ name: string }>; + }> { + return { + keys: this.keyNames + .filter((name) => !options?.prefix || name.startsWith(options.prefix)) + .map((name) => ({ name })), + }; + } +} + function repo(overrides: Record = {}): Record { return { id: 1, @@ -34,9 +48,10 @@ function repo(overrides: Record = {}): Record }; } -function env(bucket = new MemoryR2()): Env { +function env(bucket = new MemoryR2(), blacklist?: MemoryKV): Env { return { PLUGIN_MARKETPLACE_BUCKET: bucket, + PLUGIN_MARKETPLACE_BLACKLIST: blacklist, GITHUB_TOKEN: "token", }; } @@ -191,6 +206,68 @@ describe("refreshPlugins", () => { ]); }); + test("excludes repositories listed in the KV blacklist", async () => { + const fetch = async (): Promise => + Response.json({ + total_count: 2, + items: [ + repo({ + id: 1, + full_name: "example/not-a-plugin", + owner: { login: "example" }, + name: "not-a-plugin", + html_url: "https://github.com/example/not-a-plugin", + }), + repo({ + id: 2, + full_name: "ogulcancelik/herdr-plugin-example", + owner: { login: "ogulcancelik" }, + name: "herdr-plugin-example", + html_url: "https://github.com/ogulcancelik/herdr-plugin-example", + }), + ], + }); + const bucket = new MemoryR2(); + + const result = await refreshPlugins(env(bucket, new MemoryKV(["repo:example/not-a-plugin"])), { + fetch, + logger: { error() {} }, + }); + + expect(result.ok).toBe(true); + const snapshot = JSON.parse(bucket.objects.get("plugins/index.json")?.value ?? ""); + expect(snapshot.plugins.map((plugin: { fullName: string }) => plugin.fullName)).toEqual([ + "ogulcancelik/herdr-plugin-example", + ]); + }); + + test("writes an empty snapshot when every listable repository is blacklisted", async () => { + const fetch = async (): Promise => + Response.json({ + total_count: 1, + items: [ + repo({ + id: 1, + full_name: "example/not-a-plugin", + owner: { login: "example" }, + name: "not-a-plugin", + html_url: "https://github.com/example/not-a-plugin", + }), + ], + }); + const bucket = new MemoryR2(); + await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}'); + + const result = await refreshPlugins(env(bucket, new MemoryKV(["repo:example/not-a-plugin"])), { + fetch, + logger: { error() {} }, + }); + + expect(result.ok).toBe(true); + const snapshot = JSON.parse(bucket.objects.get("plugins/index.json")?.value ?? ""); + expect(snapshot.plugins).toEqual([]); + }); + test("marks snapshots truncated at the GitHub search cap", async () => { const fetch = async (): Promise => { const items = Array.from({ length: 100 }, (_, index) => diff --git a/workers/plugin-marketplace/src/index.ts b/workers/plugin-marketplace/src/index.ts index b85abdb8..e4523340 100644 --- a/workers/plugin-marketplace/src/index.ts +++ b/workers/plugin-marketplace/src/index.ts @@ -3,6 +3,7 @@ const SNAPSHOT_CACHE_CONTROL = "public, max-age=300, s-maxage=1800, stale-while- const GITHUB_QUERY = "topic:herdr-plugin is:public"; const GITHUB_API_VERSION = "2022-11-28"; const GITHUB_SEARCH_URL = "https://api.github.com/search/repositories"; +const BLACKLIST_REPO_KEY_PREFIX = "repo:"; const PER_PAGE = 100; const MAX_REPOS = 1000; const REQUEST_TIMEOUT_MS = 10_000; @@ -20,6 +21,13 @@ type R2Bucket = { ): Promise; }; +type KVNamespace = { + list(options?: { prefix?: string; cursor?: string }): Promise<{ + keys: Array<{ name: string }>; + cursor?: string; + }>; +}; + type ExecutionContext = { waitUntil(promise: Promise): void; }; @@ -28,6 +36,7 @@ type ScheduledController = unknown; export type Env = { PLUGIN_MARKETPLACE_BUCKET: R2Bucket; + PLUGIN_MARKETPLACE_BLACKLIST?: KVNamespace; GITHUB_TOKEN?: string; }; @@ -99,11 +108,19 @@ export async function refreshPlugins( const fetchFn = options.fetch ?? fetch; const result = await fetchGitHubRepositories(fetchFn, token); - const plugins = normalizeRepositories(result.repositories); - if (plugins.length === 0) { + const normalizedPlugins = normalizeRepositories(result.repositories); + if (normalizedPlugins.length === 0) { throw new Error("GitHub returned no listable plugin repositories"); } + const blockedRepositories = await readBlacklistedRepositories(env); + const plugins = + blockedRepositories.size === 0 + ? normalizedPlugins + : normalizedPlugins.filter( + (plugin) => !blockedRepositories.has(plugin.fullName.toLowerCase()), + ); + const snapshot: PluginSnapshot = { schemaVersion: 1, generatedAt: (options.now ?? new Date()).toISOString(), @@ -216,6 +233,28 @@ export function normalizeRepositories(repositories: GitHubRepository[]): PluginL .sort(comparePlugins); } +async function readBlacklistedRepositories(env: Env): Promise> { + const kv = env.PLUGIN_MARKETPLACE_BLACKLIST; + const blockedRepositories = new Set(); + if (!kv) { + return blockedRepositories; + } + + let cursor: string | undefined; + do { + const page = await kv.list({ prefix: BLACKLIST_REPO_KEY_PREFIX, cursor }); + for (const key of page.keys) { + const repository = key.name.slice(BLACKLIST_REPO_KEY_PREFIX.length).trim().toLowerCase(); + if (repository.includes("/")) { + blockedRepositories.add(repository); + } + } + cursor = page.cursor; + } while (cursor); + + return blockedRepositories; +} + function normalizeRepository(repo: GitHubRepository): PluginListing | null { if ( readBoolean(repo.disabled) || diff --git a/workers/plugin-marketplace/wrangler.toml b/workers/plugin-marketplace/wrangler.toml index 55b2e069..d80c2bfb 100644 --- a/workers/plugin-marketplace/wrangler.toml +++ b/workers/plugin-marketplace/wrangler.toml @@ -7,5 +7,9 @@ workers_dev = false binding = "PLUGIN_MARKETPLACE_BUCKET" bucket_name = "herdr-plugin-marketplace" +[[kv_namespaces]] +binding = "PLUGIN_MARKETPLACE_BLACKLIST" +id = "6504b3b84171492db56e805f6aad1686" + [triggers] crons = ["*/30 * * * *"]