starred/herdr
1
0
Fork 0
mirror of https://github.com/herdrdev/herdr.git synced 2026-09-22 08:01:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: move marketplace worker to website repository

Browse Source
This commit is contained in:
Ogulcan Celik
2026-09-08 15:58:14 +03:00
parent 9e01168b14
commit ffa0892ef8
7 changed files with 0 additions and 2262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
justfile
-6
View File
@@ -9,7 +9,6 @@ test:
just maintenance-test
just ui-hot-path-architecture-test
just integration-assets-test
just plugin-marketplace-test
just docs-contract-test
# Run repository maintenance contract tests
@@ -41,7 +40,6 @@ ci filter='all()': lint
just maintenance-test
just ui-hot-path-architecture-test
just integration-assets-test
just plugin-marketplace-test
# Run Windows target lint from Unix/macOS to catch cfg(windows) compile and clippy failures before CI
[unix]
@@ -90,10 +88,6 @@ integration-assets-test:
bun test src/integration/assets/opencode/herdr-agent-state.test.ts
bun test src/integration/assets/opencode/herdr-tui-session.test.ts
# Run plugin marketplace Worker tests
plugin-marketplace-test:
cd workers/plugin-marketplace && bun install --frozen-lockfile && bun test
# Build the vendored libghostty-vt source dist
build-libghostty-vt:
scripts/build_vendored_libghostty_vt.sh
workers/plugin-marketplace/bun.lock
-15
View File
@@ -1,15 +0,0 @@
{
"lockfileVersion": 1,
"configVersion": 1,
"workspaces": {
"": {
"name": "herdr-plugin-marketplace-worker",
"dependencies": {
"smol-toml": "1.7.1",
},
},
},
"packages": {
"smol-toml": ["smol-toml@1.7.1", "", {}, "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ=="],
}
}
workers/plugin-marketplace/package.json
-13
View File
@@ -1,13 +0,0 @@
{
"name": "herdr-plugin-marketplace-worker",
"private": true,
"type": "module",
"scripts": {
"dev": "bunx wrangler dev",
"deploy": "bunx wrangler deploy",
"test": "bun test"
},
"dependencies": {
"smol-toml": "1.7.1"
}
}
workers/plugin-marketplace/r2-cors.json
-13
View File
@@ -1,13 +0,0 @@
{
"rules": [
{
"allowed": {
"origins": ["https://herdr.dev"],
"methods": ["GET", "HEAD"],
"headers": ["*"]
},
"exposeHeaders": ["Content-Type", "Cache-Control", "ETag"],
"maxAgeSeconds": 3600
}
]
}
workers/plugin-marketplace/src/index.test.ts
-924
View File
@@ -1,924 +0,0 @@
import { describe, expect, test } from "bun:test";
import worker, {
normalizeRepositories,
parseManifestSummary,
refreshPlugins,
type Env,
} from "./index";
const HEAD_COMMIT = "a".repeat(40);
const SECOND_COMMIT = "b".repeat(40);
type TreeFixture = {
path: string;
content?: string;
mode?: string;
type?: string;
size?: number;
};
class MemoryR2 {
objects = new Map<string, { value: string; options: unknown }>();
putCount = 0;
failPuts = false;
async put(key: string, value: string, options?: unknown): Promise<void> {
if (this.failPuts) throw new Error("simulated R2 put failure");
this.putCount += 1;
this.objects.set(key, { value, options });
}
async get(key: string): Promise<{ text(): Promise<string> } | null> {
const object = this.objects.get(key);
return object
? {
async text() {
return object.value;
},
}
: null;
}
}
class MemoryKV {
constructor(private readonly keyNames: string[]) {}
async list(options?: { prefix?: string }): Promise<{ keys: Array<{ name: string }> }> {
return {
keys: this.keyNames
.filter((name) => !options?.prefix || name.startsWith(options.prefix))
.map((name) => ({ name })),
};
}
}
function repo(overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
id: 1,
full_name: "ogulcancelik/herdr-plugin-example",
owner: { login: "ogulcancelik" },
name: "herdr-plugin-example",
description: "Example plugin repository",
html_url: "https://github.com/ogulcancelik/herdr-plugin-example",
default_branch: "main",
stargazers_count: 5,
forks_count: 1,
open_issues_count: 0,
language: "TypeScript",
topics: ["herdr-plugin"],
created_at: "2026-06-01T00:00:00Z",
updated_at: "2026-06-02T00:00:00Z",
pushed_at: "2026-06-03T00:00:00Z",
archived: false,
fork: false,
disabled: false,
private: false,
visibility: "public",
...overrides,
};
}
function manifest(overrides = ""): string {
return `
id = "example.plugin"
name = "Example Plugin"
version = "0.2.0"
min_herdr_version = "0.7.0"
description = "Example manifest"
platforms = ["linux", "macos"]
${overrides}`;
}
function env(bucket = new MemoryR2(), blacklist?: MemoryKV, backupBucket = new MemoryR2()): Env {
return {
PLUGIN_MARKETPLACE_BUCKET: bucket,
PLUGIN_MARKETPLACE_BACKUP_BUCKET: backupBucket,
PLUGIN_MARKETPLACE_BLACKLIST: blacklist,
GITHUB_TOKEN: "token",
};
}
function repositoryFetch(options: {
repositories: Record<string, unknown>[];
trees?: Record<string, TreeFixture[]>;
commits?: Record<string, string>;
totalCount?: number;
incompleteResults?: boolean;
searchStatus?: number;
treeStatus?: Record<string, number>;
truncatedTrees?: Set<string>;
onRequest?: (kind: "search" | "head" | "tree" | "manifest", detail: string) => void;
}): typeof fetch {
return (async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
const url = new URL(input.toString());
if (url.pathname === "/search/repositories") {
options.onRequest?.("search", url.toString());
if (options.searchStatus) return new Response("search failed", { status: options.searchStatus });
return Response.json({
total_count: options.totalCount ?? options.repositories.length,
incomplete_results: options.incompleteResults ?? false,
items: options.repositories,
});
}
const treeMatch = url.pathname.match(/^\/repos\/([^/]+)\/([^/]+)\/git\/trees\/([^/]+)$/);
if (treeMatch) {
const fullName = `${decodeURIComponent(treeMatch[1])}/${decodeURIComponent(treeMatch[2])}`;
options.onRequest?.("tree", fullName);
const status = options.treeStatus?.[fullName];
if (status) return new Response("tree failed", { status });
const fixtures = options.trees?.[fullName] ?? [
{ path: "herdr-plugin.toml", content: manifest() },
];
return Response.json({
sha: "tree-sha",
truncated: options.truncatedTrees?.has(fullName) ?? false,
tree: fixtures.map((fixture) => ({
path: fixture.path,
mode: fixture.mode ?? "100644",
type: fixture.type ?? "blob",
size:
fixture.size ??
new TextEncoder().encode(fixture.content ?? "").byteLength,
})),
});
}
if (url.pathname === "/graphql") {
const request = JSON.parse(String(init?.body ?? "{}"));
const query = String(request.query ?? "");
const data: Record<string, unknown> = {};
if (query.includes("PluginMarketplaceHeads")) {
options.onRequest?.("head", query);
for (const match of query.matchAll(
/repo(\d+): repository\(owner: "([^"]+)", name: "([^"]+)"\)/g,
)) {
const [, alias, owner, name] = match;
const fullName = `${owner}/${name}`;
const repository = options.repositories.find(
(candidate) => candidate.full_name === fullName,
);
data[`repo${alias}`] = repository
? {
defaultBranchRef: {
name: repository.default_branch ?? "main",
target: { oid: options.commits?.[fullName] ?? HEAD_COMMIT },
},
}
: null;
}
} else if (query.includes("PluginMarketplaceManifests")) {
options.onRequest?.("manifest", query);
for (const match of query.matchAll(
/item(\d+): repository\(owner: "([^"]+)", name: "([^"]+)"\) \{\s+manifest: object\(expression: "[a-f0-9]+:([^"]+)"\)/g,
)) {
const [, alias, owner, name, path] = match;
const fullName = `${owner}/${name}`;
const fixture = (options.trees?.[fullName] ?? [
{ path: "herdr-plugin.toml", content: manifest() },
]).find((entry) => entry.path === path);
data[`item${alias}`] = fixture?.content === undefined
? { manifest: null }
: { manifest: { text: fixture.content } };
}
} else {
throw new Error(`unexpected GraphQL query: ${query}`);
}
return Response.json({ data });
}
throw new Error(`unexpected request: ${url}`);
}) as typeof fetch;
}
describe("normalizeRepositories", () => {
test("normalizes fields while preserving repository-card ordering", () => {
const plugins = normalizeRepositories([
repo({
id: 2,
full_name: "other/newer",
owner: { login: "other" },
name: "newer",
html_url: "https://github.com/other/newer",
stargazers_count: 5,
pushed_at: "2026-06-04T00:00:00Z",
}),
repo(),
]);
expect(plugins.map((plugin) => plugin.fullName)).toEqual([
"other/newer",
"ogulcancelik/herdr-plugin-example",
]);
expect(plugins[1]).toMatchObject({
id: 1,
owner: "ogulcancelik",
name: "herdr-plugin-example",
defaultBranch: "main",
stars: 5,
});
});
test("deduplicates repositories by immutable GitHub id", () => {
const plugins = normalizeRepositories([
repo(),
repo({ full_name: "duplicate/name", owner: { login: "duplicate" }, name: "name", html_url: "https://github.com/duplicate/name" }),
]);
expect(plugins).toHaveLength(1);
expect(plugins[0].fullName).toBe("ogulcancelik/herdr-plugin-example");
});
test("drops unsafe, unavailable, and default-branch-less repositories", () => {
const plugins = normalizeRepositories([
repo({ html_url: "https://example.com/ogulcancelik/herdr-plugin-example" }),
repo({ archived: true }),
repo({ fork: true }),
repo({ disabled: true }),
repo({ private: true }),
repo({ visibility: "private" }),
repo({ default_branch: undefined }),
repo({ id: 5 }),
]);
expect(plugins.map((plugin) => plugin.id)).toEqual([5]);
});
});
describe("parseManifestSummary", () => {
test("extracts metadata and accepts the UTF-8 BOM accepted by Herdr", () => {
expect(parseManifestSummary(`\uFEFF${manifest()}`)).toEqual({
id: "example.plugin",
name: "Example Plugin",
version: "0.2.0",
minHerdrVersion: "0.7.0",
description: "Example manifest",
platforms: ["linux", "macos"],
});
});
test("rejects malformed TOML and invalid required metadata", () => {
expect(parseManifestSummary("not = [valid")).toBeNull();
expect(parseManifestSummary(manifest().replace("example.plugin", "bad/plugin"))).toBeNull();
expect(parseManifestSummary(manifest().replace("0.7.0", "next"))).toBeNull();
expect(parseManifestSummary(manifest().replace('["linux", "macos"]', "[]"))).toBeNull();
expect(
parseManifestSummary(manifest().replace("Example Plugin", "n".repeat(121))),
).toBeNull();
});
});
describe("refreshPlugins", () => {
test("publishes one backward-compatible repository card with multiple manifests", async () => {
const bucket = new MemoryR2();
const fullName = "ogulcancelik/herdr-plugin-example";
const fetch = repositoryFetch({
repositories: [repo()],
trees: {
[fullName]: [
{ path: "herdr-plugin.toml", content: manifest() },
{
path: "plugins/second/herdr-plugin.toml",
content: manifest().replace("example.plugin", "example.second").replace("Example Plugin", "Second Plugin"),
},
],
},
});
const result = await refreshPlugins(env(bucket), {
fetch,
now: new Date("2026-06-20T12:00:00.000Z"),
logger: { error() {} },
});
expect(result.ok).toBe(true);
const snapshotObject = bucket.objects.get("plugins/index.json");
const snapshot = JSON.parse(snapshotObject?.value ?? "");
expect(snapshotObject?.options).toEqual({
httpMetadata: {
contentType: "application/json; charset=utf-8",
cacheControl: "public, max-age=300, s-maxage=1800, stale-while-revalidate=3600",
},
});
expect(snapshot).toMatchObject({
schemaVersion: 1,
generatedAt: "2026-06-20T12:00:00.000Z",
pluginCount: 2,
repositoryCount: 1,
source: {
missingManifestCount: 0,
invalidManifestCount: 0,
},
});
expect(snapshot.plugins[0]).toMatchObject({
id: 1,
fullName,
name: "herdr-plugin-example",
headCommit: HEAD_COMMIT,
manifests: [
{ path: "herdr-plugin.toml", id: "example.plugin" },
{ path: "plugins/second/herdr-plugin.toml", id: "example.second" },
],
});
expect(snapshot.plugins[0]).not.toHaveProperty("defaultBranch");
expect(snapshot.plugins[0]).toMatchObject({
firstSeenAt: "2026-06-20T12:00:00.000Z",
starsDelta7d: null,
starsDelta30d: null,
});
expect(bucket.objects.has("plugins/scan-cache.json")).toBe(true);
const history = JSON.parse(bucket.objects.get("plugins/star-history.json")?.value ?? "");
expect(history).toEqual({
schemaVersion: 1,
entries: [
{
repositoryId: 1,
fullName,
firstSeenAt: "2026-06-20T12:00:00.000Z",
samples: [{ date: "2026-06-20", stars: 5 }],
},
],
});
});
test("reuses cached manifests without resolving or rescanning an unchanged repository", async () => {
const bucket = new MemoryR2();
const requests: string[] = [];
const fetch = repositoryFetch({
repositories: [repo()],
onRequest(kind) {
requests.push(kind);
},
});
expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true);
requests.length = 0;
expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true);
expect(requests).toEqual(["search"]);
});
test("rescans a repository when its pushed timestamp and head change", async () => {
const bucket = new MemoryR2();
const repository = repo();
const fullName = String(repository.full_name);
const commits = { [fullName]: HEAD_COMMIT };
const trees = { [fullName]: [{ path: "herdr-plugin.toml", content: manifest() }] };
const fetch = repositoryFetch({ repositories: [repository], commits, trees });
expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true);
commits[fullName] = SECOND_COMMIT;
repository.pushed_at = "2026-06-04T00:00:00Z";
trees[fullName][0].content = manifest().replace("0.2.0", "0.3.0");
const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } });
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.plugins[0].headCommit).toBe(SECOND_COMMIT);
expect(result.snapshot.plugins[0].manifests[0].version).toBe("0.3.0");
});
test("counts manifests separately from repository cards and omits empty cards", async () => {
const repositories = [
repo(),
repo({
id: 2,
full_name: "example/empty",
owner: { login: "example" },
name: "empty",
html_url: "https://github.com/example/empty",
}),
repo({
id: 3,
full_name: "example/invalid",
owner: { login: "example" },
name: "invalid",
html_url: "https://github.com/example/invalid",
}),
];
const result = await refreshPlugins(env(), {
fetch: repositoryFetch({
repositories,
trees: {
"ogulcancelik/herdr-plugin-example": [
{ path: "one/herdr-plugin.toml", content: manifest() },
{ path: "two/herdr-plugin.toml", content: manifest().replace("example.plugin", "example.two") },
],
"example/empty": [{ path: "README.md", content: "empty" }],
"example/invalid": [{ path: "herdr-plugin.toml", content: "id = [broken" }],
},
}),
logger: { error() {} },
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.pluginCount).toBe(2);
expect(result.snapshot.repositoryCount).toBe(1);
expect(result.snapshot.source.missingManifestCount).toBe(1);
expect(result.snapshot.source.invalidManifestCount).toBe(1);
});
test("ignores manifest symlinks and indexes their regular target", async () => {
const fullName = "ogulcancelik/herdr-plugin-example";
const result = await refreshPlugins(env(), {
fetch: repositoryFetch({
repositories: [repo()],
trees: {
[fullName]: [
{ path: "herdr-plugin.toml", content: "plugin/herdr-plugin.toml", mode: "120000" },
{ path: "plugin/herdr-plugin.toml", content: manifest() },
],
},
}),
logger: { error() {} },
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.plugins[0].manifests.map((item) => item.path)).toEqual([
"plugin/herdr-plugin.toml",
]);
expect(result.snapshot.source.invalidManifestCount).toBe(0);
});
test("ignores test fixtures and deduplicates repeated plugin ids", async () => {
const fullName = "ogulcancelik/herdr-plugin-example";
const result = await refreshPlugins(env(), {
fetch: repositoryFetch({
repositories: [repo()],
trees: {
[fullName]: [
{ path: "herdr-plugin.toml", content: manifest() },
{ path: "platform/herdr-plugin.toml", content: manifest() },
{
path: "tests/fixtures/helper/herdr-plugin.toml",
content: manifest().replace("example.plugin", "example.fixture"),
},
],
},
}),
logger: { error() {} },
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.plugins[0].manifests.map((item) => item.path)).toEqual([
"herdr-plugin.toml",
]);
expect(result.snapshot.source.duplicateManifestCount).toBe(1);
});
test("skips a truncated tree without blocking the marketplace", async () => {
const fullName = "ogulcancelik/herdr-plugin-example";
const result = await refreshPlugins(env(), {
fetch: repositoryFetch({
repositories: [repo()],
truncatedTrees: new Set([fullName]),
}),
logger: { error() {} },
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.plugins).toEqual([]);
expect(result.snapshot.source.skippedRepositoryCount).toBe(1);
expect(result.snapshot.source.warnings?.[0]).toContain(fullName);
});
test("writes an empty snapshot when every repository is blacklisted", async () => {
const bucket = new MemoryR2();
const result = await refreshPlugins(
env(bucket, new MemoryKV(["repo:ogulcancelik/herdr-plugin-example"])),
{ fetch: repositoryFetch({ repositories: [repo()] }), logger: { error() {} } },
);
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.plugins).toEqual([]);
expect(result.snapshot.source.blacklistedCount).toBe(1);
});
test("pins tree and manifest reads to the resolved commit", async () => {
const requests: Array<[string, string]> = [];
const result = await refreshPlugins(env(), {
fetch: repositoryFetch({
repositories: [repo()],
onRequest(kind, detail) {
requests.push([kind, detail]);
},
}),
logger: { error() {} },
});
expect(result.ok).toBe(true);
expect(requests.find(([kind]) => kind === "tree")?.[1]).toBe(
"ogulcancelik/herdr-plugin-example",
);
const manifestQuery = requests.find(([kind]) => kind === "manifest")?.[1] ?? "";
expect(manifestQuery).toContain(`${HEAD_COMMIT}:herdr-plugin.toml`);
expect(manifestQuery).not.toContain("main:herdr-plugin.toml");
});
test("does not overwrite the public snapshot when scanning fails", async () => {
const bucket = new MemoryR2();
await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}');
const result = await refreshPlugins(env(bucket), {
fetch: repositoryFetch({
repositories: [repo()],
treeStatus: { "ogulcancelik/herdr-plugin-example": 429 },
}),
logger: { error() {} },
});
expect(result.ok).toBe(false);
expect(bucket.objects.get("plugins/index.json")?.value).toBe(
'{"schemaVersion":1,"plugins":[{"id":1}]}',
);
});
test("discards malformed cache state and rebuilds it", async () => {
const bucket = new MemoryR2();
await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}');
await bucket.put("plugins/scan-cache.json", "broken");
const errors: string[] = [];
const result = await refreshPlugins(env(bucket), {
fetch: repositoryFetch({ repositories: [repo()] }),
logger: { error(message) { errors.push(String(message)); } },
});
expect(result.ok).toBe(true);
expect(errors[0]).toContain("discarding invalid plugin marketplace scan cache");
expect(JSON.parse(bucket.objects.get("plugins/scan-cache.json")?.value ?? "").entries).toHaveLength(1);
});
test("rejects an implausible empty search when a healthy cache exists", async () => {
const bucket = new MemoryR2();
const normalFetch = repositoryFetch({ repositories: [repo()] });
expect((await refreshPlugins(env(bucket), { fetch: normalFetch, logger: { error() {} } })).ok).toBe(true);
const previous = bucket.objects.get("plugins/index.json")?.value;
const result = await refreshPlugins(env(bucket), {
fetch: repositoryFetch({ repositories: [] }),
logger: { error() {} },
});
expect(result.ok).toBe(false);
expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous);
});
test("publishes an empty snapshot when the initial complete search is empty", async () => {
const bucket = new MemoryR2();
const result = await refreshPlugins(env(bucket), {
fetch: repositoryFetch({ repositories: [] }),
logger: { error() {} },
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.plugins).toEqual([]);
expect(result.snapshot.pluginCount).toBe(0);
});
test("preserves the snapshot when head resolution omits a cached repository", async () => {
const bucket = new MemoryR2();
const repository = repo();
const baseFetch = repositoryFetch({ repositories: [repository] });
expect((await refreshPlugins(env(bucket), { fetch: baseFetch, logger: { error() {} } })).ok).toBe(true);
const previous = bucket.objects.get("plugins/index.json")?.value;
repository.pushed_at = "2026-06-04T00:00:00Z";
const missingHeadFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
const request = String(init?.body ?? "");
if (new URL(input.toString()).pathname === "/graphql" && request.includes("PluginMarketplaceHeads")) {
return Response.json({
data: { repo0: null },
errors: [{ type: "NOT_FOUND", path: ["repo0"], message: "not found" }],
});
}
return baseFetch(input, init);
};
const result = await refreshPlugins(env(bucket), {
fetch: missingHeadFetch as typeof fetch,
logger: { error() {} },
});
expect(result.ok).toBe(false);
expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous);
});
test("does not cache a transiently missing manifest response", async () => {
const bucket = new MemoryR2();
const repository = repo();
const fullName = String(repository.full_name);
const commits = { [fullName]: HEAD_COMMIT };
const baseFetch = repositoryFetch({ repositories: [repository], commits });
expect((await refreshPlugins(env(bucket), { fetch: baseFetch, logger: { error() {} } })).ok).toBe(true);
const previous = bucket.objects.get("plugins/index.json")?.value;
repository.pushed_at = "2026-06-04T00:00:00Z";
commits[fullName] = SECOND_COMMIT;
const missingFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
const request = String(init?.body ?? "");
if (new URL(input.toString()).pathname === "/graphql" && request.includes("PluginMarketplaceManifests")) {
return Response.json({
data: { item0: { manifest: null } },
errors: [{ type: "NOT_FOUND", path: ["item0"], message: "not found" }],
});
}
return baseFetch(input, init);
};
const result = await refreshPlugins(env(bucket), {
fetch: missingFetch as typeof fetch,
logger: { error() {} },
});
expect(result.ok).toBe(false);
expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous);
});
test("retries budget-skipped repositories when capacity becomes available", async () => {
const repositories = Array.from({ length: 51 }, (_, index) =>
repo({
id: index + 1,
full_name: `owner/repo-${index}`,
owner: { login: "owner" },
name: `repo-${index}`,
html_url: `https://github.com/owner/repo-${index}`,
}),
);
const trees = Object.fromEntries(
repositories.map((repository) => [
repository.full_name,
Array.from({ length: 100 }, (_, index) => ({
path: `plugins/${index}/herdr-plugin.toml`,
content: manifest(),
})),
]),
);
const bucket = new MemoryR2();
const fetch = repositoryFetch({ repositories, trees });
const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } });
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.source.skippedRepositoryCount).toBe(1);
expect(result.snapshot.source.warnings?.[0]).toContain("scan budget was exhausted");
repositories.shift();
const recovered = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } });
expect(recovered.ok).toBe(true);
if (!recovered.ok) return;
expect(recovered.snapshot.source.skippedRepositoryCount).toBe(0);
expect(recovered.snapshot.repositoryCount).toBe(50);
});
test("keeps one star sample per UTC day with the first observation winning", async () => {
const bucket = new MemoryR2();
const backupBucket = new MemoryR2();
const repository = repo({ stargazers_count: 5 });
const logger = { error() {} };
const run = (stars: number, now: string) => {
repository.stargazers_count = stars;
return refreshPlugins(env(bucket, undefined, backupBucket), {
fetch: repositoryFetch({ repositories: [repository] }),
now: new Date(now),
logger,
});
};
expect((await run(5, "2026-06-20T00:10:00.000Z")).ok).toBe(true);
expect((await run(9, "2026-06-20T23:50:00.000Z")).ok).toBe(true);
const sameDay = JSON.parse(bucket.objects.get("plugins/star-history.json")?.value ?? "");
expect(sameDay.entries[0].samples).toEqual([{ date: "2026-06-20", stars: 5 }]);
const nextDay = await run(12, "2026-06-21T00:10:00.000Z");
expect(nextDay.ok).toBe(true);
if (!nextDay.ok) return;
expect(nextDay.snapshot.plugins[0].firstSeenAt).toBe("2026-06-20T00:10:00.000Z");
const history = JSON.parse(bucket.objects.get("plugins/star-history.json")?.value ?? "");
expect(history.entries[0].samples).toEqual([
{ date: "2026-06-20", stars: 5 },
{ date: "2026-06-21", stars: 12 },
]);
// One dated backup per UTC day, written on the first run of that day and
// never touched by later intraday runs.
expect(backupBucket.putCount).toBe(2);
expect([...backupBucket.objects.keys()]).toEqual([
"backups/star-history/2026-06-20.json",
"backups/star-history/2026-06-21.json",
]);
const firstBackup = JSON.parse(
backupBucket.objects.get("backups/star-history/2026-06-20.json")?.value ?? "",
);
expect(firstBackup.entries[0].samples).toEqual([{ date: "2026-06-20", stars: 5 }]);
});
test("fails the refresh without touching the primary bucket when the backup put fails", async () => {
const bucket = new MemoryR2();
await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}');
const backupBucket = new MemoryR2();
backupBucket.failPuts = true;
const errors: string[] = [];
const options = {
fetch: repositoryFetch({ repositories: [repo()] }),
now: new Date("2026-06-20T12:00:00.000Z"),
logger: { error(message: unknown) { errors.push(String(message)); } },
};
const failed = await refreshPlugins(env(bucket, undefined, backupBucket), options);
expect(failed.ok).toBe(false);
expect(errors[0]).toContain("simulated R2 put failure");
expect(bucket.objects.get("plugins/index.json")?.value).toBe(
'{"schemaVersion":1,"plugins":[{"id":1}]}',
);
expect(bucket.objects.has("plugins/star-history.json")).toBe(false);
expect(bucket.objects.has("plugins/scan-cache.json")).toBe(false);
backupBucket.failPuts = false;
const retried = await refreshPlugins(env(bucket, undefined, backupBucket), options);
expect(retried.ok).toBe(true);
expect(backupBucket.objects.has("backups/star-history/2026-06-20.json")).toBe(true);
expect(bucket.objects.has("plugins/star-history.json")).toBe(true);
});
test("computes star deltas from near-boundary baselines and rejects stale ones", async () => {
const bucket = new MemoryR2();
await bucket.put(
"plugins/star-history.json",
JSON.stringify({
schemaVersion: 1,
entries: [
{
repositoryId: 1,
fullName: "ogulcancelik/herdr-plugin-example",
firstSeenAt: "2026-05-01T00:00:00.000Z",
samples: [
{ date: "2026-05-01", stars: 10 },
{ date: "2026-05-20", stars: 30 },
{ date: "2026-06-13", stars: 40 },
{ date: "2026-06-19", stars: 90 },
],
},
{
repositoryId: 2,
fullName: "example/rejoined",
firstSeenAt: "2026-05-01T00:00:00.000Z",
samples: [{ date: "2026-05-01", stars: 3 }],
},
],
}),
);
const result = await refreshPlugins(env(bucket), {
fetch: repositoryFetch({
repositories: [
repo({ stargazers_count: 100 }),
repo({
id: 2,
full_name: "example/rejoined",
owner: { login: "example" },
name: "rejoined",
html_url: "https://github.com/example/rejoined",
stargazers_count: 50,
}),
],
}),
now: new Date("2026-06-20T12:00:00.000Z"),
logger: { error() {} },
});
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.snapshot.plugins[0]).toMatchObject({
firstSeenAt: "2026-05-01T00:00:00.000Z",
starsDelta7d: 60,
starsDelta30d: 70,
});
// The rejoined repository only has a six-week-old sample: far past both
// window boundaries, so no delta is reported instead of an inflated one.
expect(result.snapshot.plugins[1]).toMatchObject({
fullName: "example/rejoined",
starsDelta7d: null,
starsDelta30d: null,
});
});
test("prunes aged samples and keeps history for delisted repositories", async () => {
const bucket = new MemoryR2();
await bucket.put(
"plugins/star-history.json",
JSON.stringify({
schemaVersion: 1,
entries: [
{
repositoryId: 1,
fullName: "ogulcancelik/herdr-plugin-example",
firstSeenAt: "2026-01-01T00:00:00.000Z",
samples: [
{ date: "2026-01-01", stars: 1 },
{ date: "2026-06-19", stars: 4 },
],
},
{
repositoryId: 2,
fullName: "example/delisted-recently",
firstSeenAt: "2026-01-01T00:00:00.000Z",
samples: [
{ date: "2026-01-05", stars: 5 },
{ date: "2026-06-01", stars: 7 },
],
},
{
repositoryId: 3,
fullName: "example/delisted-long-ago",
firstSeenAt: "2026-01-01T00:00:00.000Z",
samples: [{ date: "2026-01-05", stars: 3 }],
},
],
}),
);
const result = await refreshPlugins(env(bucket), {
fetch: repositoryFetch({ repositories: [repo()] }),
now: new Date("2026-06-20T12:00:00.000Z"),
logger: { error() {} },
});
expect(result.ok).toBe(true);
const history = JSON.parse(bucket.objects.get("plugins/star-history.json")?.value ?? "");
expect(history.entries.map((entry: { repositoryId: number }) => entry.repositoryId)).toEqual([
1, 2,
]);
expect(history.entries[0].samples.map((sample: { date: string }) => sample.date)).toEqual([
"2026-06-19",
"2026-06-20",
]);
expect(history.entries[1].samples).toEqual([{ date: "2026-06-01", stars: 7 }]);
});
for (const { name, record } of [
{ name: "malformed JSON", record: "broken" },
{ name: "an unsupported shape", record: '{"schemaVersion":2,"entries":[]}' },
{
name: "an invalid entry",
record: JSON.stringify({
schemaVersion: 1,
entries: [{ repositoryId: -5, samples: "broken" }],
}),
},
]) {
test(`fails the refresh without overwriting anything when star history has ${name}`, async () => {
const bucket = new MemoryR2();
await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}');
await bucket.put("plugins/star-history.json", record);
const backupBucket = new MemoryR2();
const errors: string[] = [];
const result = await refreshPlugins(env(bucket, undefined, backupBucket), {
fetch: repositoryFetch({ repositories: [repo()] }),
now: new Date("2026-06-20T12:00:00.000Z"),
logger: { error(message: unknown) { errors.push(String(message)); } },
});
expect(result.ok).toBe(false);
expect(errors[0]).toContain("invalid plugin marketplace star history");
expect(bucket.objects.get("plugins/star-history.json")?.value).toBe(record);
expect(bucket.objects.get("plugins/index.json")?.value).toBe(
'{"schemaVersion":1,"plugins":[{"id":1}]}',
);
expect(bucket.objects.has("plugins/scan-cache.json")).toBe(false);
// Broken history must never be preserved as a "backup".
expect(backupBucket.objects.size).toBe(0);
});
}
for (const { name, fetch } of [
{
name: "GitHub failure",
fetch: repositoryFetch({ repositories: [], searchStatus: 429 }),
},
{
name: "incomplete search results",
fetch: repositoryFetch({ repositories: [repo()], incompleteResults: true }),
},
]) {
test(`preserves the current snapshot on ${name}`, async () => {
const bucket = new MemoryR2();
await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}');
const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } });
expect(result.ok).toBe(false);
expect(bucket.objects.get("plugins/index.json")?.value).toBe(
'{"schemaVersion":1,"plugins":[{"id":1}]}',
);
});
}
});
describe("fetch handler", () => {
test("does not expose a public Worker API", async () => {
const response = await worker.fetch(new Request("https://herdr.dev/api/plugins"), env());
expect(response.status).toBe(404);
expect(response.headers.get("Cache-Control")).toBe("no-store");
});
});
workers/plugin-marketplace/src/index.ts
-1269
View File
File diff suppressed because it is too large Load Diff
workers/plugin-marketplace/wrangler.toml
-22
View File
@@ -1,22 +0,0 @@
name = "herdr-plugin-marketplace"
main = "src/index.ts"
compatibility_date = "2026-06-21"
workers_dev = false
[limits]
subrequests = 3500
[[r2_buckets]]
binding = "PLUGIN_MARKETPLACE_BUCKET"
bucket_name = "herdr-plugin-marketplace"
[[r2_buckets]]
binding = "PLUGIN_MARKETPLACE_BACKUP_BUCKET"
bucket_name = "herdr-plugin-marketplace-backup"
[[kv_namespaces]]
binding = "PLUGIN_MARKETPLACE_BLACKLIST"
id = "6504b3b84171492db56e805f6aad1686"
[triggers]
crons = ["*/30 * * * *"]