// Build the Kandev marketplace catalog index (index.json) from plugins.yaml. // // This is the "formulae.brew.sh"-style enrichment step of the plugin // marketplace: it reads the curated pointer list (plugins.yaml), resolves each // entry against the GitHub API to discover its latest release, package asset, // star count, last-push time, and manifest presentation metadata, and emits a // single static index.json document. That document is the fetch contract Kandev // consumes (docs/specs/plugins/requirements/marketplace.md → "Data model" → index.json); // additional corporate/team sources serve the same shape. // // Zero dependencies: Node stdlib + global fetch only, matching the repo's other // automation scripts (e.g. scripts/validate-public-docs.mjs). plugins.yaml is a // small, schema-constrained pointer list (plugin-registry/schema.json), so a // focused parser reads it without pulling a YAML library into CI. // // Robustness: one bad entry (missing release, deleted asset, API hiccup) never // fails the whole build — its error is logged to stderr and it is skipped. A // repo whose star lookup fails is emitted with `stars: null`, never `0`, so a // transient outage can't corrupt the catalog's ranking. // // Auth: GitHub API calls use GITHUB_TOKEN when present (in CI, secrets.GITHUB_TOKEN). // That works for the public repos here; at larger scale a PAT with `public_repo` // scope set as GITHUB_TOKEN gives higher, more predictable rate limits. import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; const GITHUB_API = "https://api.github.com"; const API_VERSION = "2022-11-28"; const USER_AGENT = "kandev-plugin-registry-index-builder"; // Bump only with a coordinated backend parser change — this is a hard contract. const SCHEMA_VERSION = 1; const SOURCE_NAME = "Kandev Official"; // The canonical Pages URL is filled in by the client from its source config, so // the document does not need to know where it is hosted. const SOURCE_URL = ""; const HERE = path.dirname(fileURLToPath(import.meta.url)); const PLUGINS_YAML = path.join(HERE, "plugins.yaml"); const OUTPUT_JSON = path.join(HERE, "index.json"); // --- Minimal plugins.yaml parser -------------------------------------------- /** * Parse the curated plugins list. plugins.yaml is intentionally a flat * `plugins:` sequence of maps with only scalar / inline-array fields * (id, repo, featured, categories), enforced by plugin-registry/schema.json — * so this focused reader is sufficient and avoids a YAML dependency in CI. * * @param {string} text Raw plugins.yaml contents. * @returns {Array>} The parsed plugin specs. */ export function parsePluginsYaml(text) { const specs = []; let current = null; let inPlugins = false; for (const rawLine of text.split("\n")) { const line = stripComment(rawLine); if (line.trim() === "") continue; if (!inPlugins) { if (line.trim() === "plugins:") inPlugins = true; continue; } const item = line.match(/^\s*-\s*(.*)$/); if (item) { current = {}; specs.push(current); assignField(current, item[1]); continue; } if (current && /^\s+\S/.test(line)) assignField(current, line.trim()); } return specs; } /** Drop a trailing `# comment` (values here never contain `#`). */ function stripComment(line) { const hash = line.indexOf("#"); return hash === -1 ? line : line.slice(0, hash); } /** Parse a `key: value` fragment onto target, coercing scalars/inline arrays. */ function assignField(target, fragment) { const colon = fragment.indexOf(":"); if (colon === -1) return; const key = fragment.slice(0, colon).trim(); if (!key) return; target[key] = parseScalar(fragment.slice(colon + 1).trim()); } function parseScalar(value) { if (value === "") return ""; if (value === "true") return true; if (value === "false") return false; if (value.startsWith("[") && value.endsWith("]")) { return value .slice(1, -1) .split(",") .map((part) => unquote(part.trim())) .filter((part) => part !== ""); } return unquote(value); } function unquote(value) { if (value.length >= 2 && (value[0] === '"' || value[0] === "'") && value[value.length - 1] === value[0]) { return value.slice(1, -1); } return value; } // --- GitHub helpers ---------------------------------------------------------- async function githubJson(apiPath) { const url = apiPath.startsWith("http") ? apiPath : `${GITHUB_API}${apiPath}`; const headers = { Accept: "application/vnd.github+json", "X-GitHub-Api-Version": API_VERSION, "User-Agent": USER_AGENT, }; if (process.env.GITHUB_TOKEN) headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`; const response = await fetchWithTimeout(url, { headers }); if (!response.ok) throw new Error(`GET ${url} -> ${response.status}`); return response.json(); } async function fetchWithTimeout(url, options = {}, timeoutMs = 30000) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { return await fetch(url, { ...options, signal: controller.signal }); } finally { clearTimeout(timer); } } const rawUrl = (repo, ref, filePath) => `https://raw.githubusercontent.com/${repo}/${ref}/${filePath.replace(/^\/+/, "")}`; /** * Fetch the plugin's manifest.yaml at the release tag for presentation metadata * (display_name/description/categories/min_kandev_version/icon). Returns {} on * any failure — the manifest only enriches presentation, so a miss must never * fail the entry. */ async function fetchManifest(repo, ref) { try { const response = await fetchWithTimeout(rawUrl(repo, ref, "manifest.yaml"), { headers: { "User-Agent": USER_AGENT }, }); if (!response.ok) throw new Error(`status ${response.status}`); return parseManifestFields(await response.text()); } catch (error) { console.error(`warning: ${repo}: manifest.yaml not read (${error.message})`); return {}; } } /** * Read just the top-level manifest fields the index needs. The manifest is a * larger YAML doc, so this only extracts the flat scalar keys we care about * (display_name, description, author, min_kandev_version, icon) plus the * `categories` inline/simple list — enough for presentation without a YAML lib. */ export function parseManifestFields(text) { const out = {}; const scalarKeys = ["display_name", "description", "author", "min_kandev_version", "icon"]; let inCategoryBlock = false; for (const rawLine of text.split("\n")) { const line = stripComment(rawLine); // Collect indented block-sequence items while inside `categories:`. const item = line.match(/^\s+-\s*(.+)$/); if (inCategoryBlock && item) { out.categories.push(unquote(item[1].trim())); continue; } const match = line.match(/^([a-z_]+):\s*(.*)$/); if (!match) continue; inCategoryBlock = false; const [, key, value] = match; const v = value.trim(); if (scalarKeys.includes(key) && v !== "") { out[key] = unquote(v); } else if (key === "categories") { if (v.startsWith("[")) { out.categories = parseScalar(v); // inline: [a, b] } else { out.categories = []; // block sequence: subsequent `- item` lines inCategoryBlock = true; } } } return out; } // --- Enrichment -------------------------------------------------------------- /** Strip a leading `v` from a release tag so versions compare cleanly. */ function normalizeVersion(tag) { return tag && /^v\d/.test(tag) ? tag.slice(1) : tag; } /** Pick this plugin's package tarball from the release assets. */ function pickPackageAsset(assets, pluginId, version) { const tarballs = assets.filter((a) => (a.name || "").endsWith(".tar.gz")); if (tarballs.length === 0) return { error: "release has no .tar.gz asset" }; const exact = tarballs.find((a) => a.name === `${pluginId}-${version}.tar.gz`); return { asset: exact || tarballs[0] }; } /** "agent-stats" -> "Agent Stats", the fallback when no manifest name is known. */ function humanize(pluginId) { return pluginId .split("-") .filter(Boolean) .map((part) => part[0].toUpperCase() + part.slice(1)) .join(" "); } /** * Resolve a single plugins.yaml spec into a full index.json record. * @returns {Promise<{record?: object, error?: string}>} */ export async function buildEntry(spec) { const pluginId = spec.id; const repo = spec.repo; if (!pluginId || !repo) return { error: `entry missing id/repo: ${JSON.stringify(spec)}` }; let release; try { release = await githubJson(`/repos/${repo}/releases/latest`); } catch (error) { return { error: `${pluginId}: no latest release (${error.message})` }; } const tag = release.tag_name || ""; const version = normalizeVersion(tag); const { asset, error: assetError } = pickPackageAsset(release.assets || [], pluginId, version); if (assetError) return { error: `${pluginId}: ${assetError}` }; const manifest = tag ? await fetchManifest(repo, tag) : {}; const iconUrl = manifest.icon && tag ? rawUrl(repo, tag, manifest.icon) : null; const meta = await fetchRepoMeta(repo, pluginId); const record = { id: pluginId, // Presentation prefers the plugin's manifest, then id-derived / release / // plugins.yaml fallbacks, so the contract shape is stable even on a miss. name: manifest.display_name || humanize(pluginId), description: manifest.description || release.name || "", author: manifest.author || meta.author, categories: manifest.categories || spec.categories || [], icon_url: iconUrl, repo_url: `https://github.com/${repo}`, version: version || null, min_kandev_version: manifest.min_kandev_version ?? null, package_url: asset.browser_download_url, // Advisory provenance digest; null is a valid, documented value. package_sha256: null, stars: meta.stars, updated_at: meta.updatedAt || release.published_at || null, }; return { record }; } /** Repo metadata → stars (null on failure, never 0), last-push time, owner. */ async function fetchRepoMeta(repo, pluginId) { try { const repoMeta = await githubJson(`/repos/${repo}`); return { stars: Number.isInteger(repoMeta.stargazers_count) ? repoMeta.stargazers_count : null, updatedAt: repoMeta.pushed_at || null, author: repoMeta.owner?.login || repo.split("/", 1)[0], }; } catch (error) { console.error(`warning: ${pluginId}: star/metadata lookup failed, emitting stars=null (${error.message})`); return { stars: null, updatedAt: null, author: repo.split("/", 1)[0] }; } } // --- Orchestration ----------------------------------------------------------- export async function buildIndex(specs) { const records = []; const errors = []; for (const spec of specs) { const { record, error } = await buildEntry(spec); if (error) { errors.push(error); console.error(`skip: ${error}`); continue; } records.push(record); } const document = { schema_version: SCHEMA_VERSION, generated_at: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"), source: { name: SOURCE_NAME, url: SOURCE_URL }, plugins: records, }; return { document, errors }; } async function main() { const text = await fs.readFile(PLUGINS_YAML, "utf8"); const specs = parsePluginsYaml(text); // An empty list is expected at launch (no plugin repos yet) — it produces a // valid, empty index.json and is NOT an error. Only a non-empty list that // resolves to zero entries (below) indicates a real failure. const { document, errors } = await buildIndex(specs); await fs.writeFile(OUTPUT_JSON, `${JSON.stringify(document, null, 2)}\n`, "utf8"); console.error( `Built index.json: ${document.plugins.length} built, ${errors.length} skipped, ` + `${specs.length} listed. Output: ${OUTPUT_JSON}`, ); // A non-empty list that produced zero entries is almost certainly a bad token // or total outage — fail so CI never publishes an empty catalog over a good one. if (specs.length > 0 && document.plugins.length === 0) { console.error("error: no entries could be built; refusing to publish empty index"); process.exitCode = 1; } } // Run only when invoked directly (not when imported by a test). if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { main().catch((error) => { console.error(`fatal: ${error.message}`); process.exitCode = 1; }); }