/** * Gitea Repository Scanner for Backlog Aggregator * * Scans all repositories in a Gitea instance for backlog/ directories * and clones/pulls them to a local directory for the aggregator to watch. * * Usage: * bun run gitea-scanner.ts --gitea-url https://gitea.example.com --output /opt/gitea-repos * * Environment variables: * GITEA_URL - Gitea instance URL * GITEA_TOKEN - API token (optional, for private repos) * GITEA_OUTPUT_DIR - Directory to clone repos to */ import { $ } from "bun"; import { mkdir, readdir, stat, rm } from "node:fs/promises"; import { join } from "node:path"; interface GiteaRepo { id: number; name: string; full_name: string; clone_url: string; ssh_url: string; html_url: string; private: boolean; empty: boolean; archived: boolean; default_branch: string; } interface GiteaContent { name: string; path: string; type: "file" | "dir"; } interface ScannerConfig { giteaUrl: string; giteaToken?: string; outputDir: string; sshKeyPath?: string; owner?: string; // Optional: only scan repos from this owner concurrency: number; verbose: boolean; } class GiteaScanner { private config: ScannerConfig; private headers: Record; constructor(config: ScannerConfig) { this.config = config; this.headers = { "Accept": "application/json", }; if (config.giteaToken) { this.headers["Authorization"] = `token ${config.giteaToken}`; } } private async fetchJson(endpoint: string): Promise { const url = `${this.config.giteaUrl}/api/v1${endpoint}`; try { const response = await fetch(url, { headers: this.headers }); if (!response.ok) { if (this.config.verbose) { console.warn(`API request failed: ${url} (${response.status})`); } return null; } return await response.json() as T; } catch (error) { if (this.config.verbose) { console.warn(`API request error: ${url}`, error); } return null; } } async getAllRepos(): Promise { const allRepos: GiteaRepo[] = []; let page = 1; const limit = 50; while (true) { const endpoint = this.config.owner ? `/users/${this.config.owner}/repos?page=${page}&limit=${limit}` : `/repos/search?page=${page}&limit=${limit}`; const repos = await this.fetchJson(endpoint); if (!repos) break; // Handle both direct array and {data: []} response formats const repoList = Array.isArray(repos) ? repos : (repos.data || []); if (repoList.length === 0) break; // Filter out empty and archived repos const activeRepos = repoList.filter(r => !r.empty && !r.archived); allRepos.push(...activeRepos); if (repoList.length < limit) break; page++; } return allRepos; } async hasBacklogDir(repo: GiteaRepo): Promise { // Check if repo has a backlog/ directory at root const contents = await this.fetchJson( `/repos/${repo.full_name}/contents` ); if (!contents || !Array.isArray(contents)) return false; return contents.some(item => item.name === "backlog" && item.type === "dir"); } async cloneOrPullRepo(repo: GiteaRepo): Promise { const repoDir = join(this.config.outputDir, repo.name); try { // Check if already cloned const exists = await stat(repoDir).then(() => true).catch(() => false); if (exists) { // Pull latest changes if (this.config.verbose) { console.log(`Pulling ${repo.full_name}...`); } const result = await $`cd ${repoDir} && git pull --ff-only 2>&1`.quiet(); if (result.exitCode !== 0) { console.warn(`Failed to pull ${repo.full_name}: ${result.stderr}`); // Try to reset and pull await $`cd ${repoDir} && git fetch origin && git reset --hard origin/${repo.default_branch} 2>&1`.quiet(); } } else { // Clone the repo if (this.config.verbose) { console.log(`Cloning ${repo.full_name}...`); } // Use SSH URL if we have an SSH key configured, otherwise HTTPS const cloneUrl = this.config.sshKeyPath ? repo.ssh_url : repo.clone_url; const result = await $`git clone --depth 1 ${cloneUrl} ${repoDir} 2>&1`.quiet(); if (result.exitCode !== 0) { console.warn(`Failed to clone ${repo.full_name}: ${result.stderr}`); return false; } } return true; } catch (error) { console.error(`Error processing ${repo.full_name}:`, error); return false; } } async cleanupStaleRepos(validRepoNames: Set): Promise { try { const entries = await readdir(this.config.outputDir, { withFileTypes: true }); for (const entry of entries) { if (!entry.isDirectory()) continue; if (entry.name.startsWith(".")) continue; if (!validRepoNames.has(entry.name)) { const repoDir = join(this.config.outputDir, entry.name); console.log(`Removing stale repo: ${entry.name}`); await rm(repoDir, { recursive: true, force: true }); } } } catch (error) { console.warn("Error cleaning up stale repos:", error); } } async scan(): Promise<{ total: number; withBacklog: number; synced: number }> { console.log(`Scanning Gitea at ${this.config.giteaUrl}...`); // Ensure output directory exists await mkdir(this.config.outputDir, { recursive: true }); // Get all repos const repos = await this.getAllRepos(); console.log(`Found ${repos.length} repositories`); // Check which repos have backlog directories const reposWithBacklog: GiteaRepo[] = []; // Process in batches for concurrency control const batchSize = this.config.concurrency; for (let i = 0; i < repos.length; i += batchSize) { const batch = repos.slice(i, i + batchSize); const results = await Promise.all( batch.map(async (repo) => { const hasBacklog = await this.hasBacklogDir(repo); return { repo, hasBacklog }; }) ); for (const { repo, hasBacklog } of results) { if (hasBacklog) { reposWithBacklog.push(repo); if (this.config.verbose) { console.log(` ✓ ${repo.full_name} has backlog/`); } } } } console.log(`Found ${reposWithBacklog.length} repositories with backlog/`); // Clone or pull repos with backlog let synced = 0; for (const repo of reposWithBacklog) { const success = await this.cloneOrPullRepo(repo); if (success) synced++; } // Cleanup repos that no longer have backlog or were deleted const validNames = new Set(reposWithBacklog.map(r => r.name)); await this.cleanupStaleRepos(validNames); console.log(`Synced ${synced}/${reposWithBacklog.length} repositories`); return { total: repos.length, withBacklog: reposWithBacklog.length, synced, }; } } // Parse CLI arguments function parseArgs(): ScannerConfig { const args = process.argv.slice(2); const getArg = (name: string, envVar: string, defaultValue?: string): string | undefined => { const index = args.indexOf(`--${name}`); if (index !== -1 && args[index + 1]) { return args[index + 1]; } return process.env[envVar] || defaultValue; }; const giteaUrl = getArg("gitea-url", "GITEA_URL", "https://gitea.jeffemmett.com"); const giteaToken = getArg("gitea-token", "GITEA_TOKEN"); const outputDir = getArg("output", "GITEA_OUTPUT_DIR", "/opt/gitea-repos"); const sshKeyPath = getArg("ssh-key", "GITEA_SSH_KEY", "/root/.ssh/gitea_ed25519"); const owner = getArg("owner", "GITEA_OWNER", "jeffemmett"); const concurrency = parseInt(getArg("concurrency", "GITEA_CONCURRENCY", "5") || "5", 10); const verbose = args.includes("--verbose") || args.includes("-v"); if (!giteaUrl) { console.error("Error: --gitea-url or GITEA_URL is required"); process.exit(1); } if (!outputDir) { console.error("Error: --output or GITEA_OUTPUT_DIR is required"); process.exit(1); } return { giteaUrl, giteaToken, outputDir, sshKeyPath, owner, concurrency, verbose, }; } // Main entry point if (import.meta.main) { const config = parseArgs(); const scanner = new GiteaScanner(config); try { const result = await scanner.scan(); console.log("\nScan complete:"); console.log(` Total repos: ${result.total}`); console.log(` With backlog/: ${result.withBacklog}`); console.log(` Successfully synced: ${result.synced}`); } catch (error) { console.error("Scan failed:", error); process.exit(1); } } export { GiteaScanner, type ScannerConfig };