From 4bfe1c82ff6fa338b06db87e31ddac820da484a3 Mon Sep 17 00:00:00 2001 From: Jeff Emmett Date: Fri, 10 Apr 2026 15:13:53 -0400 Subject: [PATCH] =?UTF-8?q?fix:=20enable=20Gitea=20scanner=20for=20aggrega?= =?UTF-8?q?tor=20=E2=80=94=20poll=20all=20repos=20with=20backlogs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch docker-compose to Dockerfile.aggregator (includes git, cron, openssh) - Remove command override so entrypoint.sh runs Gitea scanner on startup - Use HTTPS cloning with token auth for private repos - CI now builds from Dockerfile.aggregator and syncs docker-compose on deploy - Cron rescans Gitea every 6 hours; aggregator now tracks 120 projects / 780 tasks Co-Authored-By: Claude Opus 4.6 --- .gitea/workflows/ci.yml | 3 ++- Dockerfile.aggregator | 4 ++-- docker-compose.yml | 17 +++++++++++++++-- entrypoint.sh | 13 ++++++++++--- src/aggregator/gitea-scanner.ts | 19 +++++++++++++------ 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 39d4453..bd8721f 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - name: Build and push image run: | - docker build -t ${{ env.IMAGE }}:${{ env.IMAGE_TAG }} -t ${{ env.IMAGE }}:latest . + docker build -f Dockerfile.aggregator -t ${{ env.IMAGE }}:${{ env.IMAGE_TAG }} -t ${{ env.IMAGE }}:latest . echo "${{ secrets.REGISTRY_TOKEN }}" | docker login ${{ env.REGISTRY }} -u ${{ secrets.REGISTRY_USER }} --password-stdin docker push ${{ env.IMAGE }}:${{ env.IMAGE_TAG }} docker push ${{ env.IMAGE }}:latest @@ -41,6 +41,7 @@ jobs: mkdir -p ~/.ssh echo "${{ secrets.DEPLOY_SSH_KEY }}" | base64 -d > ~/.ssh/deploy_key chmod 600 ~/.ssh/deploy_key + scp -o StrictHostKeyChecking=no -i ~/.ssh/deploy_key docker-compose.yml root@${{ secrets.DEPLOY_HOST }}:/opt/apps/backlog-md/docker-compose.yml ssh -o StrictHostKeyChecking=no -i ~/.ssh/deploy_key root@${{ secrets.DEPLOY_HOST }} " cd /opt/apps/backlog-md cat .last-deployed-tag 2>/dev/null > .rollback-tag || true diff --git a/Dockerfile.aggregator b/Dockerfile.aggregator index 90391d8..ddcd6ae 100644 --- a/Dockerfile.aggregator +++ b/Dockerfile.aggregator @@ -24,8 +24,8 @@ RUN bun run build:css || true # Make entrypoint executable RUN chmod +x /app/entrypoint.sh || true -# Create cron job for daily Gitea sync (runs at 2 AM and 2 PM) -RUN echo "0 2,14 * * * cd /app && bun run src/aggregator/gitea-scanner.ts --verbose >> /var/log/gitea-scanner.log 2>&1" > /etc/cron.d/gitea-scanner \ +# Create cron job for Gitea sync (every 6 hours) +RUN echo "0 */6 * * * GIT_SSH_COMMAND='ssh -i /tmp/gitea_ed25519 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' cd /app && bun run src/aggregator/gitea-scanner.ts --verbose >> /var/log/gitea-scanner.log 2>&1" > /etc/cron.d/gitea-scanner \ && chmod 0644 /etc/cron.d/gitea-scanner \ && crontab /etc/cron.d/gitea-scanner diff --git a/docker-compose.yml b/docker-compose.yml index 88d7748..761aebf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,37 @@ services: backlog: - build: . + image: localhost:3000/jeffemmett/backlog-md:${IMAGE_TAG:-latest} + build: + context: . + dockerfile: Dockerfile.aggregator container_name: backlog-aggregator restart: unless-stopped volumes: - /opt/websites:/projects/websites - /opt/apps:/projects/apps - /opt/gitea-repos:/projects/gitea - command: ["bun", "src/cli.ts", "aggregator", "--port", "6420", "--paths", "/projects/websites,/projects/apps,/projects/gitea"] labels: - "traefik.enable=true" - "traefik.http.routers.backlog.rule=Host(`backlog.jeffemmett.com`)" - "traefik.http.routers.backlog.entrypoints=web" - "traefik.http.services.backlog.loadbalancer.server.port=6420" - "traefik.docker.network=traefik-public" + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:6420/"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s networks: - traefik-public + env_file: + - .env environment: - PORT=6420 - NODE_ENV=production + - GITEA_URL=https://gitea.jeffemmett.com + - GITEA_OUTPUT_DIR=/projects/gitea + - GITEA_OWNER=jeffemmett networks: traefik-public: diff --git a/entrypoint.sh b/entrypoint.sh index 7a51d12..35a6b9a 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,12 +4,19 @@ set -e # Start cron daemon cron -# Configure SSH for git -export GIT_SSH_COMMAND="ssh -i /root/.ssh/gitea_ed25519 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" +# Configure SSH for git — copy mounted key so we can fix permissions +if [ -f /root/.ssh/gitea_ed25519 ]; then + cp /root/.ssh/gitea_ed25519 /tmp/gitea_ed25519 + chmod 600 /tmp/gitea_ed25519 + export GIT_SSH_COMMAND="ssh -i /tmp/gitea_ed25519 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + echo "SSH key configured for Gitea access" +else + echo "WARNING: No SSH key found at /root/.ssh/gitea_ed25519 — Gitea scan will use HTTPS" +fi # Run initial Gitea scan echo "Running initial Gitea scan..." -cd /app && bun run src/aggregator/gitea-scanner.ts --verbose 2>&1 || echo "Scan completed with errors" +cd /app && bun run src/aggregator/gitea-scanner.ts --verbose 2>&1 || echo "Gitea scan completed with errors (non-fatal)" # Start aggregator echo "Starting aggregator..." diff --git a/src/aggregator/gitea-scanner.ts b/src/aggregator/gitea-scanner.ts index 02a55ef..5b23878 100644 --- a/src/aggregator/gitea-scanner.ts +++ b/src/aggregator/gitea-scanner.ts @@ -118,8 +118,20 @@ class GiteaScanner { return contents.some((item) => item.name === "backlog" && item.type === "dir"); } + private getAuthCloneUrl(repo: GiteaRepo): string { + // Embed token in HTTPS URL for private repo access + if (this.config.giteaToken && repo.private) { + const url = new URL(repo.clone_url); + url.username = "git"; + url.password = this.config.giteaToken; + return url.toString(); + } + return repo.clone_url; + } + async cloneOrPullRepo(repo: GiteaRepo): Promise { const repoDir = join(this.config.outputDir, repo.name); + const cloneUrl = this.getAuthCloneUrl(repo); try { // Check if already cloned @@ -139,14 +151,9 @@ class GiteaScanner { await $`cd ${repoDir} && git fetch origin && git reset --hard origin/${repo.default_branch} 2>&1`.quiet(); } } else { - // Clone the repo if (this.config.verbose) { console.log(`Cloning ${repo.full_name}...`); } - - // Use SSH URL if we have an SSH key configured, otherwise HTTPS - const cloneUrl = this.config.sshKeyPath ? repo.ssh_url : repo.clone_url; - const result = await $`git clone --depth 1 ${cloneUrl} ${repoDir} 2>&1`.quiet(); if (result.exitCode !== 0) { console.warn(`Failed to clone ${repo.full_name}: ${result.stderr}`); @@ -252,7 +259,7 @@ function parseArgs(): ScannerConfig { const giteaUrl = getArg("gitea-url", "GITEA_URL", "https://gitea.jeffemmett.com"); const giteaToken = getArg("gitea-token", "GITEA_TOKEN"); const outputDir = getArg("output", "GITEA_OUTPUT_DIR", "/opt/gitea-repos"); - const sshKeyPath = getArg("ssh-key", "GITEA_SSH_KEY", "/root/.ssh/gitea_ed25519"); + const sshKeyPath = getArg("ssh-key", "GITEA_SSH_KEY"); const owner = getArg("owner", "GITEA_OWNER", "jeffemmett"); const concurrency = Number.parseInt(getArg("concurrency", "GITEA_CONCURRENCY", "5") || "5", 10); const verbose = args.includes("--verbose") || args.includes("-v");