From 4e66b0cf2511205e6033a0aa0d68b9726b927562 Mon Sep 17 00:00:00 2001 From: Jeff Emmett Date: Wed, 26 Nov 2025 20:04:01 -0800 Subject: [PATCH] Jellyfin media server for Netcup RS 8000 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Jellyfin-based media streaming - Direct upload to Netcup storage (3TB NVMe) - Upload script for shows/movies via rsync - Docker Compose setup with Traefik labels 🤖 Generated with Claude Code --- .env.example | 23 ++++ .gitignore | 38 ++++++ README.md | 105 ++++++++++++++ docker-compose-server.yml | 176 ++++++++++++++++++++++++ docker-compose.yml | 228 +++++++++++++++++++++++++++++++ scripts/deploy-to-netcup.sh | 94 +++++++++++++ scripts/setup-ssh-key.sh | 30 ++++ scripts/upload-to-netcup.sh | 59 ++++++++ services/cost-monitor/Dockerfile | 25 ++++ services/cost-monitor/monitor.py | 207 ++++++++++++++++++++++++++++ 10 files changed, 985 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docker-compose-server.yml create mode 100644 docker-compose.yml create mode 100755 scripts/deploy-to-netcup.sh create mode 100755 scripts/setup-ssh-key.sh create mode 100755 scripts/upload-to-netcup.sh create mode 100644 services/cost-monitor/Dockerfile create mode 100644 services/cost-monitor/monitor.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..4bd2f9e --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# Domain Configuration +DOMAIN=jeffemmett.com +MEDIA_SUBDOMAIN=movies + +# Timezone +TZ=Europe/Berlin + +# User/Group IDs (run `id` to find yours) +PUID=1000 +PGID=1000 + +# Jellyfin +JELLYFIN_PublishedServerUrl=https://movies.jeffemmett.com + +# Transmission (optional auth) +TRANSMISSION_USER=admin +TRANSMISSION_PASS=changeme + +# VPN (optional - for download privacy) +VPN_ENABLED=false +VPN_PROVIDER=mullvad +VPN_USER= +VPN_PASS= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3ec1082 --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +# Environment and secrets +.env +config/rclone/rclone.conf + +# Logs +*.log +logs/ + +# Cache directories +cache/ +downloads/ + +# Watch directories +watch/ + +# SSL certificates +config/traefik/letsencrypt/ + +# OS files +.DS_Store +Thumbs.db + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# Python +__pycache__/ +*.py[cod] +*$py.class +.Python +venv/ +.venv/ + +# Docker +.docker/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..a47d96d --- /dev/null +++ b/README.md @@ -0,0 +1,105 @@ +# Media Server on Netcup RS 8000 + +A self-hosted media server stack running on Netcup RS 8000 with Jellyfin for streaming. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Netcup RS 8000 │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Jellyfin │ │ Sonarr │ │ Radarr │ │ +│ │ (Stream) │ │ (TV) │ │ (Movies) │ │ +│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ +│ │ │ │ │ +│ └────────────────┼────────────────┘ │ +│ │ │ +│ ┌───────┴───────┐ │ +│ │ Local NVMe │ │ +│ │ Storage │ │ +│ │ (3TB) │ │ +│ └───────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Server Specs + +- **Netcup RS 8000 G12 Pro**: €45/month +- **20 CPU cores** for transcoding +- **64GB RAM** for caching +- **3TB NVMe storage** for media +- **1Gbps bandwidth** included + +## Quick Start + +### 1. Clone and Configure + +```bash +git clone https://gitea.jeffemmett.com/jeffemmett/plex-media.git +cd plex-media +cp .env.example .env +``` + +### 2. Deploy to Netcup + +```bash +./scripts/deploy-to-netcup.sh +``` + +### 3. Upload Your Media + +```bash +# Upload TV shows +./scripts/upload-to-netcup.sh /home/jeffe/Shows shows + +# Upload movies +./scripts/upload-to-netcup.sh /path/to/movies movies +``` + +## Services + +| Service | Port | Description | +|---------|------|-------------| +| Jellyfin | 8096 | Media player | +| Sonarr | 8989 | TV show management | +| Radarr | 7878 | Movie management | +| Prowlarr | 9696 | Indexer management | +| Transmission | 9091 | Download client | + +## Access + +All services accessible via Cloudflare Tunnel: +- **Jellyfin**: https://movies.jeffemmett.com + +## Folder Structure + +``` +/opt/media-server/ +├── media/ +│ ├── movies/ # Movie files +│ └── shows/ # TV show files +├── config/ +│ ├── jellyfin/ # Jellyfin config +│ ├── sonarr/ # Sonarr config +│ ├── radarr/ # Radarr config +│ └── prowlarr/ # Prowlarr config +└── downloads/ + └── complete/ # Completed downloads +``` + +## Upload Script Usage + +```bash +# From local machine +./scripts/upload-to-netcup.sh + +# Examples: +./scripts/upload-to-netcup.sh /home/jeffe/Shows shows +./scripts/upload-to-netcup.sh /home/jeffe/Movies movies +``` + +The script uses rsync for efficient incremental uploads. + +## License + +MIT diff --git a/docker-compose-server.yml b/docker-compose-server.yml new file mode 100644 index 0000000..760ae8c --- /dev/null +++ b/docker-compose-server.yml @@ -0,0 +1,176 @@ +services: + jellyfin: + image: jellyfin/jellyfin:latest + container_name: jellyfin + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + volumes: + - ./config/jellyfin:/config + - ./cache/jellyfin:/cache + - /mnt/r2-media:/media:ro + networks: + - media-network + - traefik-public + depends_on: + - r2-mount + labels: + - "traefik.enable=true" + - "traefik.http.routers.jellyfin.rule=Host(`movies.jeffemmett.com`)" + - "traefik.http.routers.jellyfin.entrypoints=web" + - "traefik.http.routers.jellyfin.middlewares=jellyfin-headers" + - "traefik.http.services.jellyfin.loadbalancer.server.port=8096" + - "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Proto=https" + - "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Host=movies.jeffemmett.com" + - "traefik.docker.network=traefik-public" + + r2-mount: + image: rclone/rclone:latest + container_name: r2-mount + restart: unless-stopped + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined + environment: + - RCLONE_CONFIG=/config/rclone/rclone.conf + volumes: + - ./config/rclone:/config/rclone:ro + - /mnt/r2-media:/mnt/r2-media:shared + - ./cache/rclone:/cache + command: > + mount r2:plex-media /mnt/r2-media + --allow-other --allow-non-empty + --vfs-cache-mode full + --vfs-cache-max-size 50G + --vfs-cache-max-age 72h + --vfs-read-chunk-size 128M + --vfs-read-chunk-size-limit 1G + --buffer-size 512M + --dir-cache-time 72h + --poll-interval 15s + --log-level INFO + --cache-dir /cache + networks: + - media-network + + sonarr: + image: linuxserver/sonarr:latest + container_name: sonarr + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + volumes: + - ./config/sonarr:/config + - /mnt/r2-media/tv:/tv + - ./downloads:/downloads + ports: + - 8989:8989 + networks: + - media-network + depends_on: + - r2-mount + + radarr: + image: linuxserver/radarr:latest + container_name: radarr + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + volumes: + - ./config/radarr:/config + - /mnt/r2-media/movies:/movies + - ./downloads:/downloads + ports: + - 7878:7878 + networks: + - media-network + depends_on: + - r2-mount + + prowlarr: + image: linuxserver/prowlarr:latest + container_name: prowlarr + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + volumes: + - ./config/prowlarr:/config + ports: + - 9696:9696 + networks: + - media-network + + transmission: + image: linuxserver/transmission:latest + container_name: transmission + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + - TRANSMISSION_WEB_HOME=/web + volumes: + - ./config/transmission:/config + - ./downloads:/downloads + - ./watch:/watch + ports: + - 9091:9091 + - 51413:51413 + - 51413:51413/udp + networks: + - media-network + + r2-sync: + build: + context: ./services/r2-sync + dockerfile: Dockerfile + container_name: r2-sync + restart: unless-stopped + environment: + - R2_ACCOUNT_ID=${R2_ACCOUNT_ID} + - R2_ACCESS_KEY=${R2_ACCESS_KEY} + - R2_SECRET_KEY=${R2_SECRET_KEY} + - R2_BUCKET=plex-media + - WATCH_DIR=/downloads/complete + - SYNC_INTERVAL=300 + volumes: + - ./downloads/complete:/downloads/complete:ro + - ./config/rclone:/config/rclone:ro + - ./logs/r2-sync:/logs + networks: + - media-network + + cost-monitor: + build: + context: ./services/cost-monitor + dockerfile: Dockerfile + container_name: cost-monitor + restart: unless-stopped + environment: + - R2_BUCKET=plex-media + - RCLONE_CONFIG=/config/rclone/rclone.conf + - METRICS_PORT=9100 + - UPDATE_INTERVAL=3600 + volumes: + - ./config/rclone:/config/rclone:ro + ports: + - 9100:9100 + networks: + - media-network + +networks: + media-network: + driver: bridge + traefik-public: + external: true diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b5648a8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,228 @@ +version: "3.9" + +services: + # Jellyfin Media Server (Plex alternative - no license needed) + jellyfin: + image: jellyfin/jellyfin:latest + container_name: jellyfin + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + volumes: + - ./config/jellyfin:/config + - ./cache/jellyfin:/cache + - /mnt/r2-media:/media:ro # R2 mounted via rclone + ports: + - "8096:8096" + - "8920:8920" # HTTPS + networks: + - media-network + depends_on: + - r2-mount + labels: + - "traefik.enable=true" + # HTTP router (redirects to HTTPS) + - "traefik.http.routers.jellyfin.rule=Host(`movies.jeffemmett.com`)" + - "traefik.http.routers.jellyfin.entrypoints=web" + - "traefik.http.routers.jellyfin.middlewares=jellyfin-https-redirect" + # HTTPS router + - "traefik.http.routers.jellyfin-secure.rule=Host(`movies.jeffemmett.com`)" + - "traefik.http.routers.jellyfin-secure.entrypoints=websecure" + - "traefik.http.routers.jellyfin-secure.tls.certresolver=letsencrypt" + - "traefik.http.routers.jellyfin-secure.middlewares=jellyfin-headers" + # Service + - "traefik.http.services.jellyfin.loadbalancer.server.port=8096" + # Middleware: HTTPS redirect + - "traefik.http.middlewares.jellyfin-https-redirect.redirectscheme.scheme=https" + - "traefik.http.middlewares.jellyfin-https-redirect.redirectscheme.permanent=true" + # Middleware: Headers for proper session handling + - "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Proto=https" + - "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Host=movies.jeffemmett.com" + - "traefik.http.middlewares.jellyfin-headers.headers.stsSeconds=31536000" + - "traefik.http.middlewares.jellyfin-headers.headers.stsIncludeSubdomains=true" + + # R2 Mount Service (rclone FUSE mount) + r2-mount: + image: rclone/rclone:latest + container_name: r2-mount + restart: unless-stopped + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined + environment: + - RCLONE_CONFIG=/config/rclone/rclone.conf + volumes: + - ./config/rclone:/config/rclone:ro + - /mnt/r2-media:/mnt/r2-media:shared + - ./cache/rclone:/cache + command: > + mount r2:plex-media /mnt/r2-media + --allow-other + --allow-non-empty + --vfs-cache-mode full + --vfs-cache-max-size 50G + --vfs-cache-max-age 72h + --vfs-read-chunk-size 128M + --vfs-read-chunk-size-limit 1G + --buffer-size 512M + --dir-cache-time 72h + --poll-interval 15s + --log-level INFO + --cache-dir /cache + networks: + - media-network + + # Sonarr - TV Show Management + sonarr: + image: linuxserver/sonarr:latest + container_name: sonarr + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + volumes: + - ./config/sonarr:/config + - /mnt/r2-media/tv:/tv + - ./downloads:/downloads + ports: + - "8989:8989" + networks: + - media-network + depends_on: + - r2-mount + + # Radarr - Movie Management + radarr: + image: linuxserver/radarr:latest + container_name: radarr + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + volumes: + - ./config/radarr:/config + - /mnt/r2-media/movies:/movies + - ./downloads:/downloads + ports: + - "7878:7878" + networks: + - media-network + depends_on: + - r2-mount + + # Prowlarr - Indexer Management + prowlarr: + image: linuxserver/prowlarr:latest + container_name: prowlarr + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + volumes: + - ./config/prowlarr:/config + ports: + - "9696:9696" + networks: + - media-network + + # Transmission - Download Client + transmission: + image: linuxserver/transmission:latest + container_name: transmission + restart: unless-stopped + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Berlin + - TRANSMISSION_WEB_HOME=/web + volumes: + - ./config/transmission:/config + - ./downloads:/downloads + - ./watch:/watch + ports: + - "9091:9091" + - "51413:51413" + - "51413:51413/udp" + networks: + - media-network + + # R2 Sync Service - Moves completed downloads to R2 + r2-sync: + build: + context: ./services/r2-sync + dockerfile: Dockerfile + container_name: r2-sync + restart: unless-stopped + environment: + - R2_ACCOUNT_ID=${R2_ACCOUNT_ID} + - R2_ACCESS_KEY=${R2_ACCESS_KEY} + - R2_SECRET_KEY=${R2_SECRET_KEY} + - R2_BUCKET=plex-media + - WATCH_DIR=/downloads/complete + - SYNC_INTERVAL=300 + volumes: + - ./downloads/complete:/downloads/complete:ro + - ./config/rclone:/config/rclone:ro + - ./logs/r2-sync:/logs + networks: + - media-network + + # Cost Monitor - Tracks R2 storage costs and exposes Prometheus metrics + cost-monitor: + build: + context: ./services/cost-monitor + dockerfile: Dockerfile + container_name: cost-monitor + restart: unless-stopped + environment: + - R2_BUCKET=plex-media + - RCLONE_CONFIG=/config/rclone/rclone.conf + - METRICS_PORT=9100 + - UPDATE_INTERVAL=3600 + volumes: + - ./config/rclone:/config/rclone:ro + ports: + - "9100:9100" + networks: + - media-network + + # Traefik Reverse Proxy + traefik: + image: traefik:v3.0 + container_name: traefik + restart: unless-stopped + command: + - "--api.dashboard=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true" + - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web" + - "--certificatesresolvers.letsencrypt.acme.email=jeff@jeffemmett.com" + - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./config/traefik/letsencrypt:/letsencrypt + networks: + - media-network + labels: + - "traefik.enable=true" + - "traefik.http.routers.traefik.rule=Host(`traefik.jeffemmett.com`)" + - "traefik.http.routers.traefik.tls.certresolver=letsencrypt" + - "traefik.http.routers.traefik.service=api@internal" + +networks: + media-network: + driver: bridge diff --git a/scripts/deploy-to-netcup.sh b/scripts/deploy-to-netcup.sh new file mode 100755 index 0000000..95fad1a --- /dev/null +++ b/scripts/deploy-to-netcup.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Deploy media server stack to Netcup RS 8000 +# This script syncs the project and starts the Docker stack + +set -e + +REMOTE_HOST="netcup" +REMOTE_DIR="/opt/media-server" +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +echo "Deploying Media Server to Netcup RS 8000" +echo "========================================" + +# Check SSH connection +echo "Testing SSH connection..." +if ! ssh -q $REMOTE_HOST exit; then + echo "Error: Cannot connect to $REMOTE_HOST" + echo "Make sure 'ssh netcup' works." + exit 1 +fi + +# Create remote directory +echo "Creating remote directory..." +ssh $REMOTE_HOST "sudo mkdir -p $REMOTE_DIR && sudo chown \$(whoami):\$(whoami) $REMOTE_DIR" + +# Sync project files +echo "Syncing project files..." +rsync -avz --progress \ + --exclude '.git' \ + --exclude 'downloads/*' \ + --exclude 'cache/*' \ + --exclude '*.log' \ + --exclude '__pycache__' \ + --exclude '.env' \ + "$PROJECT_DIR/" \ + "$REMOTE_HOST:$REMOTE_DIR/" + +# Copy .env if it exists locally (but not in git) +if [ -f "$PROJECT_DIR/.env" ]; then + echo "Copying .env file..." + scp "$PROJECT_DIR/.env" "$REMOTE_HOST:$REMOTE_DIR/.env" +fi + +# Setup on remote +echo "Setting up on remote..." +ssh $REMOTE_HOST << 'REMOTE_SCRIPT' +cd /opt/media-server + +# Create required directories +mkdir -p config/{jellyfin,sonarr,radarr,prowlarr,transmission,rclone,traefik/letsencrypt} +mkdir -p cache/{jellyfin,rclone} +mkdir -p downloads/{complete/movies,complete/tv} +mkdir -p logs/r2-sync +mkdir -p /mnt/r2-media + +# Install FUSE if needed +if ! command -v fusermount &> /dev/null; then + sudo apt-get update && sudo apt-get install -y fuse3 +fi + +# Install rclone if needed +if ! command -v rclone &> /dev/null; then + curl https://rclone.org/install.sh | sudo bash +fi + +# Set permissions +chmod +x scripts/*.sh +chmod 600 config/rclone/rclone.conf 2>/dev/null || true + +# Build and start services +echo "Starting Docker services..." +docker compose pull +docker compose build +docker compose up -d + +echo "" +echo "Waiting for services to start..." +sleep 10 + +docker compose ps +REMOTE_SCRIPT + +echo "" +echo "Deployment complete!" +echo "" +echo "Access your services:" +echo " Jellyfin: https://movies.jeffemmett.com (or http://SERVER_IP:8096)" +echo " Sonarr: http://SERVER_IP:8989" +echo " Radarr: http://SERVER_IP:7878" +echo " Prowlarr: http://SERVER_IP:9696" +echo " Transmission: http://SERVER_IP:9091" +echo "" +echo "SSH to server: ssh netcup" +echo "View logs: ssh netcup 'cd /opt/media-server && docker compose logs -f'" diff --git a/scripts/setup-ssh-key.sh b/scripts/setup-ssh-key.sh new file mode 100755 index 0000000..8743c9d --- /dev/null +++ b/scripts/setup-ssh-key.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Add your SSH public key to the Netcup server +# This requires password authentication once + +SERVER="159.195.32.209" +USER="root" +KEY_FILE="$HOME/.ssh/netcup_ed25519.pub" + +echo "Adding SSH key to Netcup server..." +echo "You will be prompted for the root password ONCE" +echo "" + +# Copy SSH key to server +ssh-copy-id -i "$KEY_FILE" "${USER}@${SERVER}" + +if [ $? -eq 0 ]; then + echo "" + echo "✓ SSH key added successfully!" + echo "Testing connection..." + ssh -o ConnectTimeout=5 netcup "echo 'SSH key authentication working!'" +else + echo "" + echo "✗ Failed to add SSH key" + echo "You may need to:" + echo "1. Log into Netcup web console" + echo "2. Go to ~/.ssh/authorized_keys" + echo "3. Add this key:" + echo "" + cat "$KEY_FILE" +fi diff --git a/scripts/upload-to-netcup.sh b/scripts/upload-to-netcup.sh new file mode 100755 index 0000000..f34b744 --- /dev/null +++ b/scripts/upload-to-netcup.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Upload media files directly to Netcup server +# Usage: ./upload-to-netcup.sh +# Example: ./upload-to-netcup.sh /home/jeffe/Shows shows +# ./upload-to-netcup.sh /home/jeffe/Movies movies + +set -e + +LOCAL_PATH="${1:-.}" +MEDIA_TYPE="${2:-shows}" + +NETCUP_HOST="netcup" +REMOTE_PATH="/opt/media-server/media/${MEDIA_TYPE}" + +echo "==========================================" +echo " Upload Media to Netcup RS 8000" +echo "==========================================" +echo "" +echo "Source: $LOCAL_PATH" +echo "Destination: ${NETCUP_HOST}:${REMOTE_PATH}" +echo "" + +# Calculate size +echo "Analyzing files..." +LOCAL_SIZE=$(du -sh "$LOCAL_PATH" 2>/dev/null | cut -f1) +FILE_COUNT=$(find "$LOCAL_PATH" -type f 2>/dev/null | wc -l) + +echo "" +echo "Upload Summary:" +echo "===============" +echo "Total size: $LOCAL_SIZE" +echo "Total files: $FILE_COUNT" +echo "" + +# Show what will be uploaded +echo "Contents to upload:" +ls -la "$LOCAL_PATH" | head -15 +echo "" + +# Run rsync with progress +echo "Starting upload..." +echo "" + +rsync -avz --progress \ + --human-readable \ + --stats \ + -e "ssh -i ~/.ssh/netcup_ed25519" \ + "$LOCAL_PATH/" \ + "${NETCUP_HOST}:${REMOTE_PATH}/" + +echo "" +echo "==========================================" +echo " Upload Complete!" +echo "==========================================" +echo "" + +# Show remote storage +echo "Current storage on Netcup:" +ssh ${NETCUP_HOST} "du -sh ${REMOTE_PATH}" diff --git a/services/cost-monitor/Dockerfile b/services/cost-monitor/Dockerfile new file mode 100644 index 0000000..ebaa82a --- /dev/null +++ b/services/cost-monitor/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11-slim + +# Install rclone +RUN apt-get update && apt-get install -y \ + curl \ + unzip \ + && curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip \ + && unzip rclone-current-linux-amd64.zip \ + && cd rclone-*-linux-amd64 \ + && cp rclone /usr/bin/ \ + && chmod +x /usr/bin/rclone \ + && cd .. \ + && rm -rf rclone-* \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +RUN pip install --no-cache-dir loguru + +COPY monitor.py . + +EXPOSE 9100 + +CMD ["python", "monitor.py"] diff --git a/services/cost-monitor/monitor.py b/services/cost-monitor/monitor.py new file mode 100644 index 0000000..8c3f932 --- /dev/null +++ b/services/cost-monitor/monitor.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +""" +R2 Cost Monitor Service +Tracks storage usage and costs for the R2-based media server. +Exposes metrics for Prometheus/Grafana integration. +""" + +import os +import subprocess +import json +import time +from datetime import datetime +from http.server import HTTPServer, BaseHTTPRequestHandler +from threading import Thread + +from loguru import logger + +# Configuration +R2_BUCKET = os.getenv("R2_BUCKET", "plex-media") +RCLONE_CONFIG = os.getenv("RCLONE_CONFIG", "/config/rclone/rclone.conf") +METRICS_PORT = int(os.getenv("METRICS_PORT", "9100")) +UPDATE_INTERVAL = int(os.getenv("UPDATE_INTERVAL", "3600")) # 1 hour + +# R2 Pricing (as of 2024) +R2_STORAGE_PRICE_PER_GB = 0.015 # per month +R2_CLASS_A_OPS_PRICE = 4.50 / 1_000_000 # per million (write, list) +R2_CLASS_B_OPS_PRICE = 0.36 / 1_000_000 # per million (read) +# NO EGRESS FEES! This is the killer feature + +# Global metrics storage +metrics = { + "total_size_bytes": 0, + "total_files": 0, + "movies_size_bytes": 0, + "movies_files": 0, + "tv_size_bytes": 0, + "tv_files": 0, + "music_size_bytes": 0, + "music_files": 0, + "estimated_monthly_cost": 0.0, + "last_updated": 0, +} + + +def get_folder_stats(folder: str) -> dict: + """Get size statistics for an R2 folder.""" + cmd = [ + "rclone", "size", + "--config", RCLONE_CONFIG, + "--json", + f"r2:{R2_BUCKET}/{folder}" + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=300) + if result.returncode == 0: + data = json.loads(result.stdout) + return { + "bytes": data.get("bytes", 0), + "count": data.get("count", 0) + } + except Exception as e: + logger.error(f"Failed to get stats for {folder}: {e}") + + return {"bytes": 0, "count": 0} + + +def update_metrics(): + """Update all metrics from R2.""" + global metrics + + logger.info("Updating R2 metrics...") + + # Get stats for each folder + movies = get_folder_stats("movies") + tv = get_folder_stats("tv") + music = get_folder_stats("music") + + # Calculate totals + total_bytes = movies["bytes"] + tv["bytes"] + music["bytes"] + total_files = movies["count"] + tv["count"] + music["count"] + + # Calculate monthly cost (storage only - no egress!) + total_gb = total_bytes / (1024**3) + monthly_cost = total_gb * R2_STORAGE_PRICE_PER_GB + + # Update global metrics + metrics.update({ + "total_size_bytes": total_bytes, + "total_files": total_files, + "movies_size_bytes": movies["bytes"], + "movies_files": movies["count"], + "tv_size_bytes": tv["bytes"], + "tv_files": tv["count"], + "music_size_bytes": music["bytes"], + "music_files": music["count"], + "estimated_monthly_cost": monthly_cost, + "last_updated": int(datetime.now().timestamp()), + }) + + logger.info( + f"R2 Stats: {total_gb:.2f} GB, {total_files} files, " + f"${monthly_cost:.2f}/month estimated" + ) + + +def metrics_to_prometheus() -> str: + """Convert metrics to Prometheus format.""" + lines = [ + "# HELP r2_storage_bytes Total storage used in R2", + "# TYPE r2_storage_bytes gauge", + f'r2_storage_bytes{{bucket="{R2_BUCKET}"}} {metrics["total_size_bytes"]}', + f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="movies"}} {metrics["movies_size_bytes"]}', + f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="tv"}} {metrics["tv_size_bytes"]}', + f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="music"}} {metrics["music_size_bytes"]}', + "", + "# HELP r2_files_total Total number of files in R2", + "# TYPE r2_files_total gauge", + f'r2_files_total{{bucket="{R2_BUCKET}"}} {metrics["total_files"]}', + f'r2_files_total{{bucket="{R2_BUCKET}",folder="movies"}} {metrics["movies_files"]}', + f'r2_files_total{{bucket="{R2_BUCKET}",folder="tv"}} {metrics["tv_files"]}', + f'r2_files_total{{bucket="{R2_BUCKET}",folder="music"}} {metrics["music_files"]}', + "", + "# HELP r2_estimated_cost_monthly Estimated monthly cost in USD", + "# TYPE r2_estimated_cost_monthly gauge", + f'r2_estimated_cost_monthly{{bucket="{R2_BUCKET}"}} {metrics["estimated_monthly_cost"]:.4f}', + "", + "# HELP r2_last_updated_timestamp Last metrics update timestamp", + "# TYPE r2_last_updated_timestamp gauge", + f'r2_last_updated_timestamp{{bucket="{R2_BUCKET}"}} {metrics["last_updated"]}', + "", + ] + return "\n".join(lines) + + +class MetricsHandler(BaseHTTPRequestHandler): + """HTTP handler for Prometheus metrics endpoint.""" + + def do_GET(self): + if self.path == "/metrics": + content = metrics_to_prometheus() + self.send_response(200) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.end_headers() + self.wfile.write(content.encode()) + elif self.path == "/health": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"status": "healthy"}') + elif self.path == "/stats": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + + # Add human-readable values + stats = metrics.copy() + stats["total_size_gb"] = metrics["total_size_bytes"] / (1024**3) + stats["movies_size_gb"] = metrics["movies_size_bytes"] / (1024**3) + stats["tv_size_gb"] = metrics["tv_size_bytes"] / (1024**3) + stats["music_size_gb"] = metrics["music_size_bytes"] / (1024**3) + + self.wfile.write(json.dumps(stats, indent=2).encode()) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, format, *args): + pass # Suppress default logging + + +def metrics_updater(): + """Background thread to update metrics periodically.""" + while True: + try: + update_metrics() + except Exception as e: + logger.exception(f"Error updating metrics: {e}") + + time.sleep(UPDATE_INTERVAL) + + +def main(): + logger.info(f"Starting R2 Cost Monitor on port {METRICS_PORT}") + logger.info(f"Monitoring bucket: {R2_BUCKET}") + logger.info(f"Update interval: {UPDATE_INTERVAL}s") + + # Initial metrics update + update_metrics() + + # Start background updater + updater = Thread(target=metrics_updater, daemon=True) + updater.start() + + # Start HTTP server + server = HTTPServer(("0.0.0.0", METRICS_PORT), MetricsHandler) + logger.info(f"Metrics server running on http://0.0.0.0:{METRICS_PORT}/metrics") + + try: + server.serve_forever() + except KeyboardInterrupt: + logger.info("Shutting down...") + server.shutdown() + + +if __name__ == "__main__": + main()