diff --git a/.gitignore b/.gitignore index 3ec1082..68cbaf2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Environment and secrets .env -config/rclone/rclone.conf # Logs *.log diff --git a/README.md b/README.md index a47d96d..53cbefe 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,8 @@ cp .env.example .env ## Access All services accessible via Cloudflare Tunnel: -- **Jellyfin**: https://movies.jeffemmett.com +- **Movies & TV**: https://movies.jeffemmett.com +- **Music**: https://music.jeffemmett.com ## Folder Structure @@ -77,7 +78,8 @@ All services accessible via Cloudflare Tunnel: /opt/media-server/ ├── media/ │ ├── movies/ # Movie files -│ └── shows/ # TV show files +│ ├── shows/ # TV show files +│ └── music/ # Music files ├── config/ │ ├── jellyfin/ # Jellyfin config │ ├── sonarr/ # Sonarr config @@ -96,6 +98,7 @@ All services accessible via Cloudflare Tunnel: # Examples: ./scripts/upload-to-netcup.sh /home/jeffe/Shows shows ./scripts/upload-to-netcup.sh /home/jeffe/Movies movies +./scripts/upload-to-netcup.sh /home/jeffe/Music music ``` The script uses rsync for efficient incremental uploads. diff --git a/docker-compose-server.yml b/docker-compose-server.yml index 760ae8c..1b7314e 100644 --- a/docker-compose-server.yml +++ b/docker-compose-server.yml @@ -10,54 +10,20 @@ services: volumes: - ./config/jellyfin:/config - ./cache/jellyfin:/cache - - /mnt/r2-media:/media:ro + - ./media:/media networks: - media-network - traefik-public - depends_on: - - r2-mount labels: - "traefik.enable=true" - - "traefik.http.routers.jellyfin.rule=Host(`movies.jeffemmett.com`)" + # Route both movies.jeffemmett.com and music.jeffemmett.com to Jellyfin + - "traefik.http.routers.jellyfin.rule=Host(`movies.jeffemmett.com`) || Host(`music.jeffemmett.com`)" - "traefik.http.routers.jellyfin.entrypoints=web" - "traefik.http.routers.jellyfin.middlewares=jellyfin-headers" - "traefik.http.services.jellyfin.loadbalancer.server.port=8096" - "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Proto=https" - - "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Host=movies.jeffemmett.com" - "traefik.docker.network=traefik-public" - r2-mount: - image: rclone/rclone:latest - container_name: r2-mount - restart: unless-stopped - cap_add: - - SYS_ADMIN - devices: - - /dev/fuse - security_opt: - - apparmor:unconfined - environment: - - RCLONE_CONFIG=/config/rclone/rclone.conf - volumes: - - ./config/rclone:/config/rclone:ro - - /mnt/r2-media:/mnt/r2-media:shared - - ./cache/rclone:/cache - command: > - mount r2:plex-media /mnt/r2-media - --allow-other --allow-non-empty - --vfs-cache-mode full - --vfs-cache-max-size 50G - --vfs-cache-max-age 72h - --vfs-read-chunk-size 128M - --vfs-read-chunk-size-limit 1G - --buffer-size 512M - --dir-cache-time 72h - --poll-interval 15s - --log-level INFO - --cache-dir /cache - networks: - - media-network - sonarr: image: linuxserver/sonarr:latest container_name: sonarr @@ -68,14 +34,12 @@ services: - TZ=Europe/Berlin volumes: - ./config/sonarr:/config - - /mnt/r2-media/tv:/tv + - ./media/shows:/tv - ./downloads:/downloads ports: - 8989:8989 networks: - media-network - depends_on: - - r2-mount radarr: image: linuxserver/radarr:latest @@ -87,14 +51,12 @@ services: - TZ=Europe/Berlin volumes: - ./config/radarr:/config - - /mnt/r2-media/movies:/movies + - ./media/movies:/movies - ./downloads:/downloads ports: - 7878:7878 networks: - media-network - depends_on: - - r2-mount prowlarr: image: linuxserver/prowlarr:latest @@ -131,44 +93,6 @@ services: networks: - media-network - r2-sync: - build: - context: ./services/r2-sync - dockerfile: Dockerfile - container_name: r2-sync - restart: unless-stopped - environment: - - R2_ACCOUNT_ID=${R2_ACCOUNT_ID} - - R2_ACCESS_KEY=${R2_ACCESS_KEY} - - R2_SECRET_KEY=${R2_SECRET_KEY} - - R2_BUCKET=plex-media - - WATCH_DIR=/downloads/complete - - SYNC_INTERVAL=300 - volumes: - - ./downloads/complete:/downloads/complete:ro - - ./config/rclone:/config/rclone:ro - - ./logs/r2-sync:/logs - networks: - - media-network - - cost-monitor: - build: - context: ./services/cost-monitor - dockerfile: Dockerfile - container_name: cost-monitor - restart: unless-stopped - environment: - - R2_BUCKET=plex-media - - RCLONE_CONFIG=/config/rclone/rclone.conf - - METRICS_PORT=9100 - - UPDATE_INTERVAL=3600 - volumes: - - ./config/rclone:/config/rclone:ro - ports: - - 9100:9100 - networks: - - media-network - networks: media-network: driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index b5648a8..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,228 +0,0 @@ -version: "3.9" - -services: - # Jellyfin Media Server (Plex alternative - no license needed) - jellyfin: - image: jellyfin/jellyfin:latest - container_name: jellyfin - restart: unless-stopped - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./config/jellyfin:/config - - ./cache/jellyfin:/cache - - /mnt/r2-media:/media:ro # R2 mounted via rclone - ports: - - "8096:8096" - - "8920:8920" # HTTPS - networks: - - media-network - depends_on: - - r2-mount - labels: - - "traefik.enable=true" - # HTTP router (redirects to HTTPS) - - "traefik.http.routers.jellyfin.rule=Host(`movies.jeffemmett.com`)" - - "traefik.http.routers.jellyfin.entrypoints=web" - - "traefik.http.routers.jellyfin.middlewares=jellyfin-https-redirect" - # HTTPS router - - "traefik.http.routers.jellyfin-secure.rule=Host(`movies.jeffemmett.com`)" - - "traefik.http.routers.jellyfin-secure.entrypoints=websecure" - - "traefik.http.routers.jellyfin-secure.tls.certresolver=letsencrypt" - - "traefik.http.routers.jellyfin-secure.middlewares=jellyfin-headers" - # Service - - "traefik.http.services.jellyfin.loadbalancer.server.port=8096" - # Middleware: HTTPS redirect - - "traefik.http.middlewares.jellyfin-https-redirect.redirectscheme.scheme=https" - - "traefik.http.middlewares.jellyfin-https-redirect.redirectscheme.permanent=true" - # Middleware: Headers for proper session handling - - "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Proto=https" - - "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Host=movies.jeffemmett.com" - - "traefik.http.middlewares.jellyfin-headers.headers.stsSeconds=31536000" - - "traefik.http.middlewares.jellyfin-headers.headers.stsIncludeSubdomains=true" - - # R2 Mount Service (rclone FUSE mount) - r2-mount: - image: rclone/rclone:latest - container_name: r2-mount - restart: unless-stopped - cap_add: - - SYS_ADMIN - devices: - - /dev/fuse - security_opt: - - apparmor:unconfined - environment: - - RCLONE_CONFIG=/config/rclone/rclone.conf - volumes: - - ./config/rclone:/config/rclone:ro - - /mnt/r2-media:/mnt/r2-media:shared - - ./cache/rclone:/cache - command: > - mount r2:plex-media /mnt/r2-media - --allow-other - --allow-non-empty - --vfs-cache-mode full - --vfs-cache-max-size 50G - --vfs-cache-max-age 72h - --vfs-read-chunk-size 128M - --vfs-read-chunk-size-limit 1G - --buffer-size 512M - --dir-cache-time 72h - --poll-interval 15s - --log-level INFO - --cache-dir /cache - networks: - - media-network - - # Sonarr - TV Show Management - sonarr: - image: linuxserver/sonarr:latest - container_name: sonarr - restart: unless-stopped - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./config/sonarr:/config - - /mnt/r2-media/tv:/tv - - ./downloads:/downloads - ports: - - "8989:8989" - networks: - - media-network - depends_on: - - r2-mount - - # Radarr - Movie Management - radarr: - image: linuxserver/radarr:latest - container_name: radarr - restart: unless-stopped - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./config/radarr:/config - - /mnt/r2-media/movies:/movies - - ./downloads:/downloads - ports: - - "7878:7878" - networks: - - media-network - depends_on: - - r2-mount - - # Prowlarr - Indexer Management - prowlarr: - image: linuxserver/prowlarr:latest - container_name: prowlarr - restart: unless-stopped - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./config/prowlarr:/config - ports: - - "9696:9696" - networks: - - media-network - - # Transmission - Download Client - transmission: - image: linuxserver/transmission:latest - container_name: transmission - restart: unless-stopped - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - - TRANSMISSION_WEB_HOME=/web - volumes: - - ./config/transmission:/config - - ./downloads:/downloads - - ./watch:/watch - ports: - - "9091:9091" - - "51413:51413" - - "51413:51413/udp" - networks: - - media-network - - # R2 Sync Service - Moves completed downloads to R2 - r2-sync: - build: - context: ./services/r2-sync - dockerfile: Dockerfile - container_name: r2-sync - restart: unless-stopped - environment: - - R2_ACCOUNT_ID=${R2_ACCOUNT_ID} - - R2_ACCESS_KEY=${R2_ACCESS_KEY} - - R2_SECRET_KEY=${R2_SECRET_KEY} - - R2_BUCKET=plex-media - - WATCH_DIR=/downloads/complete - - SYNC_INTERVAL=300 - volumes: - - ./downloads/complete:/downloads/complete:ro - - ./config/rclone:/config/rclone:ro - - ./logs/r2-sync:/logs - networks: - - media-network - - # Cost Monitor - Tracks R2 storage costs and exposes Prometheus metrics - cost-monitor: - build: - context: ./services/cost-monitor - dockerfile: Dockerfile - container_name: cost-monitor - restart: unless-stopped - environment: - - R2_BUCKET=plex-media - - RCLONE_CONFIG=/config/rclone/rclone.conf - - METRICS_PORT=9100 - - UPDATE_INTERVAL=3600 - volumes: - - ./config/rclone:/config/rclone:ro - ports: - - "9100:9100" - networks: - - media-network - - # Traefik Reverse Proxy - traefik: - image: traefik:v3.0 - container_name: traefik - restart: unless-stopped - command: - - "--api.dashboard=true" - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true" - - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web" - - "--certificatesresolvers.letsencrypt.acme.email=jeff@jeffemmett.com" - - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" - ports: - - "80:80" - - "443:443" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./config/traefik/letsencrypt:/letsencrypt - networks: - - media-network - labels: - - "traefik.enable=true" - - "traefik.http.routers.traefik.rule=Host(`traefik.jeffemmett.com`)" - - "traefik.http.routers.traefik.tls.certresolver=letsencrypt" - - "traefik.http.routers.traefik.service=api@internal" - -networks: - media-network: - driver: bridge diff --git a/scripts/deploy-to-netcup.sh b/scripts/deploy-to-netcup.sh index 95fad1a..a199fc2 100755 --- a/scripts/deploy-to-netcup.sh +++ b/scripts/deploy-to-netcup.sh @@ -29,6 +29,7 @@ rsync -avz --progress \ --exclude '.git' \ --exclude 'downloads/*' \ --exclude 'cache/*' \ + --exclude 'media/*' \ --exclude '*.log' \ --exclude '__pycache__' \ --exclude '.env' \ @@ -47,30 +48,20 @@ ssh $REMOTE_HOST << 'REMOTE_SCRIPT' cd /opt/media-server # Create required directories -mkdir -p config/{jellyfin,sonarr,radarr,prowlarr,transmission,rclone,traefik/letsencrypt} -mkdir -p cache/{jellyfin,rclone} +mkdir -p config/{jellyfin,sonarr,radarr,prowlarr,transmission} +mkdir -p cache/jellyfin mkdir -p downloads/{complete/movies,complete/tv} -mkdir -p logs/r2-sync -mkdir -p /mnt/r2-media - -# Install FUSE if needed -if ! command -v fusermount &> /dev/null; then - sudo apt-get update && sudo apt-get install -y fuse3 -fi - -# Install rclone if needed -if ! command -v rclone &> /dev/null; then - curl https://rclone.org/install.sh | sudo bash -fi +mkdir -p media/{movies,shows,music} # Set permissions chmod +x scripts/*.sh -chmod 600 config/rclone/rclone.conf 2>/dev/null || true + +# Rename docker-compose-server.yml to docker-compose.yml for deployment +cp docker-compose-server.yml docker-compose.yml # Build and start services echo "Starting Docker services..." docker compose pull -docker compose build docker compose up -d echo "" @@ -84,7 +75,8 @@ echo "" echo "Deployment complete!" echo "" echo "Access your services:" -echo " Jellyfin: https://movies.jeffemmett.com (or http://SERVER_IP:8096)" +echo " Jellyfin: https://movies.jeffemmett.com" +echo " Music: https://music.jeffemmett.com" echo " Sonarr: http://SERVER_IP:8989" echo " Radarr: http://SERVER_IP:7878" echo " Prowlarr: http://SERVER_IP:9696" diff --git a/scripts/upload-to-netcup.sh b/scripts/upload-to-netcup.sh index f34b744..402876a 100755 --- a/scripts/upload-to-netcup.sh +++ b/scripts/upload-to-netcup.sh @@ -3,6 +3,7 @@ # Usage: ./upload-to-netcup.sh # Example: ./upload-to-netcup.sh /home/jeffe/Shows shows # ./upload-to-netcup.sh /home/jeffe/Movies movies +# ./upload-to-netcup.sh /home/jeffe/Music music set -e diff --git a/services/cost-monitor/Dockerfile b/services/cost-monitor/Dockerfile deleted file mode 100644 index ebaa82a..0000000 --- a/services/cost-monitor/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM python:3.11-slim - -# Install rclone -RUN apt-get update && apt-get install -y \ - curl \ - unzip \ - && curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip \ - && unzip rclone-current-linux-amd64.zip \ - && cd rclone-*-linux-amd64 \ - && cp rclone /usr/bin/ \ - && chmod +x /usr/bin/rclone \ - && cd .. \ - && rm -rf rclone-* \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -RUN pip install --no-cache-dir loguru - -COPY monitor.py . - -EXPOSE 9100 - -CMD ["python", "monitor.py"] diff --git a/services/cost-monitor/monitor.py b/services/cost-monitor/monitor.py deleted file mode 100644 index 8c3f932..0000000 --- a/services/cost-monitor/monitor.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -""" -R2 Cost Monitor Service -Tracks storage usage and costs for the R2-based media server. -Exposes metrics for Prometheus/Grafana integration. -""" - -import os -import subprocess -import json -import time -from datetime import datetime -from http.server import HTTPServer, BaseHTTPRequestHandler -from threading import Thread - -from loguru import logger - -# Configuration -R2_BUCKET = os.getenv("R2_BUCKET", "plex-media") -RCLONE_CONFIG = os.getenv("RCLONE_CONFIG", "/config/rclone/rclone.conf") -METRICS_PORT = int(os.getenv("METRICS_PORT", "9100")) -UPDATE_INTERVAL = int(os.getenv("UPDATE_INTERVAL", "3600")) # 1 hour - -# R2 Pricing (as of 2024) -R2_STORAGE_PRICE_PER_GB = 0.015 # per month -R2_CLASS_A_OPS_PRICE = 4.50 / 1_000_000 # per million (write, list) -R2_CLASS_B_OPS_PRICE = 0.36 / 1_000_000 # per million (read) -# NO EGRESS FEES! This is the killer feature - -# Global metrics storage -metrics = { - "total_size_bytes": 0, - "total_files": 0, - "movies_size_bytes": 0, - "movies_files": 0, - "tv_size_bytes": 0, - "tv_files": 0, - "music_size_bytes": 0, - "music_files": 0, - "estimated_monthly_cost": 0.0, - "last_updated": 0, -} - - -def get_folder_stats(folder: str) -> dict: - """Get size statistics for an R2 folder.""" - cmd = [ - "rclone", "size", - "--config", RCLONE_CONFIG, - "--json", - f"r2:{R2_BUCKET}/{folder}" - ] - - try: - result = subprocess.run(cmd, capture_output=True, text=True, timeout=300) - if result.returncode == 0: - data = json.loads(result.stdout) - return { - "bytes": data.get("bytes", 0), - "count": data.get("count", 0) - } - except Exception as e: - logger.error(f"Failed to get stats for {folder}: {e}") - - return {"bytes": 0, "count": 0} - - -def update_metrics(): - """Update all metrics from R2.""" - global metrics - - logger.info("Updating R2 metrics...") - - # Get stats for each folder - movies = get_folder_stats("movies") - tv = get_folder_stats("tv") - music = get_folder_stats("music") - - # Calculate totals - total_bytes = movies["bytes"] + tv["bytes"] + music["bytes"] - total_files = movies["count"] + tv["count"] + music["count"] - - # Calculate monthly cost (storage only - no egress!) - total_gb = total_bytes / (1024**3) - monthly_cost = total_gb * R2_STORAGE_PRICE_PER_GB - - # Update global metrics - metrics.update({ - "total_size_bytes": total_bytes, - "total_files": total_files, - "movies_size_bytes": movies["bytes"], - "movies_files": movies["count"], - "tv_size_bytes": tv["bytes"], - "tv_files": tv["count"], - "music_size_bytes": music["bytes"], - "music_files": music["count"], - "estimated_monthly_cost": monthly_cost, - "last_updated": int(datetime.now().timestamp()), - }) - - logger.info( - f"R2 Stats: {total_gb:.2f} GB, {total_files} files, " - f"${monthly_cost:.2f}/month estimated" - ) - - -def metrics_to_prometheus() -> str: - """Convert metrics to Prometheus format.""" - lines = [ - "# HELP r2_storage_bytes Total storage used in R2", - "# TYPE r2_storage_bytes gauge", - f'r2_storage_bytes{{bucket="{R2_BUCKET}"}} {metrics["total_size_bytes"]}', - f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="movies"}} {metrics["movies_size_bytes"]}', - f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="tv"}} {metrics["tv_size_bytes"]}', - f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="music"}} {metrics["music_size_bytes"]}', - "", - "# HELP r2_files_total Total number of files in R2", - "# TYPE r2_files_total gauge", - f'r2_files_total{{bucket="{R2_BUCKET}"}} {metrics["total_files"]}', - f'r2_files_total{{bucket="{R2_BUCKET}",folder="movies"}} {metrics["movies_files"]}', - f'r2_files_total{{bucket="{R2_BUCKET}",folder="tv"}} {metrics["tv_files"]}', - f'r2_files_total{{bucket="{R2_BUCKET}",folder="music"}} {metrics["music_files"]}', - "", - "# HELP r2_estimated_cost_monthly Estimated monthly cost in USD", - "# TYPE r2_estimated_cost_monthly gauge", - f'r2_estimated_cost_monthly{{bucket="{R2_BUCKET}"}} {metrics["estimated_monthly_cost"]:.4f}', - "", - "# HELP r2_last_updated_timestamp Last metrics update timestamp", - "# TYPE r2_last_updated_timestamp gauge", - f'r2_last_updated_timestamp{{bucket="{R2_BUCKET}"}} {metrics["last_updated"]}', - "", - ] - return "\n".join(lines) - - -class MetricsHandler(BaseHTTPRequestHandler): - """HTTP handler for Prometheus metrics endpoint.""" - - def do_GET(self): - if self.path == "/metrics": - content = metrics_to_prometheus() - self.send_response(200) - self.send_header("Content-Type", "text/plain; charset=utf-8") - self.end_headers() - self.wfile.write(content.encode()) - elif self.path == "/health": - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(b'{"status": "healthy"}') - elif self.path == "/stats": - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.end_headers() - - # Add human-readable values - stats = metrics.copy() - stats["total_size_gb"] = metrics["total_size_bytes"] / (1024**3) - stats["movies_size_gb"] = metrics["movies_size_bytes"] / (1024**3) - stats["tv_size_gb"] = metrics["tv_size_bytes"] / (1024**3) - stats["music_size_gb"] = metrics["music_size_bytes"] / (1024**3) - - self.wfile.write(json.dumps(stats, indent=2).encode()) - else: - self.send_response(404) - self.end_headers() - - def log_message(self, format, *args): - pass # Suppress default logging - - -def metrics_updater(): - """Background thread to update metrics periodically.""" - while True: - try: - update_metrics() - except Exception as e: - logger.exception(f"Error updating metrics: {e}") - - time.sleep(UPDATE_INTERVAL) - - -def main(): - logger.info(f"Starting R2 Cost Monitor on port {METRICS_PORT}") - logger.info(f"Monitoring bucket: {R2_BUCKET}") - logger.info(f"Update interval: {UPDATE_INTERVAL}s") - - # Initial metrics update - update_metrics() - - # Start background updater - updater = Thread(target=metrics_updater, daemon=True) - updater.start() - - # Start HTTP server - server = HTTPServer(("0.0.0.0", METRICS_PORT), MetricsHandler) - logger.info(f"Metrics server running on http://0.0.0.0:{METRICS_PORT}/metrics") - - try: - server.serve_forever() - except KeyboardInterrupt: - logger.info("Shutting down...") - server.shutdown() - - -if __name__ == "__main__": - main()