Remove R2 storage configs, use local NVMe storage
- Remove r2-mount, r2-sync, cost-monitor services from docker-compose - Delete old docker-compose.yml (R2 version) - Remove services/cost-monitor directory - Update deploy script to remove FUSE/rclone setup - Add music support to upload script - All media now stored on Netcup RS 8000 local NVMe (3TB) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
4e66b0cf25
commit
a04f5d0111
|
|
@ -1,6 +1,5 @@
|
|||
# Environment and secrets
|
||||
.env
|
||||
config/rclone/rclone.conf
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
|
|
|||
|
|
@ -69,7 +69,8 @@ cp .env.example .env
|
|||
## Access
|
||||
|
||||
All services accessible via Cloudflare Tunnel:
|
||||
- **Jellyfin**: https://movies.jeffemmett.com
|
||||
- **Movies & TV**: https://movies.jeffemmett.com
|
||||
- **Music**: https://music.jeffemmett.com
|
||||
|
||||
## Folder Structure
|
||||
|
||||
|
|
@ -77,7 +78,8 @@ All services accessible via Cloudflare Tunnel:
|
|||
/opt/media-server/
|
||||
├── media/
|
||||
│ ├── movies/ # Movie files
|
||||
│ └── shows/ # TV show files
|
||||
│ ├── shows/ # TV show files
|
||||
│ └── music/ # Music files
|
||||
├── config/
|
||||
│ ├── jellyfin/ # Jellyfin config
|
||||
│ ├── sonarr/ # Sonarr config
|
||||
|
|
@ -96,6 +98,7 @@ All services accessible via Cloudflare Tunnel:
|
|||
# Examples:
|
||||
./scripts/upload-to-netcup.sh /home/jeffe/Shows shows
|
||||
./scripts/upload-to-netcup.sh /home/jeffe/Movies movies
|
||||
./scripts/upload-to-netcup.sh /home/jeffe/Music music
|
||||
```
|
||||
|
||||
The script uses rsync for efficient incremental uploads.
|
||||
|
|
|
|||
|
|
@ -10,54 +10,20 @@ services:
|
|||
volumes:
|
||||
- ./config/jellyfin:/config
|
||||
- ./cache/jellyfin:/cache
|
||||
- /mnt/r2-media:/media:ro
|
||||
- ./media:/media
|
||||
networks:
|
||||
- media-network
|
||||
- traefik-public
|
||||
depends_on:
|
||||
- r2-mount
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.routers.jellyfin.rule=Host(`movies.jeffemmett.com`)"
|
||||
# Route both movies.jeffemmett.com and music.jeffemmett.com to Jellyfin
|
||||
- "traefik.http.routers.jellyfin.rule=Host(`movies.jeffemmett.com`) || Host(`music.jeffemmett.com`)"
|
||||
- "traefik.http.routers.jellyfin.entrypoints=web"
|
||||
- "traefik.http.routers.jellyfin.middlewares=jellyfin-headers"
|
||||
- "traefik.http.services.jellyfin.loadbalancer.server.port=8096"
|
||||
- "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Proto=https"
|
||||
- "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Host=movies.jeffemmett.com"
|
||||
- "traefik.docker.network=traefik-public"
|
||||
|
||||
r2-mount:
|
||||
image: rclone/rclone:latest
|
||||
container_name: r2-mount
|
||||
restart: unless-stopped
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
devices:
|
||||
- /dev/fuse
|
||||
security_opt:
|
||||
- apparmor:unconfined
|
||||
environment:
|
||||
- RCLONE_CONFIG=/config/rclone/rclone.conf
|
||||
volumes:
|
||||
- ./config/rclone:/config/rclone:ro
|
||||
- /mnt/r2-media:/mnt/r2-media:shared
|
||||
- ./cache/rclone:/cache
|
||||
command: >
|
||||
mount r2:plex-media /mnt/r2-media
|
||||
--allow-other --allow-non-empty
|
||||
--vfs-cache-mode full
|
||||
--vfs-cache-max-size 50G
|
||||
--vfs-cache-max-age 72h
|
||||
--vfs-read-chunk-size 128M
|
||||
--vfs-read-chunk-size-limit 1G
|
||||
--buffer-size 512M
|
||||
--dir-cache-time 72h
|
||||
--poll-interval 15s
|
||||
--log-level INFO
|
||||
--cache-dir /cache
|
||||
networks:
|
||||
- media-network
|
||||
|
||||
sonarr:
|
||||
image: linuxserver/sonarr:latest
|
||||
container_name: sonarr
|
||||
|
|
@ -68,14 +34,12 @@ services:
|
|||
- TZ=Europe/Berlin
|
||||
volumes:
|
||||
- ./config/sonarr:/config
|
||||
- /mnt/r2-media/tv:/tv
|
||||
- ./media/shows:/tv
|
||||
- ./downloads:/downloads
|
||||
ports:
|
||||
- 8989:8989
|
||||
networks:
|
||||
- media-network
|
||||
depends_on:
|
||||
- r2-mount
|
||||
|
||||
radarr:
|
||||
image: linuxserver/radarr:latest
|
||||
|
|
@ -87,14 +51,12 @@ services:
|
|||
- TZ=Europe/Berlin
|
||||
volumes:
|
||||
- ./config/radarr:/config
|
||||
- /mnt/r2-media/movies:/movies
|
||||
- ./media/movies:/movies
|
||||
- ./downloads:/downloads
|
||||
ports:
|
||||
- 7878:7878
|
||||
networks:
|
||||
- media-network
|
||||
depends_on:
|
||||
- r2-mount
|
||||
|
||||
prowlarr:
|
||||
image: linuxserver/prowlarr:latest
|
||||
|
|
@ -131,44 +93,6 @@ services:
|
|||
networks:
|
||||
- media-network
|
||||
|
||||
r2-sync:
|
||||
build:
|
||||
context: ./services/r2-sync
|
||||
dockerfile: Dockerfile
|
||||
container_name: r2-sync
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- R2_ACCOUNT_ID=${R2_ACCOUNT_ID}
|
||||
- R2_ACCESS_KEY=${R2_ACCESS_KEY}
|
||||
- R2_SECRET_KEY=${R2_SECRET_KEY}
|
||||
- R2_BUCKET=plex-media
|
||||
- WATCH_DIR=/downloads/complete
|
||||
- SYNC_INTERVAL=300
|
||||
volumes:
|
||||
- ./downloads/complete:/downloads/complete:ro
|
||||
- ./config/rclone:/config/rclone:ro
|
||||
- ./logs/r2-sync:/logs
|
||||
networks:
|
||||
- media-network
|
||||
|
||||
cost-monitor:
|
||||
build:
|
||||
context: ./services/cost-monitor
|
||||
dockerfile: Dockerfile
|
||||
container_name: cost-monitor
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- R2_BUCKET=plex-media
|
||||
- RCLONE_CONFIG=/config/rclone/rclone.conf
|
||||
- METRICS_PORT=9100
|
||||
- UPDATE_INTERVAL=3600
|
||||
volumes:
|
||||
- ./config/rclone:/config/rclone:ro
|
||||
ports:
|
||||
- 9100:9100
|
||||
networks:
|
||||
- media-network
|
||||
|
||||
networks:
|
||||
media-network:
|
||||
driver: bridge
|
||||
|
|
|
|||
|
|
@ -1,228 +0,0 @@
|
|||
version: "3.9"
|
||||
|
||||
services:
|
||||
# Jellyfin Media Server (Plex alternative - no license needed)
|
||||
jellyfin:
|
||||
image: jellyfin/jellyfin:latest
|
||||
container_name: jellyfin
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
- TZ=Europe/Berlin
|
||||
volumes:
|
||||
- ./config/jellyfin:/config
|
||||
- ./cache/jellyfin:/cache
|
||||
- /mnt/r2-media:/media:ro # R2 mounted via rclone
|
||||
ports:
|
||||
- "8096:8096"
|
||||
- "8920:8920" # HTTPS
|
||||
networks:
|
||||
- media-network
|
||||
depends_on:
|
||||
- r2-mount
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
# HTTP router (redirects to HTTPS)
|
||||
- "traefik.http.routers.jellyfin.rule=Host(`movies.jeffemmett.com`)"
|
||||
- "traefik.http.routers.jellyfin.entrypoints=web"
|
||||
- "traefik.http.routers.jellyfin.middlewares=jellyfin-https-redirect"
|
||||
# HTTPS router
|
||||
- "traefik.http.routers.jellyfin-secure.rule=Host(`movies.jeffemmett.com`)"
|
||||
- "traefik.http.routers.jellyfin-secure.entrypoints=websecure"
|
||||
- "traefik.http.routers.jellyfin-secure.tls.certresolver=letsencrypt"
|
||||
- "traefik.http.routers.jellyfin-secure.middlewares=jellyfin-headers"
|
||||
# Service
|
||||
- "traefik.http.services.jellyfin.loadbalancer.server.port=8096"
|
||||
# Middleware: HTTPS redirect
|
||||
- "traefik.http.middlewares.jellyfin-https-redirect.redirectscheme.scheme=https"
|
||||
- "traefik.http.middlewares.jellyfin-https-redirect.redirectscheme.permanent=true"
|
||||
# Middleware: Headers for proper session handling
|
||||
- "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Proto=https"
|
||||
- "traefik.http.middlewares.jellyfin-headers.headers.customRequestHeaders.X-Forwarded-Host=movies.jeffemmett.com"
|
||||
- "traefik.http.middlewares.jellyfin-headers.headers.stsSeconds=31536000"
|
||||
- "traefik.http.middlewares.jellyfin-headers.headers.stsIncludeSubdomains=true"
|
||||
|
||||
# R2 Mount Service (rclone FUSE mount)
|
||||
r2-mount:
|
||||
image: rclone/rclone:latest
|
||||
container_name: r2-mount
|
||||
restart: unless-stopped
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
devices:
|
||||
- /dev/fuse
|
||||
security_opt:
|
||||
- apparmor:unconfined
|
||||
environment:
|
||||
- RCLONE_CONFIG=/config/rclone/rclone.conf
|
||||
volumes:
|
||||
- ./config/rclone:/config/rclone:ro
|
||||
- /mnt/r2-media:/mnt/r2-media:shared
|
||||
- ./cache/rclone:/cache
|
||||
command: >
|
||||
mount r2:plex-media /mnt/r2-media
|
||||
--allow-other
|
||||
--allow-non-empty
|
||||
--vfs-cache-mode full
|
||||
--vfs-cache-max-size 50G
|
||||
--vfs-cache-max-age 72h
|
||||
--vfs-read-chunk-size 128M
|
||||
--vfs-read-chunk-size-limit 1G
|
||||
--buffer-size 512M
|
||||
--dir-cache-time 72h
|
||||
--poll-interval 15s
|
||||
--log-level INFO
|
||||
--cache-dir /cache
|
||||
networks:
|
||||
- media-network
|
||||
|
||||
# Sonarr - TV Show Management
|
||||
sonarr:
|
||||
image: linuxserver/sonarr:latest
|
||||
container_name: sonarr
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
- TZ=Europe/Berlin
|
||||
volumes:
|
||||
- ./config/sonarr:/config
|
||||
- /mnt/r2-media/tv:/tv
|
||||
- ./downloads:/downloads
|
||||
ports:
|
||||
- "8989:8989"
|
||||
networks:
|
||||
- media-network
|
||||
depends_on:
|
||||
- r2-mount
|
||||
|
||||
# Radarr - Movie Management
|
||||
radarr:
|
||||
image: linuxserver/radarr:latest
|
||||
container_name: radarr
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
- TZ=Europe/Berlin
|
||||
volumes:
|
||||
- ./config/radarr:/config
|
||||
- /mnt/r2-media/movies:/movies
|
||||
- ./downloads:/downloads
|
||||
ports:
|
||||
- "7878:7878"
|
||||
networks:
|
||||
- media-network
|
||||
depends_on:
|
||||
- r2-mount
|
||||
|
||||
# Prowlarr - Indexer Management
|
||||
prowlarr:
|
||||
image: linuxserver/prowlarr:latest
|
||||
container_name: prowlarr
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
- TZ=Europe/Berlin
|
||||
volumes:
|
||||
- ./config/prowlarr:/config
|
||||
ports:
|
||||
- "9696:9696"
|
||||
networks:
|
||||
- media-network
|
||||
|
||||
# Transmission - Download Client
|
||||
transmission:
|
||||
image: linuxserver/transmission:latest
|
||||
container_name: transmission
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
- TZ=Europe/Berlin
|
||||
- TRANSMISSION_WEB_HOME=/web
|
||||
volumes:
|
||||
- ./config/transmission:/config
|
||||
- ./downloads:/downloads
|
||||
- ./watch:/watch
|
||||
ports:
|
||||
- "9091:9091"
|
||||
- "51413:51413"
|
||||
- "51413:51413/udp"
|
||||
networks:
|
||||
- media-network
|
||||
|
||||
# R2 Sync Service - Moves completed downloads to R2
|
||||
r2-sync:
|
||||
build:
|
||||
context: ./services/r2-sync
|
||||
dockerfile: Dockerfile
|
||||
container_name: r2-sync
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- R2_ACCOUNT_ID=${R2_ACCOUNT_ID}
|
||||
- R2_ACCESS_KEY=${R2_ACCESS_KEY}
|
||||
- R2_SECRET_KEY=${R2_SECRET_KEY}
|
||||
- R2_BUCKET=plex-media
|
||||
- WATCH_DIR=/downloads/complete
|
||||
- SYNC_INTERVAL=300
|
||||
volumes:
|
||||
- ./downloads/complete:/downloads/complete:ro
|
||||
- ./config/rclone:/config/rclone:ro
|
||||
- ./logs/r2-sync:/logs
|
||||
networks:
|
||||
- media-network
|
||||
|
||||
# Cost Monitor - Tracks R2 storage costs and exposes Prometheus metrics
|
||||
cost-monitor:
|
||||
build:
|
||||
context: ./services/cost-monitor
|
||||
dockerfile: Dockerfile
|
||||
container_name: cost-monitor
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- R2_BUCKET=plex-media
|
||||
- RCLONE_CONFIG=/config/rclone/rclone.conf
|
||||
- METRICS_PORT=9100
|
||||
- UPDATE_INTERVAL=3600
|
||||
volumes:
|
||||
- ./config/rclone:/config/rclone:ro
|
||||
ports:
|
||||
- "9100:9100"
|
||||
networks:
|
||||
- media-network
|
||||
|
||||
# Traefik Reverse Proxy
|
||||
traefik:
|
||||
image: traefik:v3.0
|
||||
container_name: traefik
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--api.dashboard=true"
|
||||
- "--providers.docker=true"
|
||||
- "--providers.docker.exposedbydefault=false"
|
||||
- "--entrypoints.web.address=:80"
|
||||
- "--entrypoints.websecure.address=:443"
|
||||
- "--certificatesresolvers.letsencrypt.acme.httpchallenge=true"
|
||||
- "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web"
|
||||
- "--certificatesresolvers.letsencrypt.acme.email=jeff@jeffemmett.com"
|
||||
- "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- ./config/traefik/letsencrypt:/letsencrypt
|
||||
networks:
|
||||
- media-network
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.routers.traefik.rule=Host(`traefik.jeffemmett.com`)"
|
||||
- "traefik.http.routers.traefik.tls.certresolver=letsencrypt"
|
||||
- "traefik.http.routers.traefik.service=api@internal"
|
||||
|
||||
networks:
|
||||
media-network:
|
||||
driver: bridge
|
||||
|
|
@ -29,6 +29,7 @@ rsync -avz --progress \
|
|||
--exclude '.git' \
|
||||
--exclude 'downloads/*' \
|
||||
--exclude 'cache/*' \
|
||||
--exclude 'media/*' \
|
||||
--exclude '*.log' \
|
||||
--exclude '__pycache__' \
|
||||
--exclude '.env' \
|
||||
|
|
@ -47,30 +48,20 @@ ssh $REMOTE_HOST << 'REMOTE_SCRIPT'
|
|||
cd /opt/media-server
|
||||
|
||||
# Create required directories
|
||||
mkdir -p config/{jellyfin,sonarr,radarr,prowlarr,transmission,rclone,traefik/letsencrypt}
|
||||
mkdir -p cache/{jellyfin,rclone}
|
||||
mkdir -p config/{jellyfin,sonarr,radarr,prowlarr,transmission}
|
||||
mkdir -p cache/jellyfin
|
||||
mkdir -p downloads/{complete/movies,complete/tv}
|
||||
mkdir -p logs/r2-sync
|
||||
mkdir -p /mnt/r2-media
|
||||
|
||||
# Install FUSE if needed
|
||||
if ! command -v fusermount &> /dev/null; then
|
||||
sudo apt-get update && sudo apt-get install -y fuse3
|
||||
fi
|
||||
|
||||
# Install rclone if needed
|
||||
if ! command -v rclone &> /dev/null; then
|
||||
curl https://rclone.org/install.sh | sudo bash
|
||||
fi
|
||||
mkdir -p media/{movies,shows,music}
|
||||
|
||||
# Set permissions
|
||||
chmod +x scripts/*.sh
|
||||
chmod 600 config/rclone/rclone.conf 2>/dev/null || true
|
||||
|
||||
# Rename docker-compose-server.yml to docker-compose.yml for deployment
|
||||
cp docker-compose-server.yml docker-compose.yml
|
||||
|
||||
# Build and start services
|
||||
echo "Starting Docker services..."
|
||||
docker compose pull
|
||||
docker compose build
|
||||
docker compose up -d
|
||||
|
||||
echo ""
|
||||
|
|
@ -84,7 +75,8 @@ echo ""
|
|||
echo "Deployment complete!"
|
||||
echo ""
|
||||
echo "Access your services:"
|
||||
echo " Jellyfin: https://movies.jeffemmett.com (or http://SERVER_IP:8096)"
|
||||
echo " Jellyfin: https://movies.jeffemmett.com"
|
||||
echo " Music: https://music.jeffemmett.com"
|
||||
echo " Sonarr: http://SERVER_IP:8989"
|
||||
echo " Radarr: http://SERVER_IP:7878"
|
||||
echo " Prowlarr: http://SERVER_IP:9696"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
# Usage: ./upload-to-netcup.sh <local-path> <media-type>
|
||||
# Example: ./upload-to-netcup.sh /home/jeffe/Shows shows
|
||||
# ./upload-to-netcup.sh /home/jeffe/Movies movies
|
||||
# ./upload-to-netcup.sh /home/jeffe/Music music
|
||||
|
||||
set -e
|
||||
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
FROM python:3.11-slim
|
||||
|
||||
# Install rclone
|
||||
RUN apt-get update && apt-get install -y \
|
||||
curl \
|
||||
unzip \
|
||||
&& curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip \
|
||||
&& unzip rclone-current-linux-amd64.zip \
|
||||
&& cd rclone-*-linux-amd64 \
|
||||
&& cp rclone /usr/bin/ \
|
||||
&& chmod +x /usr/bin/rclone \
|
||||
&& cd .. \
|
||||
&& rm -rf rclone-* \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN pip install --no-cache-dir loguru
|
||||
|
||||
COPY monitor.py .
|
||||
|
||||
EXPOSE 9100
|
||||
|
||||
CMD ["python", "monitor.py"]
|
||||
|
|
@ -1,207 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
R2 Cost Monitor Service
|
||||
Tracks storage usage and costs for the R2-based media server.
|
||||
Exposes metrics for Prometheus/Grafana integration.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from threading import Thread
|
||||
|
||||
from loguru import logger
|
||||
|
||||
# Configuration
|
||||
R2_BUCKET = os.getenv("R2_BUCKET", "plex-media")
|
||||
RCLONE_CONFIG = os.getenv("RCLONE_CONFIG", "/config/rclone/rclone.conf")
|
||||
METRICS_PORT = int(os.getenv("METRICS_PORT", "9100"))
|
||||
UPDATE_INTERVAL = int(os.getenv("UPDATE_INTERVAL", "3600")) # 1 hour
|
||||
|
||||
# R2 Pricing (as of 2024)
|
||||
R2_STORAGE_PRICE_PER_GB = 0.015 # per month
|
||||
R2_CLASS_A_OPS_PRICE = 4.50 / 1_000_000 # per million (write, list)
|
||||
R2_CLASS_B_OPS_PRICE = 0.36 / 1_000_000 # per million (read)
|
||||
# NO EGRESS FEES! This is the killer feature
|
||||
|
||||
# Global metrics storage
|
||||
metrics = {
|
||||
"total_size_bytes": 0,
|
||||
"total_files": 0,
|
||||
"movies_size_bytes": 0,
|
||||
"movies_files": 0,
|
||||
"tv_size_bytes": 0,
|
||||
"tv_files": 0,
|
||||
"music_size_bytes": 0,
|
||||
"music_files": 0,
|
||||
"estimated_monthly_cost": 0.0,
|
||||
"last_updated": 0,
|
||||
}
|
||||
|
||||
|
||||
def get_folder_stats(folder: str) -> dict:
|
||||
"""Get size statistics for an R2 folder."""
|
||||
cmd = [
|
||||
"rclone", "size",
|
||||
"--config", RCLONE_CONFIG,
|
||||
"--json",
|
||||
f"r2:{R2_BUCKET}/{folder}"
|
||||
]
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
|
||||
if result.returncode == 0:
|
||||
data = json.loads(result.stdout)
|
||||
return {
|
||||
"bytes": data.get("bytes", 0),
|
||||
"count": data.get("count", 0)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get stats for {folder}: {e}")
|
||||
|
||||
return {"bytes": 0, "count": 0}
|
||||
|
||||
|
||||
def update_metrics():
|
||||
"""Update all metrics from R2."""
|
||||
global metrics
|
||||
|
||||
logger.info("Updating R2 metrics...")
|
||||
|
||||
# Get stats for each folder
|
||||
movies = get_folder_stats("movies")
|
||||
tv = get_folder_stats("tv")
|
||||
music = get_folder_stats("music")
|
||||
|
||||
# Calculate totals
|
||||
total_bytes = movies["bytes"] + tv["bytes"] + music["bytes"]
|
||||
total_files = movies["count"] + tv["count"] + music["count"]
|
||||
|
||||
# Calculate monthly cost (storage only - no egress!)
|
||||
total_gb = total_bytes / (1024**3)
|
||||
monthly_cost = total_gb * R2_STORAGE_PRICE_PER_GB
|
||||
|
||||
# Update global metrics
|
||||
metrics.update({
|
||||
"total_size_bytes": total_bytes,
|
||||
"total_files": total_files,
|
||||
"movies_size_bytes": movies["bytes"],
|
||||
"movies_files": movies["count"],
|
||||
"tv_size_bytes": tv["bytes"],
|
||||
"tv_files": tv["count"],
|
||||
"music_size_bytes": music["bytes"],
|
||||
"music_files": music["count"],
|
||||
"estimated_monthly_cost": monthly_cost,
|
||||
"last_updated": int(datetime.now().timestamp()),
|
||||
})
|
||||
|
||||
logger.info(
|
||||
f"R2 Stats: {total_gb:.2f} GB, {total_files} files, "
|
||||
f"${monthly_cost:.2f}/month estimated"
|
||||
)
|
||||
|
||||
|
||||
def metrics_to_prometheus() -> str:
|
||||
"""Convert metrics to Prometheus format."""
|
||||
lines = [
|
||||
"# HELP r2_storage_bytes Total storage used in R2",
|
||||
"# TYPE r2_storage_bytes gauge",
|
||||
f'r2_storage_bytes{{bucket="{R2_BUCKET}"}} {metrics["total_size_bytes"]}',
|
||||
f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="movies"}} {metrics["movies_size_bytes"]}',
|
||||
f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="tv"}} {metrics["tv_size_bytes"]}',
|
||||
f'r2_storage_bytes{{bucket="{R2_BUCKET}",folder="music"}} {metrics["music_size_bytes"]}',
|
||||
"",
|
||||
"# HELP r2_files_total Total number of files in R2",
|
||||
"# TYPE r2_files_total gauge",
|
||||
f'r2_files_total{{bucket="{R2_BUCKET}"}} {metrics["total_files"]}',
|
||||
f'r2_files_total{{bucket="{R2_BUCKET}",folder="movies"}} {metrics["movies_files"]}',
|
||||
f'r2_files_total{{bucket="{R2_BUCKET}",folder="tv"}} {metrics["tv_files"]}',
|
||||
f'r2_files_total{{bucket="{R2_BUCKET}",folder="music"}} {metrics["music_files"]}',
|
||||
"",
|
||||
"# HELP r2_estimated_cost_monthly Estimated monthly cost in USD",
|
||||
"# TYPE r2_estimated_cost_monthly gauge",
|
||||
f'r2_estimated_cost_monthly{{bucket="{R2_BUCKET}"}} {metrics["estimated_monthly_cost"]:.4f}',
|
||||
"",
|
||||
"# HELP r2_last_updated_timestamp Last metrics update timestamp",
|
||||
"# TYPE r2_last_updated_timestamp gauge",
|
||||
f'r2_last_updated_timestamp{{bucket="{R2_BUCKET}"}} {metrics["last_updated"]}',
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class MetricsHandler(BaseHTTPRequestHandler):
|
||||
"""HTTP handler for Prometheus metrics endpoint."""
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/metrics":
|
||||
content = metrics_to_prometheus()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||
self.end_headers()
|
||||
self.wfile.write(content.encode())
|
||||
elif self.path == "/health":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{"status": "healthy"}')
|
||||
elif self.path == "/stats":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
|
||||
# Add human-readable values
|
||||
stats = metrics.copy()
|
||||
stats["total_size_gb"] = metrics["total_size_bytes"] / (1024**3)
|
||||
stats["movies_size_gb"] = metrics["movies_size_bytes"] / (1024**3)
|
||||
stats["tv_size_gb"] = metrics["tv_size_bytes"] / (1024**3)
|
||||
stats["music_size_gb"] = metrics["music_size_bytes"] / (1024**3)
|
||||
|
||||
self.wfile.write(json.dumps(stats, indent=2).encode())
|
||||
else:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, format, *args):
|
||||
pass # Suppress default logging
|
||||
|
||||
|
||||
def metrics_updater():
|
||||
"""Background thread to update metrics periodically."""
|
||||
while True:
|
||||
try:
|
||||
update_metrics()
|
||||
except Exception as e:
|
||||
logger.exception(f"Error updating metrics: {e}")
|
||||
|
||||
time.sleep(UPDATE_INTERVAL)
|
||||
|
||||
|
||||
def main():
|
||||
logger.info(f"Starting R2 Cost Monitor on port {METRICS_PORT}")
|
||||
logger.info(f"Monitoring bucket: {R2_BUCKET}")
|
||||
logger.info(f"Update interval: {UPDATE_INTERVAL}s")
|
||||
|
||||
# Initial metrics update
|
||||
update_metrics()
|
||||
|
||||
# Start background updater
|
||||
updater = Thread(target=metrics_updater, daemon=True)
|
||||
updater.start()
|
||||
|
||||
# Start HTTP server
|
||||
server = HTTPServer(("0.0.0.0", METRICS_PORT), MetricsHandler)
|
||||
logger.info(f"Metrics server running on http://0.0.0.0:{METRICS_PORT}/metrics")
|
||||
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Shutting down...")
|
||||
server.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue