commit 7b9ac6d26b8dfb44fb118dbcd812f80c7c1dcd0b Author: Jeff-Emmett Date: Thu Nov 20 01:47:02 2025 -0800 Initial commit: GitLab Self-Hosting Deployment System Complete automation for deploying production-ready GitLab on Digital Ocean with: - Automated GitLab CE installation - Let's Encrypt SSL with auto-renewal - Multiple email provider support (Gmail, SendGrid, Mailgun, AWS SES) - Automated daily backups with cloud storage option - Health monitoring scripts - Security hardening and firewall rules - Performance tuning for 4GB+ RAM droplets - Comprehensive testing suite - Complete documentation 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..31b3fb3 --- /dev/null +++ b/.env.example @@ -0,0 +1,44 @@ +# Digital Ocean Droplet Configuration +DROPLET_IP=your_droplet_ip +DROPLET_USER=root +SSH_KEY_PATH=~/.ssh/id_rsa + +# Domain Configuration +GITLAB_DOMAIN=gitlab.yourdomain.com +ADMIN_EMAIL=your@email.com + +# GitLab Configuration +GITLAB_ROOT_PASSWORD=change_me_after_first_login +GITLAB_BACKUP_BUCKET=your-backup-bucket # For DO Spaces or S3 + +# Email Configuration (REQUIRED for GitLab notifications) +GITLAB_EMAIL_FROM=gitlab@yourdomain.com +GITLAB_EMAIL_DISPLAY_NAME="GitLab" +GITLAB_EMAIL_REPLY_TO=noreply@yourdomain.com + +# Email Delivery Method (choose one: smtp, sendgrid, mailgun, ses) +EMAIL_METHOD=smtp + +# SMTP Configuration (if using external SMTP like Gmail) +SMTP_ENABLED=true +SMTP_ADDRESS=smtp.gmail.com +SMTP_PORT=587 +SMTP_USER_NAME=your-email@gmail.com +SMTP_PASSWORD=your-app-specific-password +SMTP_DOMAIN=gmail.com +SMTP_AUTHENTICATION=login +SMTP_ENABLE_STARTTLS_AUTO=true +SMTP_TLS=false +SMTP_OPENSSL_VERIFY_MODE=peer + +# SendGrid Configuration (alternative - recommended for production) +SENDGRID_API_KEY=your_sendgrid_api_key + +# Mailgun Configuration (alternative) +MAILGUN_API_KEY=your_mailgun_api_key +MAILGUN_DOMAIN=mg.yourdomain.com + +# AWS SES Configuration (alternative) +AWS_SES_ACCESS_KEY_ID=your_access_key +AWS_SES_SECRET_ACCESS_KEY=your_secret_key +AWS_SES_REGION=us-east-1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9cf2016 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# Environment variables +.env + +# Backup files +*.backup +*.bak + +# Logs +*.log + +# OS files +.DS_Store +Thumbs.db + +# Editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ diff --git a/README.md b/README.md new file mode 100644 index 0000000..2f1ce68 --- /dev/null +++ b/README.md @@ -0,0 +1,293 @@ +# GitLab Self-Hosting Deployment + +Complete automation for deploying production-ready GitLab on Digital Ocean with custom domain, SSL, email delivery, automated backups, and monitoring. + +## Features + +- ✅ Automated GitLab CE installation +- ✅ Let's Encrypt SSL with auto-renewal +- ✅ Multiple email provider support (Gmail, SendGrid, Mailgun, AWS SES) +- ✅ Automated daily backups with cloud storage option +- ✅ Health monitoring scripts +- ✅ Security hardening and firewall rules +- ✅ Performance tuning for 4GB+ RAM droplets +- ✅ Comprehensive testing suite +- ✅ Complete documentation + +## Quick Start + +### 1. Prerequisites + +- Digital Ocean droplet (4GB RAM minimum, 8GB recommended) +- Domain name with DNS access +- Email provider account (see docs/EMAIL_SETUP.md) +- SSH access to droplet + +### 2. Local Setup + +```bash +# Clone or create this directory structure +cd gitlab-deployment + +# Copy environment template +cp .env.example .env + +# Edit with your configuration +nano .env + +# Make scripts executable +chmod +x scripts/*.sh tests/*.sh +``` + +### 3. Configure DNS + +**Before deployment**, add this A record to your DNS: + +``` +Type: A +Name: gitlab (or @ for root domain) +Value: YOUR_DROPLET_IP +TTL: 3600 +``` + +Wait for DNS propagation: `dig gitlab.yourdomain.com` + +### 4. Deploy GitLab + +Run scripts in order: + +```bash +# 1. Setup droplet +ssh root@your_droplet_ip "bash -s" < scripts/setup_droplet.sh + +# 2. Install GitLab (takes 5-10 minutes) +ssh root@your_droplet_ip "bash -s" < scripts/install_gitlab.sh + +# 3. Configure SSL +ssh root@your_droplet_ip "bash -s" < scripts/configure_ssl.sh + +# 4. Configure email (see docs/EMAIL_SETUP.md first!) +ssh root@your_droplet_ip "bash -s" < scripts/configure_email.sh + +# 5. Setup email DNS records +./scripts/setup_dns_records.sh +# Follow output to add DNS records + +# 6. Test email +./scripts/test_email.sh +``` + +### 5. First Login + +```bash +# Get initial password +ssh root@your_droplet_ip 'cat /etc/gitlab/initial_root_password' + +# Visit your GitLab +https://gitlab.yourdomain.com + +# Login as root with the password above +# IMMEDIATELY change the password! +``` + +### 6. Setup Automated Backups + +```bash +ssh root@your_droplet_ip +crontab -e + +# Add daily backup at 2 AM +0 2 * * * /root/gitlab-deployment/scripts/backup_gitlab.sh >> /var/log/gitlab_backup.log 2>&1 +``` + +## Documentation + +- **[Deployment Guide](docs/DEPLOYMENT.md)** - Complete step-by-step deployment +- **[Email Setup](docs/EMAIL_SETUP.md)** - Email configuration for all providers +- **[Testing Guide](docs/TESTING.md)** - Comprehensive testing procedures +- **[Troubleshooting](docs/TROUBLESHOOTING.md)** - Common issues and solutions + +## Requirements + +### Minimum +- 4GB RAM, 2 vCPU cores +- 25GB SSD storage +- Ubuntu 22.04 LTS + +### Recommended +- 8GB RAM, 4 vCPU cores +- 50GB SSD storage +- Ubuntu 22.04 LTS + +### For 50+ Users +- 16GB RAM, 8 vCPU cores +- 100GB SSD storage +- Ubuntu 22.04 LTS + +## Project Structure + +``` +gitlab-deployment/ +├── README.md # This file +├── .env.example # Environment variables template +├── scripts/ +│ ├── setup_droplet.sh # Initial server setup +│ ├── install_gitlab.sh # GitLab installation +│ ├── configure_ssl.sh # SSL certificate setup +│ ├── configure_email.sh # Email configuration +│ ├── setup_dns_records.sh # DNS record generator +│ ├── test_email.sh # Email testing suite +│ ├── backup_gitlab.sh # Backup automation +│ └── health_check.sh # Health monitoring +├── configs/ +│ └── gitlab.rb.template # GitLab configuration template +├── docs/ +│ ├── DEPLOYMENT.md # Deployment guide +│ ├── EMAIL_SETUP.md # Email setup guide +│ ├── TESTING.md # Testing procedures +│ └── TROUBLESHOOTING.md # Troubleshooting guide +└── tests/ + └── integration_tests.sh # Automated testing +``` + +## Security Notes + +1. **Change root password immediately** after first login +2. **Enable 2FA** for all admin accounts +3. **Review SSH key access** regularly +4. **Keep GitLab updated** monthly +5. **Monitor logs** for suspicious activity +6. **Use strong passwords** for all accounts +7. **Rotate credentials** every 90 days + +## Backup & Recovery + +### Create Backup +```bash +ssh root@your_droplet_ip +sudo gitlab-backup create +``` + +### Restore Backup +```bash +# Stop services +sudo gitlab-ctl stop puma +sudo gitlab-ctl stop sidekiq + +# Restore (replace TIMESTAMP) +sudo gitlab-backup restore BACKUP=TIMESTAMP + +# Restart +sudo gitlab-ctl restart +sudo gitlab-rake gitlab:check SANITIZE=true +``` + +Backups stored in: `/var/opt/gitlab/backups/` + +## Updating GitLab + +```bash +# SSH to droplet +ssh root@your_droplet_ip + +# Create backup first! +sudo gitlab-backup create + +# Update +sudo apt update +sudo apt upgrade gitlab-ce + +# Verify +sudo gitlab-rake gitlab:check +``` + +## Monitoring + +Run health checks: +```bash +ssh root@your_droplet_ip '/root/gitlab-deployment/scripts/health_check.sh' +``` + +Set up automated monitoring: +```bash +# Edit crontab +crontab -e + +# Add hourly health check +0 * * * * /root/gitlab-deployment/scripts/health_check.sh >> /var/log/gitlab_health.log 2>&1 +``` + +## Cost Estimate (Monthly) + +- **Droplet (4GB):** $24/month +- **Droplet (8GB):** $48/month +- **Email (SendGrid):** Free (100 emails/day) +- **Email (Mailgun):** Free (5,000 emails/month) +- **Backups (DO Spaces):** $5/month (250GB) +- **Domain:** $10-15/year + +**Total:** ~$24-48/month + +## Common Issues + +### GitLab won't start +```bash +# Check memory and disk space +free -h +df -h + +# Check logs +sudo gitlab-ctl tail +``` + +### SSL certificate issues +```bash +# Verify DNS +dig gitlab.yourdomain.com + +# Renew certificate +sudo gitlab-ctl renew-le-certs +``` + +### Email not working +See **[docs/EMAIL_SETUP.md](docs/EMAIL_SETUP.md)** for comprehensive troubleshooting. + +### More help +See **[docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md)** + +## Support + +- **Documentation:** docs/ +- **GitLab Docs:** docs.gitlab.com +- **GitLab Forum:** forum.gitlab.com +- **Digital Ocean Community:** digitalocean.com/community + +## License + +This deployment configuration is provided as-is for personal and commercial use. + +## Contributing + +Improvements welcome! Please test thoroughly before submitting changes. + +## Next Steps After Deployment + +1. Import existing repositories +2. Set up CI/CD pipelines +3. Configure integrations (Slack, Discord, etc.) +4. Set up project templates +5. Configure issue boards and milestones +6. Explore GitLab Container Registry (optional) +7. Set up GitLab Pages for documentation (optional) + +## Resources + +- [GitLab Documentation](https://docs.gitlab.com) +- [Digital Ocean Tutorials](https://www.digitalocean.com/community/tutorials) +- [Let's Encrypt Documentation](https://letsencrypt.org/docs/) +- [Git Documentation](https://git-scm.com/doc) + +--- + +**Version:** 1.0.0 +**Last Updated:** 2024 diff --git a/configs/gitlab.rb.template b/configs/gitlab.rb.template new file mode 100644 index 0000000..bec80ae --- /dev/null +++ b/configs/gitlab.rb.template @@ -0,0 +1,30 @@ +## GitLab Configuration Template +## Copy to /etc/gitlab/gitlab.rb and customize + +external_url 'https://gitlab.yourdomain.com' + +## Email Settings (Configured via scripts/configure_email.sh) +# See scripts for automatic configuration + +## Backup Configuration +gitlab_rails['backup_keep_time'] = 604800 # 7 days in seconds +gitlab_rails['backup_path'] = "/var/opt/gitlab/backups" + +## Performance Tuning (for 4GB RAM) +postgresql['shared_buffers'] = "256MB" +postgresql['max_worker_processes'] = 8 +sidekiq['max_concurrency'] = 10 + +## Monitoring +prometheus_monitoring['enable'] = true + +## Git LFS +gitlab_rails['lfs_enabled'] = true + +## Container Registry (Optional) +# registry_external_url 'https://registry.yourdomain.com' +# gitlab_rails['registry_enabled'] = true + +## Pages (Optional - for hosting static sites) +# pages_external_url 'https://pages.yourdomain.com' +# gitlab_pages['enable'] = true diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..dd8642c --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,244 @@ +# GitLab Deployment Guide + +## Prerequisites + +- Digital Ocean account with droplet created (4GB RAM minimum) +- Domain name with DNS access +- Email provider account (Gmail, SendGrid, Mailgun, or AWS SES) +- Local machine with SSH access + +## Local Setup + +1. Clone this repository or create the directory structure +2. Copy `.env.example` to `.env` +3. Fill in your environment variables (see EMAIL_SETUP.md for email config) +4. Make scripts executable: + ```bash + chmod +x scripts/*.sh + ``` + +## DNS Configuration (BEFORE DEPLOYMENT) + +Configure your DNS before running scripts: + +### 1. GitLab Domain (A Record) +- Name: `gitlab` (or `@` for root domain) +- Type: A +- Value: Your droplet IP address +- TTL: 3600 + +### 2. Wait for DNS Propagation +Check with: `dig gitlab.yourdomain.com` + +Expected output should show your droplet IP. + +## Deployment Steps + +### Step 1: Initial Droplet Setup +```bash +ssh root@your_droplet_ip "bash -s" < scripts/setup_droplet.sh +``` + +This script: +- Updates system packages +- Configures firewall (UFW) +- Creates swap file for memory management +- Installs essential tools + +### Step 2: Install GitLab +```bash +ssh root@your_droplet_ip "bash -s" < scripts/install_gitlab.sh +``` + +This script: +- Adds GitLab repository +- Installs GitLab CE +- Performs initial configuration + +⏱️ This step takes 5-10 minutes. + +### Step 3: Configure SSL +```bash +ssh root@your_droplet_ip "bash -s" < scripts/configure_ssl.sh +``` + +This script: +- Enables Let's Encrypt +- Configures automatic certificate renewal +- Enforces HTTPS + +### Step 4: Configure Email (CRITICAL) + +Email is required for GitLab to function properly. + +1. **Choose email provider** (see docs/EMAIL_SETUP.md for details): + - Gmail (testing only, 500 emails/day limit) + - SendGrid (recommended for production, 100 emails/day free) + - Mailgun (5,000 emails/month free) + - AWS SES (best for scale, $0.10/1000 emails) + +2. **Update .env with email settings** + +3. **Run email configuration:** + ```bash + ssh root@your_droplet_ip "bash -s" < scripts/configure_email.sh + ``` + +4. **Configure DNS records for email:** + ```bash + ./scripts/setup_dns_records.sh + ``` + Follow the output to add SPF, DMARC, and DKIM records to your DNS. + +5. **Configure Reverse DNS in Digital Ocean:** + - Go to your droplet → Networking tab + - Click Edit next to your IP address + - Set Reverse DNS to: `gitlab.yourdomain.com` + +6. **Wait for DNS propagation (10-60 minutes)** + +7. **Test email delivery:** + ```bash + ./scripts/test_email.sh + ``` + +8. **Verify test email received** (check spam folder too) + +⚠️ **DO NOT PROCEED** until email is working - GitLab won't function properly without it. + +### Step 5: Initial Login + +1. Visit `https://gitlab.yourdomain.com` +2. Get initial root password: + ```bash + ssh root@your_droplet_ip 'cat /etc/gitlab/initial_root_password' + ``` +3. Login as `root` with that password +4. **Immediately change the password** +5. Set up your user account +6. Configure 2FA (recommended) + +### Step 6: Configure Automated Backups + +```bash +# Add to crontab on the droplet +ssh root@your_droplet_ip +crontab -e + +# Add this line (daily backup at 2 AM): +0 2 * * * /root/gitlab-deployment/scripts/backup_gitlab.sh >> /var/log/gitlab_backup.log 2>&1 +``` + +Optional: Configure cloud backup to Digital Ocean Spaces or S3 +- Install and configure s3cmd +- Update GITLAB_BACKUP_BUCKET in .env +- Backups will automatically upload to cloud storage + +### Step 7: Post-Deployment Configuration + +1. **Configure Admin Settings:** + - Admin Area → Settings → General + - Set sign-up restrictions + - Configure session duration + - Set rate limits + +2. **Create User Accounts:** + - Admin Area → Users → New User + - Or enable user registration with approval + +3. **Configure SSH Keys:** + - User Settings → SSH Keys + - Add your public SSH key for git operations + +4. **Create Your First Project:** + - New Project → Create blank project + - Test git clone and push + +5. **Configure CI/CD Runners (Optional):** + - Admin Area → CI/CD → Runners + - Register a runner if you need CI/CD + +## Testing + +See TESTING.md for comprehensive testing procedures. + +## Monitoring + +Set up health check cron job: +```bash +# Check health every hour +0 * * * * /root/gitlab-deployment/scripts/health_check.sh >> /var/log/gitlab_health.log 2>&1 +``` + +## Troubleshooting + +See TROUBLESHOOTING.md for common issues and solutions. + +## Security Hardening + +1. **Change root password immediately after first login** +2. **Enable 2FA for all admin accounts** +3. **Review SSH key access regularly** +4. **Keep GitLab updated:** + ```bash + sudo apt update + sudo apt upgrade gitlab-ce + ``` +5. **Monitor logs for suspicious activity** +6. **Set up fail2ban (optional but recommended)** + +## Backup & Recovery + +### Manual Backup +```bash +ssh root@your_droplet_ip +sudo gitlab-backup create +``` + +### Restore from Backup +```bash +# Stop processes that connect to the database +sudo gitlab-ctl stop puma +sudo gitlab-ctl stop sidekiq + +# Restore (replace TIMESTAMP with your backup file timestamp) +sudo gitlab-backup restore BACKUP=TIMESTAMP + +# Restart GitLab +sudo gitlab-ctl restart +sudo gitlab-rake gitlab:check SANITIZE=true +``` + +## Updating GitLab + +```bash +# SSH into droplet +ssh root@your_droplet_ip + +# Create backup before updating +sudo gitlab-backup create + +# Update GitLab +sudo apt update +sudo apt upgrade gitlab-ce + +# Verify update +sudo gitlab-rake gitlab:check +``` + +## Cost Optimization + +- **Droplet Size:** Start with 4GB RAM ($24/month), scale as needed +- **Backups:** Use object storage (DO Spaces or S3) - cheaper than snapshots +- **Email:** Use SendGrid free tier (100 emails/day) or Mailgun (5,000/month) +- **Monitoring:** Use built-in Prometheus instead of external services + +## Next Steps After Deployment + +1. Import existing repositories +2. Set up CI/CD pipelines +3. Configure integrations (Slack, Discord, etc.) +4. Set up project templates +5. Configure issue boards and milestones +6. Explore GitLab Container Registry (optional) +7. Set up GitLab Pages for documentation (optional) diff --git a/docs/EMAIL_SETUP.md b/docs/EMAIL_SETUP.md new file mode 100644 index 0000000..3715d2b --- /dev/null +++ b/docs/EMAIL_SETUP.md @@ -0,0 +1,322 @@ +# Email Configuration Guide + +## Overview + +GitLab requires email for: +- User registration and password resets +- Notifications (commits, issues, merge requests) +- Two-factor authentication codes +- System alerts + +Without proper email configuration, your GitLab instance will not function correctly. + +## Email Provider Options + +### Option 1: Gmail (Simple, Good for Testing) + +**Pros:** Free, easy setup, reliable +**Cons:** Daily sending limits (500/day), requires app password, not recommended for production + +**Setup:** +1. Enable 2FA on your Gmail account +2. Generate App Password: Google Account → Security → 2-Step Verification → App passwords +3. Select "Mail" and your device +4. Copy the 16-character password +5. Use these settings in `.env`: + ```bash + EMAIL_METHOD=smtp + SMTP_ADDRESS=smtp.gmail.com + SMTP_PORT=587 + SMTP_USER_NAME=your-email@gmail.com + SMTP_PASSWORD=your-16-char-app-password + SMTP_DOMAIN=gmail.com + SMTP_AUTHENTICATION=login + SMTP_ENABLE_STARTTLS_AUTO=true + SMTP_TLS=false + SMTP_OPENSSL_VERIFY_MODE=peer + ``` + +### Option 2: SendGrid (Recommended for Production) + +**Pros:** 100 emails/day free, excellent deliverability, good for production, easy setup +**Cons:** Requires account verification, may need to warm up domain + +**Setup:** +1. Sign up at sendgrid.com +2. Verify your email address +3. Create API Key: Settings → API Keys → Create API Key + - Give it a name + - Select "Full Access" + - Copy the API key (you won't see it again) +4. Authenticate domain: Settings → Sender Authentication → Domain Authentication + - Follow wizard to add DNS records + - This improves deliverability significantly +5. Use these settings in `.env`: + ```bash + EMAIL_METHOD=sendgrid + SENDGRID_API_KEY=your_sendgrid_api_key + SMTP_DOMAIN=yourdomain.com + ``` + +### Option 3: Mailgun (Good Balance) + +**Pros:** 5,000 emails/month free, good API, flexible, reliable +**Cons:** Requires domain verification, slight learning curve + +**Setup:** +1. Sign up at mailgun.com +2. Add and verify your domain + - Go to Sending → Domains → Add New Domain + - Add the DNS records provided (TXT, CNAME, MX) + - Wait for verification (usually 5-10 minutes) +3. Get SMTP credentials from domain settings +4. Use these settings in `.env`: + ```bash + EMAIL_METHOD=mailgun + MAILGUN_API_KEY=your_mailgun_api_key + MAILGUN_DOMAIN=mg.yourdomain.com + SMTP_DOMAIN=yourdomain.com + ``` + +### Option 4: AWS SES (Best for Scale) + +**Pros:** Highly scalable, extremely cheap ($0.10/1000 emails), reliable, production-grade +**Cons:** Requires AWS account, starts in sandbox mode, more complex setup + +**Setup:** +1. Create AWS account +2. Go to AWS SES console +3. Verify your domain and email addresses +4. Request production access if needed (for sending to any address) +5. Create SMTP credentials: Account Dashboard → SMTP Settings → Create SMTP Credentials +6. Use these settings in `.env`: + ```bash + EMAIL_METHOD=ses + AWS_SES_ACCESS_KEY_ID=your_access_key + AWS_SES_SECRET_ACCESS_KEY=your_secret_key + AWS_SES_REGION=us-east-1 + SMTP_DOMAIN=yourdomain.com + ``` + +## DNS Configuration (CRITICAL) + +Without proper DNS records, your emails WILL go to spam or bounce entirely. + +### 1. SPF Record (Sender Policy Framework) +Tells receiving servers that your droplet is authorized to send email for your domain. + +``` +Type: TXT +Name: @ (or leave blank for root domain) +Value: v=spf1 ip4:YOUR_DROPLET_IP include:_spf.gmail.com ~all +TTL: 3600 +``` + +Replace `YOUR_DROPLET_IP` with your actual droplet IP. + +If using SendGrid, use: `v=spf1 include:sendgrid.net ~all` +If using Mailgun, use: `v=spf1 include:mailgun.org ~all` + +### 2. DMARC Record (Domain-based Message Authentication) +Tells receiving servers how to handle emails that fail authentication. + +``` +Type: TXT +Name: _dmarc +Value: v=DMARC1; p=quarantine; rua=mailto:admin@yourdomain.com +TTL: 3600 +``` + +This tells servers to quarantine suspicious emails and send reports to your admin email. + +### 3. DKIM Record (DomainKeys Identified Mail) +Digital signature for your emails. Get from your email provider: + +- **SendGrid:** Settings → Sender Authentication → Domain Authentication → Follow wizard +- **Mailgun:** Domains → Select Domain → Domain Settings → Copy CNAME records +- **AWS SES:** Verified Identities → Select Domain → DKIM tab → Copy records + +These will be CNAME records that look like: +``` +Type: CNAME +Name: s1._domainkey +Value: s1.domainkey.u1234567.wl.sendgrid.net +TTL: 3600 +``` + +### 4. Reverse DNS (PTR Record) +Links your IP back to your domain. Configure in Digital Ocean: + +1. Go to your droplet in Digital Ocean dashboard +2. Click "Networking" tab +3. Find your droplet's IP address +4. Click "Edit" button next to the IP +5. Enter "Reverse DNS": `gitlab.yourdomain.com` +6. Save + +This is critical - many mail servers reject email from IPs without reverse DNS. + +### 5. MX Record (If Receiving Email - Optional) +Only needed if you want to receive email at your domain. + +``` +Type: MX +Name: @ (or leave blank) +Priority: 10 +Value: gitlab.yourdomain.com +TTL: 3600 +``` + +## DNS Verification + +After adding DNS records, verify them: + +```bash +# Check SPF +dig TXT yourdomain.com | grep spf + +# Check DMARC +dig TXT _dmarc.yourdomain.com + +# Check DKIM (replace with your actual record name) +dig CNAME s1._domainkey.yourdomain.com + +# Check MX +dig MX yourdomain.com + +# Check Reverse DNS +dig -x YOUR_DROPLET_IP +``` + +Wait 10-60 minutes for DNS propagation before testing email. + +## Testing Email Setup + +### Quick Test via Script +```bash +./scripts/test_email.sh +``` + +### Manual Test via GitLab Console +```bash +ssh root@your_droplet_ip +gitlab-rails console + +# In the console: +Notify.test_email('your@email.com', 'Test Subject', 'Test Body').deliver_now +exit +``` + +### Check Email Logs +```bash +ssh root@your_droplet_ip +tail -f /var/log/gitlab/gitlab-rails/production.log | grep -i mail +``` + +### Test Email Deliverability Score +1. Send test email to: check@mail-tester.com +2. Visit mail-tester.com and enter the unique address +3. Review your score (aim for 9/10 or higher) + +## Troubleshooting + +### Emails Going to Spam + +**Check:** +- ✅ SPF record is set correctly +- ✅ DKIM is configured and passing +- ✅ DMARC is set +- ✅ Reverse DNS is configured +- ✅ Not sending from a residential IP +- ✅ Domain has been "warmed up" (start with low volume) + +**Solutions:** +1. Use mail-tester.com to identify issues +2. Check your IP reputation: mxtoolbox.com/SuperTool.aspx +3. Request delisting if blacklisted +4. Switch to a dedicated email service (SendGrid, Mailgun) + +### Emails Not Sending At All + +**Check SMTP settings:** +```bash +ssh root@your_droplet_ip +gitlab-rails console + +# Check SMTP configuration +ActionMailer::Base.smtp_settings + +# Test SMTP connection +gitlab-rake gitlab:smtp:check +``` + +**Common issues:** +- Wrong SMTP credentials (especially with Gmail app passwords) +- Firewall blocking outbound port 587/465 +- SMTP server requires TLS +- Email provider blocking connection from your IP + +### Connection Refused / Timeout + +1. **Check firewall allows outbound SMTP:** + ```bash + ssh root@your_droplet_ip + sudo ufw status + # Should allow outbound traffic by default + ``` + +2. **Test SMTP connection manually:** + ```bash + telnet smtp.gmail.com 587 + # Should connect successfully + ``` + +3. **Check if Digital Ocean blocks SMTP:** + - New accounts may have SMTP blocked to prevent spam + - Contact DO support to unblock port 25/587 + +### Gmail "Less Secure Apps" Error + +- Gmail no longer supports "less secure apps" +- You **MUST** use an App Password +- Enable 2FA first, then generate App Password +- Use the 16-character app password, not your regular password + +## Production Checklist + +Before going live, verify: + +- [ ] Email provider account created and verified +- [ ] API key/SMTP credentials generated and working +- [ ] Domain authenticated with email provider +- [ ] SPF record added to DNS and verified +- [ ] DKIM configured and passing +- [ ] DMARC record added to DNS and verified +- [ ] Reverse DNS configured in Digital Ocean +- [ ] Test email sent successfully +- [ ] Test email received (not in spam) +- [ ] Email deliverability score checked (mail-tester.com) +- [ ] Monitoring configured for email delivery +- [ ] Backup email method configured (optional) + +## Email Provider Comparison + +| Provider | Free Tier | Best For | Setup Difficulty | Deliverability | +|----------|-----------|----------|------------------|----------------| +| Gmail | 500/day | Testing | Easy | Good | +| SendGrid | 100/day | Production | Medium | Excellent | +| Mailgun | 5,000/month | Production | Medium | Excellent | +| AWS SES | 62,000/month* | Scale | Hard | Excellent | + +*First year only with AWS Free Tier + +## Recommended Configuration + +For most self-hosted GitLab instances: + +1. **Testing/Personal:** Use Gmail with App Password +2. **Small Team (<50 users):** Use SendGrid free tier +3. **Medium Team (50-500 users):** Use Mailgun or SendGrid paid +4. **Large Team (500+ users):** Use AWS SES + +All require proper DNS configuration for best results. diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 0000000..dd9a042 --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,312 @@ +# GitLab Testing Procedures + +## Pre-Deployment Tests (Local Environment) + +Run these tests before deploying to production. + +### 1. DNS Resolution Test +```bash +# Test A record +dig gitlab.yourdomain.com + +# Should return your droplet IP +# Alternative using nslookup +nslookup gitlab.yourdomain.com +``` + +**Expected Result:** Your droplet IP address should be returned. + +### 2. SSH Access Test +```bash +# Test SSH connection with verbose output +ssh -v root@your_droplet_ip + +# Should connect without errors +``` + +**Expected Result:** Successful SSH connection to droplet. + +### 3. Port Accessibility Test +```bash +# Test required ports +nc -zv your_droplet_ip 22 # SSH +nc -zv your_droplet_ip 80 # HTTP +nc -zv your_droplet_ip 443 # HTTPS + +# All should show "succeeded" +``` + +**Expected Result:** All three ports should be accessible. + +## Post-Deployment Tests + +Run these tests after each deployment step. + +### 1. Service Status Check +```bash +ssh root@your_droplet_ip 'gitlab-ctl status' +``` + +**Expected Result:** All services should be "run" status. + +### 2. HTTPS/SSL Test +```bash +# Test HTTPS response +curl -I https://gitlab.yourdomain.com + +# Should return 200 OK with HTTPS headers + +# Test SSL certificate +openssl s_client -connect gitlab.yourdomain.com:443 -servername gitlab.yourdomain.com + +# Should show valid certificate from Let's Encrypt +``` + +**Expected Result:** +- HTTP 200 OK response +- Valid Let's Encrypt certificate +- No SSL warnings + +### 3. Web Interface Test + +**Manual Steps:** +1. Visit `https://gitlab.yourdomain.com` in browser +2. Verify no certificate warnings +3. Should see GitLab login page +4. Get root password: `ssh root@your_droplet_ip 'cat /etc/gitlab/initial_root_password'` +5. Login with username `root` and the password +6. Should successfully reach GitLab dashboard + +**Expected Result:** Successful login and functional UI. + +### 4. Git Operations Test (HTTPS) +```bash +# Create a test repository via web UI first +# Then test clone: +git clone https://gitlab.yourdomain.com/root/test-repo.git +cd test-repo + +# Create test file +echo "# Test Repository" > README.md + +# Commit and push +git add README.md +git commit -m "Initial commit" +git push origin main +``` + +**Expected Result:** Successful clone, commit, and push operations. + +### 5. SSH Git Access Test +```bash +# First, add your SSH key in GitLab UI: +# User Settings → SSH Keys → Add new key + +# Test SSH connection +ssh -T git@gitlab.yourdomain.com +# Should return: Welcome to GitLab, @username! + +# Clone via SSH +git clone git@gitlab.yourdomain.com:root/test-repo.git test-repo-ssh +cd test-repo-ssh + +# Make changes +echo "SSH test" >> README.md +git add README.md +git commit -m "SSH test commit" +git push origin main +``` + +**Expected Result:** Successful SSH authentication and git operations. + +### 6. Email Delivery Test + +Run the comprehensive email test script: +```bash +./scripts/test_email.sh +``` + +**Manual Email Test:** +```bash +ssh root@your_droplet_ip +gitlab-rails console + +# Send test email +Notify.test_email('your@email.com', 'GitLab Test', 'This is a test').deliver_now +exit + +# Check logs +tail -f /var/log/gitlab/gitlab-rails/production.log | grep -i mail +``` + +**Expected Result:** +- Test email received within 5 minutes +- Email NOT in spam folder +- Email has correct from address +- All DNS records verified (SPF, DKIM, DMARC) + +### 7. Backup Test +```bash +# Run backup script +ssh root@your_droplet_ip '/root/gitlab-deployment/scripts/backup_gitlab.sh' + +# Verify backup file created +ssh root@your_droplet_ip 'ls -lh /var/opt/gitlab/backups/' + +# Should show recent .tar file +``` + +**Expected Result:** +- Backup completes without errors +- Backup file exists in /var/opt/gitlab/backups/ +- Backup file size is reasonable (not empty) + +### 8. Health Check Test +```bash +# Run health check script +ssh root@your_droplet_ip '/root/gitlab-deployment/scripts/health_check.sh' +``` + +**Expected Result:** +- All services running +- Adequate disk space (>20% free) +- Reasonable memory usage (<80%) +- Health endpoint returns success +- Valid SSL certificate + +## Integration Tests + +### GitLab Rake Checks +```bash +ssh root@your_droplet_ip 'sudo gitlab-rake gitlab:check' +``` + +**Expected Result:** All checks should pass or show warnings only (no failures). + +### GitLab Environment Info +```bash +ssh root@your_droplet_ip 'sudo gitlab-rake gitlab:env:info' +``` + +Review output for correct configuration. + +### Database Connectivity +```bash +ssh root@your_droplet_ip 'sudo gitlab-rake gitlab:db:check' +``` + +**Expected Result:** Database connection successful. + +## Monitoring Checklist + +Create this checklist for regular monitoring: + +- [ ] GitLab web UI accessible and responsive +- [ ] SSL certificate valid and auto-renewing +- [ ] Git clone/push operations work via HTTPS +- [ ] Git clone/push operations work via SSH +- [ ] Email delivery working (test weekly) +- [ ] Emails not going to spam +- [ ] Backups completing successfully (check logs) +- [ ] All GitLab services running +- [ ] Disk space adequate (>20% free) +- [ ] Memory usage reasonable (<80%) +- [ ] No errors in logs +- [ ] SSL certificate expiry > 30 days +- [ ] DNS records still valid + +## Automated Testing Script + +Create `tests/integration_tests.sh`: + +```bash +#!/bin/bash +# Run all integration tests + +source .env + +echo "=== GitLab Integration Tests ===" + +FAILED=0 + +# Test 1: HTTP Response +echo -n "Testing HTTP response... " +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" https://${GITLAB_DOMAIN}) +if [ "$HTTP_CODE" = "200" ]; then + echo "✓ PASS" +else + echo "✗ FAIL (HTTP $HTTP_CODE)" + FAILED=$((FAILED + 1)) +fi + +# Test 2: SSL Certificate +echo -n "Testing SSL certificate... " +if echo | openssl s_client -servername ${GITLAB_DOMAIN} -connect ${GITLAB_DOMAIN}:443 2>/dev/null | grep -q "Verify return code: 0"; then + echo "✓ PASS" +else + echo "✗ FAIL" + FAILED=$((FAILED + 1)) +fi + +# Test 3: Services Running +echo -n "Testing GitLab services... " +if ssh root@${DROPLET_IP} 'gitlab-ctl status' | grep -q "run:"; then + echo "✓ PASS" +else + echo "✗ FAIL" + FAILED=$((FAILED + 1)) +fi + +# Test 4: Disk Space +echo -n "Testing disk space... " +DISK_USAGE=$(ssh root@${DROPLET_IP} "df -h / | tail -1 | awk '{print \$5}' | sed 's/%//'") +if [ "$DISK_USAGE" -lt 80 ]; then + echo "✓ PASS (${DISK_USAGE}% used)" +else + echo "✗ FAIL (${DISK_USAGE}% used - critically high)" + FAILED=$((FAILED + 1)) +fi + +# Test 5: Email DNS Records +echo -n "Testing email DNS records... " +DOMAIN=$(echo $GITLAB_EMAIL_FROM | cut -d'@' -f2) +if dig +short TXT ${DOMAIN} | grep -q "spf"; then + echo "✓ PASS" +else + echo "⚠ WARNING (SPF not found)" +fi + +# Summary +echo "" +echo "=== Test Summary ===" +if [ $FAILED -eq 0 ]; then + echo "✓ All tests passed" + exit 0 +else + echo "✗ $FAILED test(s) failed" + exit 1 +fi +``` + +Make executable: `chmod +x tests/integration_tests.sh` + +## Production Readiness Checklist + +Before declaring production ready: + +- [ ] All pre-deployment tests pass +- [ ] All post-deployment tests pass +- [ ] Integration tests pass +- [ ] Email delivery works (not in spam) +- [ ] Backup and restore tested successfully +- [ ] Load testing completed satisfactorily +- [ ] Disaster recovery procedure tested +- [ ] Monitoring and alerting configured +- [ ] Documentation reviewed and updated +- [ ] Credentials rotated and secured +- [ ] Team trained on GitLab usage +- [ ] Support plan in place + +## Troubleshooting Tests + +If any test fails, see TROUBLESHOOTING.md for solutions. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md new file mode 100644 index 0000000..5b4dc54 --- /dev/null +++ b/docs/TROUBLESHOOTING.md @@ -0,0 +1,478 @@ +# GitLab Troubleshooting Guide + +## Common Issues and Solutions + +### 1. GitLab Not Starting + +**Symptoms:** +- Services won't start +- Services keep crashing +- 502 Bad Gateway error + +**Diagnosis:** +```bash +# Check service status +sudo gitlab-ctl status + +# Check logs for errors +sudo gitlab-ctl tail + +# Check disk space +df -h + +# Check memory +free -h +``` + +**Solutions:** + +**A. Out of Memory:** +```bash +# Check memory usage +free -h + +# If memory is full, restart services +sudo gitlab-ctl restart + +# Add swap if not present +sudo fallocate -l 4G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab + +# Consider upgrading droplet size if issue persists +``` + +**B. Disk Space Full:** +```bash +# Check disk usage +df -h + +# Find large files +sudo du -h /var | sort -rh | head -20 + +# Clean up old backups +sudo find /var/opt/gitlab/backups -type f -mtime +7 -delete + +# Clean up logs +sudo gitlab-ctl cleanup-logs + +# Consider adding more storage or upgrading droplet +``` + +**C. Services Not Starting:** +```bash +# Check specific service +sudo gitlab-ctl status servicename + +# View service logs +sudo gitlab-ctl tail servicename + +# Restart specific service +sudo gitlab-ctl restart servicename + +# Full reconfigure +sudo gitlab-ctl reconfigure +``` + +### 2. SSL Certificate Issues + +**Symptoms:** +- Certificate not issuing +- HTTPS not working +- Browser shows certificate error +- Let's Encrypt failing + +**Diagnosis:** +```bash +# Check Let's Encrypt logs +sudo gitlab-ctl tail lets-encrypt + +# Check certificate status +echo | openssl s_client -servername gitlab.yourdomain.com -connect gitlab.yourdomain.com:443 2>/dev/null | openssl x509 -noout -dates + +# Verify DNS is correct +dig gitlab.yourdomain.com +``` + +**Solutions:** + +**A. DNS Not Pointing to Server:** +```bash +# Verify A record +dig gitlab.yourdomain.com + +# Should return your droplet IP +# If not, update DNS and wait for propagation (up to 48 hours, usually 10-60 minutes) +``` + +**B. Ports Not Open:** +```bash +# Check firewall +sudo ufw status + +# Allow HTTP and HTTPS +sudo ufw allow http +sudo ufw allow https +sudo ufw reload +``` + +**C. Manual Certificate Renewal:** +```bash +# Force certificate renewal +sudo gitlab-ctl renew-le-certs + +# If fails, try reconfigure +sudo gitlab-ctl reconfigure +``` + +### 3. 502 Bad Gateway + +**Symptoms:** +- 502 error when accessing GitLab +- Page won't load + +**Diagnosis:** +```bash +# Check if services are running +sudo gitlab-ctl status + +# Check nginx logs +sudo gitlab-ctl tail nginx +``` + +**Solutions:** + +**A. Services Starting Up:** +GitLab can take 5-10 minutes to fully start. Wait and refresh. + +**B. Services Crashed:** +```bash +# Restart all services +sudo gitlab-ctl restart + +# Wait 5 minutes then check status +sudo gitlab-ctl status + +# If still failing, check logs +sudo gitlab-ctl tail +``` + +**C. Nginx Configuration Error:** +```bash +# Test nginx configuration +sudo gitlab-ctl nginx -t + +# Reconfigure +sudo gitlab-ctl reconfigure +``` + +### 4. Email Issues + +**Quick Checks:** +```bash +# Test SMTP connection +gitlab-rake gitlab:smtp:check + +# Send test email +gitlab-rails runner "Notify.test_email('your@email.com', 'Test', 'Body').deliver_now" + +# Check email logs +tail -f /var/log/gitlab/gitlab-rails/production.log | grep -i mail +``` + +See docs/EMAIL_SETUP.md for comprehensive email troubleshooting. + +### 5. Git Push/Pull Failures + +**Symptoms:** +- Can't push or pull +- Authentication errors +- Connection refused + +**Diagnosis:** +```bash +# Test HTTPS git access +git clone https://gitlab.yourdomain.com/root/test.git + +# Test SSH git access +ssh -T git@gitlab.yourdomain.com +``` + +**Solutions:** + +**A. SSH Key Issues:** +```bash +# Check SSH keys in GitLab UI: User Settings → SSH Keys + +# Test SSH connection +ssh -vT git@gitlab.yourdomain.com + +# Generate new SSH key if needed +ssh-keygen -t ed25519 -C "your_email@example.com" +cat ~/.ssh/id_ed25519.pub # Add to GitLab +``` + +**B. HTTPS Authentication:** +```bash +# Use personal access token instead of password +# GitLab UI: User Settings → Access Tokens → Create token + +# Clone with token +git clone https://oauth2:TOKEN@gitlab.yourdomain.com/user/repo.git +``` + +### 6. Backup Failures + +**Symptoms:** +- Backup script failing +- Backups not completing +- Backup files missing + +**Diagnosis:** +```bash +# Check disk space +df -h + +# Check backup logs +tail -f /var/log/gitlab_backup.log + +# Try manual backup +sudo gitlab-backup create +``` + +**Solutions:** + +**A. Out of Disk Space:** +```bash +# Clean old backups +sudo find /var/opt/gitlab/backups -type f -mtime +7 -delete + +# Move backups to object storage +# Configure s3cmd for DO Spaces or AWS S3 +``` + +**B. Permissions Issues:** +```bash +# Fix backup directory permissions +sudo chown -R git:git /var/opt/gitlab/backups +sudo chmod 0700 /var/opt/gitlab/backups +``` + +### 7. Slow Performance + +**Symptoms:** +- GitLab is slow to load +- Git operations timeout +- High CPU or memory usage + +**Diagnosis:** +```bash +# Check resource usage +htop + +# Check disk I/O +iostat -x 1 + +# Check GitLab performance +sudo gitlab-rake gitlab:check +``` + +**Solutions:** + +**A. Insufficient Resources:** +Upgrade your droplet: +- Minimum: 4GB RAM, 2 vCPUs +- Recommended: 8GB RAM, 4 vCPUs +- For >50 users: 16GB RAM, 8 vCPUs + +**B. Database Issues:** +```bash +# Analyze and optimize database +sudo gitlab-rake db:migrate + +# Vacuum database +sudo gitlab-psql -c "VACUUM ANALYZE;" +``` + +**C. Performance Tuning:** +Edit /etc/gitlab/gitlab.rb: +```ruby +# PostgreSQL tuning +postgresql['shared_buffers'] = "256MB" +postgresql['work_mem'] = "16MB" +postgresql['maintenance_work_mem'] = "64MB" + +# Sidekiq tuning +sidekiq['max_concurrency'] = 10 + +# Puma tuning +puma['worker_processes'] = 2 +puma['max_threads'] = 4 +``` + +Then reconfigure: +```bash +sudo gitlab-ctl reconfigure +``` + +### 8. User Can't Login + +**Symptoms:** +- "Invalid login or password" error +- Account locked +- 2FA issues + +**Solutions:** + +**A. Reset Root Password:** +```bash +# Access GitLab console +sudo gitlab-rails console + +# Find and reset password +user = User.where(username: 'root').first +user.password = 'newpassword' +user.password_confirmation = 'newpassword' +user.save! +exit +``` + +**B. Unlock Account:** +```bash +sudo gitlab-rails console + +user = User.find_by(username: 'username') +user.unlock_access! +exit +``` + +**C. Disable 2FA:** +```bash +sudo gitlab-rails console + +user = User.find_by(username: 'username') +user.disable_two_factor! +exit +``` + +### 9. Database Connection Issues + +**Symptoms:** +- "Could not connect to database" error +- Database timeout errors + +**Solutions:** + +**A. Restart Database:** +```bash +sudo gitlab-ctl restart postgresql +``` + +**B. Check Database Status:** +```bash +sudo gitlab-ctl status postgresql + +# Check connections +sudo gitlab-psql -c "SELECT count(*) FROM pg_stat_activity;" +``` + +**C. Reset Database Connections:** +```bash +sudo gitlab-rake db:migrate:status +sudo gitlab-ctl restart +``` + +## Getting More Help + +### Check System Status +```bash +# Comprehensive check +sudo gitlab-rake gitlab:check + +# Environment info +sudo gitlab-rake gitlab:env:info + +# Check configuration +sudo gitlab-rake gitlab:check_config +``` + +### View All Logs +```bash +# Tail all logs +sudo gitlab-ctl tail + +# Specific service +sudo gitlab-ctl tail nginx +sudo gitlab-ctl tail gitlab-rails +sudo gitlab-ctl tail sidekiq +sudo gitlab-ctl tail postgresql +``` + +## Useful Commands Reference + +```bash +# Service Management +sudo gitlab-ctl start +sudo gitlab-ctl stop +sudo gitlab-ctl restart +sudo gitlab-ctl status + +# Configuration +sudo gitlab-ctl reconfigure +sudo gitlab-ctl show-config + +# Logs +sudo gitlab-ctl tail +sudo gitlab-ctl tail SERVICE_NAME + +# Maintenance +sudo gitlab-ctl cleanup-logs +sudo gitlab-rake gitlab:check + +# Backups +sudo gitlab-backup create +sudo gitlab-backup restore BACKUP=timestamp + +# Console Access +sudo gitlab-rails console +sudo gitlab-psql + +# Updates +sudo apt update +sudo apt upgrade gitlab-ce +``` + +## Prevention Best Practices + +1. **Monitor Resource Usage** + - Set up alerts for disk space (<20% free) + - Monitor memory usage + - Check CPU load regularly + +2. **Regular Backups** + - Automate daily backups + - Test restore procedure monthly + - Store backups off-server + +3. **Keep Updated** + - Update GitLab monthly + - Subscribe to security announcements + - Test updates in staging first + +4. **Monitor Logs** + - Check logs weekly for errors + - Set up log aggregation + - Configure error notifications + +5. **Document Everything** + - Keep change log + - Document customizations + - Maintain runbook + +## Emergency Contacts + +- **GitLab Community Forum:** forum.gitlab.com +- **GitLab Documentation:** docs.gitlab.com +- **Digital Ocean Support:** cloud.digitalocean.com/support diff --git a/scripts/backup_gitlab.sh b/scripts/backup_gitlab.sh new file mode 100755 index 0000000..f02e8b9 --- /dev/null +++ b/scripts/backup_gitlab.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Automated GitLab backup script + +set -e + +source .env + +BACKUP_DIR="/var/opt/gitlab/backups" +RETENTION_DAYS=7 + +echo "=== Creating GitLab Backup ===" + +# Create backup +sudo gitlab-backup create STRATEGY=copy + +# Find the latest backup +LATEST_BACKUP=$(find "${BACKUP_DIR}" -name "*.tar" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-) + +echo "Backup created: ${LATEST_BACKUP}" + +# Optional: Upload to Digital Ocean Spaces or S3 +if [ -n "${GITLAB_BACKUP_BUCKET}" ]; then + echo "Uploading to cloud storage..." + # Install s3cmd if not present + if ! command -v s3cmd &> /dev/null; then + sudo apt install -y s3cmd + fi + + # Upload (configure s3cmd separately for DO Spaces) + s3cmd put "${LATEST_BACKUP}" "s3://${GITLAB_BACKUP_BUCKET}/" + echo "✓ Backup uploaded to ${GITLAB_BACKUP_BUCKET}" +fi + +# Clean up old backups (keep last 7 days) +find "${BACKUP_DIR}" -name "*.tar" -type f -mtime +"${RETENTION_DAYS}" -delete + +echo "✓ Backup complete" diff --git a/scripts/configure_email.sh b/scripts/configure_email.sh new file mode 100755 index 0000000..3dcea71 --- /dev/null +++ b/scripts/configure_email.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# Configure email delivery for GitLab + +set -e + +source .env + +echo "=== Configuring GitLab Email Settings ===" + +# Backup original config +sudo cp /etc/gitlab/gitlab.rb /etc/gitlab/gitlab.rb.email_backup + +# Function to configure SMTP +configure_smtp() { + echo "Configuring SMTP email delivery..." + sudo bash -c "cat >> /etc/gitlab/gitlab.rb" <> /etc/gitlab/gitlab.rb" <> /etc/gitlab/gitlab.rb" <> /etc/gitlab/gitlab.rb" <> /etc/gitlab/gitlab.rb" </dev/null | openssl x509 -noout -dates + +echo "✓ Health check complete" diff --git a/scripts/install_gitlab.sh b/scripts/install_gitlab.sh new file mode 100755 index 0000000..bbcec0c --- /dev/null +++ b/scripts/install_gitlab.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Install and configure GitLab + +set -e + +source .env + +echo "=== Installing GitLab CE ===" + +# Add GitLab repository +curl -sS https://packages.gitlab.com/install/repositories/gitlab/gitlab-ce/script.deb.sh | sudo bash + +# Install GitLab +sudo EXTERNAL_URL="https://${GITLAB_DOMAIN}" apt install -y gitlab-ce + +# Wait for GitLab to start +echo "Waiting for GitLab to initialize..." +sleep 30 + +echo "✓ GitLab installed successfully" +echo "Initial root password location: /etc/gitlab/initial_root_password" diff --git a/scripts/setup_dns_records.sh b/scripts/setup_dns_records.sh new file mode 100755 index 0000000..fdc212a --- /dev/null +++ b/scripts/setup_dns_records.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Generate DNS records for email authentication + +source .env + +DOMAIN=$(echo "$GITLAB_EMAIL_FROM" | cut -d'@' -f2) + +echo "=== DNS Records Required for Email Delivery ===" +echo "" +echo "Add these records to your DNS provider:" +echo "" +echo "1. SPF Record (TXT)" +echo " Name: @" +echo " Type: TXT" +echo " Value: v=spf1 ip4:${DROPLET_IP} include:_spf.${SMTP_DOMAIN} ~all" +echo "" +echo "2. DMARC Record (TXT)" +echo " Name: _dmarc" +echo " Type: TXT" +echo " Value: v=DMARC1; p=quarantine; rua=mailto:${ADMIN_EMAIL}" +echo "" +echo "3. MX Record (if receiving email)" +echo " Name: @" +echo " Type: MX" +echo " Priority: 10" +echo " Value: ${GITLAB_DOMAIN}" +echo "" +echo "4. Reverse DNS (PTR) Record" +echo " Configure in Digital Ocean droplet settings:" +echo " Networking → Edit → Reverse DNS → ${GITLAB_DOMAIN}" +echo "" + +# If using SendGrid, show additional records +if [ "$EMAIL_METHOD" = "sendgrid" ]; then + echo "5. SendGrid Domain Authentication Records" + echo " Log into SendGrid → Settings → Sender Authentication" + echo " Follow the wizard to get your specific CNAME records" + echo "" +fi + +# If using Mailgun +if [ "$EMAIL_METHOD" = "mailgun" ]; then + echo "5. Mailgun Domain Verification Records" + echo " Log into Mailgun → Sending → Domains → ${MAILGUN_DOMAIN}" + echo " Copy the TXT and CNAME records shown" + echo "" +fi + +echo "⚠ After adding DNS records, wait 10-60 minutes for propagation" +echo "⚠ Use 'dig TXT ${DOMAIN}' to verify SPF record" +echo "⚠ Use 'dig TXT _dmarc.${DOMAIN}' to verify DMARC record" diff --git a/scripts/setup_droplet.sh b/scripts/setup_droplet.sh new file mode 100755 index 0000000..1bf4459 --- /dev/null +++ b/scripts/setup_droplet.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Initial droplet setup and hardening + +set -e + +source .env + +echo "=== Setting up Digital Ocean Droplet ===" + +# Update system +sudo apt update && sudo apt upgrade -y + +# Install essential packages +sudo apt install -y curl openssh-server ca-certificates tzdata perl ufw + +# Configure firewall +sudo ufw --force enable +sudo ufw allow OpenSSH +sudo ufw allow http +sudo ufw allow https + +# Install postfix for email (lightweight MTA) +sudo DEBIAN_FRONTEND=noninteractive apt install -y postfix + +# Set timezone +sudo timedatectl set-timezone UTC + +# Create swap file if not exists (helps with 4GB RAM) +if [ ! -f /swapfile ]; then + sudo fallocate -l 4G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab +fi + +# Install monitoring tools +sudo apt install -y htop ncdu + +echo "✓ Droplet setup complete" diff --git a/scripts/test_email.sh b/scripts/test_email.sh new file mode 100755 index 0000000..de5eeb5 --- /dev/null +++ b/scripts/test_email.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Comprehensive email testing + +set -e + +source .env + +echo "=== GitLab Email Testing Suite ===" + +# Test 1: Check SMTP connection +echo -e "\n1. Testing SMTP Connection..." +ssh root@"${DROPLET_IP}" "gitlab-rails runner \" + smtp = Net::SMTP.new('${SMTP_ADDRESS}', ${SMTP_PORT}) + smtp.enable_starttls + smtp.start('${SMTP_DOMAIN}', '${SMTP_USER_NAME}', '${SMTP_PASSWORD}', :login) do + puts '✓ SMTP connection successful' + end +\"" + +# Test 2: Send test email via GitLab console +echo -e "\n2. Sending test email via GitLab console..." +ssh root@"${DROPLET_IP}" "gitlab-rails runner \" + Notify.test_email('${ADMIN_EMAIL}', 'GitLab Email Test', 'If you receive this, email is working!').deliver_now + puts '✓ Test email queued' +\"" + +# Test 3: Check email logs +echo -e "\n3. Checking email delivery logs..." +ssh root@"${DROPLET_IP}" "tail -n 50 /var/log/gitlab/gitlab-rails/production.log | grep -i 'mail\|smtp\|email'" + +# Test 4: Verify DNS records +echo -e "\n4. Verifying DNS records..." +DOMAIN=$(echo "$GITLAB_EMAIL_FROM" | cut -d'@' -f2) + +echo " SPF Record:" +dig +short TXT "${DOMAIN}" | grep spf || echo " ⚠ SPF record not found" + +echo " DMARC Record:" +dig +short TXT "_dmarc.${DOMAIN}" || echo " ⚠ DMARC record not found" + +echo " MX Record:" +dig +short MX "${DOMAIN}" || echo " ⚠ MX record not found" + +# Test 5: Check reverse DNS +echo -e "\n5. Checking reverse DNS..." +REVERSE_DNS=$(dig +short -x "${DROPLET_IP}") +if [ -n "$REVERSE_DNS" ]; then + echo " ✓ Reverse DNS: $REVERSE_DNS" +else + echo " ⚠ No reverse DNS configured (configure in Digital Ocean)" +fi + +echo -e "\n=== Test Summary ===" +echo "Check your inbox at ${ADMIN_EMAIL} for test emails" +echo "If no email received, check spam folder and review logs above" diff --git a/tests/integration_tests.sh b/tests/integration_tests.sh new file mode 100755 index 0000000..8113bc8 --- /dev/null +++ b/tests/integration_tests.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Run all integration tests + +source .env + +echo "=== GitLab Integration Tests ===" + +FAILED=0 + +# Test 1: HTTP Response +echo -n "Testing HTTP response... " +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "https://${GITLAB_DOMAIN}") +if [ "$HTTP_CODE" = "200" ]; then + echo "✓ PASS" +else + echo "✗ FAIL (HTTP $HTTP_CODE)" + FAILED=$((FAILED + 1)) +fi + +# Test 2: SSL Certificate +echo -n "Testing SSL certificate... " +if echo | openssl s_client -servername "${GITLAB_DOMAIN}" -connect "${GITLAB_DOMAIN}:443" 2>/dev/null | grep -q "Verify return code: 0"; then + echo "✓ PASS" +else + echo "✗ FAIL" + FAILED=$((FAILED + 1)) +fi + +# Test 3: Services Running +echo -n "Testing GitLab services... " +if ssh root@"${DROPLET_IP}" 'gitlab-ctl status' | grep -q "run:"; then + echo "✓ PASS" +else + echo "✗ FAIL" + FAILED=$((FAILED + 1)) +fi + +# Test 4: Disk Space +echo -n "Testing disk space... " +DISK_USAGE=$(ssh root@"${DROPLET_IP}" "df -h / | tail -1 | awk '{print \$5}' | sed 's/%//'") +if [ "$DISK_USAGE" -lt 80 ]; then + echo "✓ PASS (${DISK_USAGE}% used)" +else + echo "✗ FAIL (${DISK_USAGE}% used - critically high)" + FAILED=$((FAILED + 1)) +fi + +# Test 5: Email DNS Records +echo -n "Testing email DNS records... " +DOMAIN=$(echo "$GITLAB_EMAIL_FROM" | cut -d'@' -f2) +if dig +short TXT "${DOMAIN}" | grep -q "spf"; then + echo "✓ PASS" +else + echo "⚠ WARNING (SPF not found)" +fi + +# Summary +echo "" +echo "=== Test Summary ===" +if [ $FAILED -eq 0 ]; then + echo "✓ All tests passed" + exit 0 +else + echo "✗ $FAILED test(s) failed" + exit 1 +fi