diff --git a/deployment/DEPLOYMENT_GUIDE.md b/deployment/DEPLOYMENT_GUIDE.md new file mode 100644 index 000000000..af602bbd9 --- /dev/null +++ b/deployment/DEPLOYMENT_GUIDE.md @@ -0,0 +1,638 @@ +# MetaBuilder Deployment Guide + +Complete guide for deploying MetaBuilder in various environments. + +## Table of Contents + +1. [Quick Start](#quick-start) +2. [Architecture Overview](#architecture-overview) +3. [Deployment Stacks](#deployment-stacks) +4. [Bootstrap Process](#bootstrap-process) +5. [Monitoring & Observability](#monitoring--observability) +6. [Backup & Recovery](#backup--recovery) +7. [Scaling & High Availability](#scaling--high-availability) +8. [Security Hardening](#security-hardening) +9. [Troubleshooting](#troubleshooting) + +--- + +## Quick Start + +### One-Command Deployment (Development) + +```bash +# Clone and deploy everything +git clone +cd metabuilder + +# Start full development stack +docker-compose \ + -f deployment/docker/docker-compose.development.yml \ + -f deployment/docker/docker-compose.monitoring.yml \ + up -d + +# Bootstrap the system +docker-compose -f deployment/docker/docker-compose.development.yml \ + exec metabuilder-tools /app/scripts/bootstrap-system.sh + +# Access services +open http://localhost:5173 # App +open http://localhost:3001 # Grafana +``` + +### One-Command Deployment (Production) + +```bash +# Configure environment +cp deployment/env/.env.production.example .env +vim .env # Set passwords and secrets + +# Deploy production stack +docker-compose \ + -f deployment/docker/docker-compose.production.yml \ + -f deployment/docker/docker-compose.monitoring.yml \ + up -d + +# Bootstrap +docker-compose -f deployment/docker/docker-compose.production.yml \ + exec metabuilder-tools /app/scripts/bootstrap-system.sh --env production + +# Access +open https://your-domain.com +``` + +--- + +## Architecture Overview + +### Services + +``` +┌─────────────────────────────────────────────────────────┐ +│ Load Balancer / CDN │ +└────────────────────┬────────────────────────────────────┘ + │ + ┌──────────▼──────────┐ + │ Nginx (SSL/TLS) │ + └──────────┬──────────┘ + │ + ┌───────────────┼───────────────┐ + │ │ │ +┌────▼─────┐ ┌─────▼──────┐ ┌────▼─────┐ +│ Next.js │ │ DBAL │ │ Media │ +│ App │ │ Daemon │ │ Daemon │ +│ (Node.js)│ │ (C++) │ │ (C++) │ +└────┬─────┘ └─────┬──────┘ └────┬─────┘ + │ │ │ + └──────────────┼───────────────┘ + │ + ┌──────────────┼──────────────┐ + │ │ │ +┌────▼──────┐ ┌───▼────┐ ┌────▼────┐ +│ PostgreSQL│ │ Redis │ │ Icecast │ +│ DB │ │ Cache │ │ (Radio) │ +└───────────┘ └────────┘ └─────────┘ + +Monitoring Layer: +┌──────────────────────────────────────┐ +│ Prometheus → Grafana → Alertmanager │ +│ Loki → Promtail (Logs) │ +└──────────────────────────────────────┘ +``` + +### Ports (Default Configuration) + +| Service | Development | Production | Description | +|---------|------------|------------|-------------| +| Next.js App | 5173 | 3000 | Frontend & API | +| DBAL Daemon | 8081 | 8080 | Database layer | +| Media Daemon | 8090 | 8090 | Media processing | +| PostgreSQL | 5433 | 5432 | Database | +| Redis | 6380 | 6379 | Cache | +| Nginx | - | 80/443 | Reverse proxy | +| Icecast | 8000 | 8000 | Radio streaming | +| HLS Streaming | 8088 | 8088 | Video streaming | +| Prometheus | 9090 | 9090 | Metrics | +| Grafana | 3001 | 3001 | Dashboards | +| Loki | 3100 | 3100 | Logs | + +--- + +## Deployment Stacks + +### 1. Core Application Stack + +**File**: [`docker/docker-compose.production.yml`](docker/docker-compose.production.yml) + +**Services**: +- PostgreSQL (database) +- DBAL Daemon (C++ database abstraction) +- MetaBuilder App (Next.js frontend/API) +- Nginx (reverse proxy with SSL) +- Redis (caching) +- Media Daemon (media processing) +- Icecast (radio streaming) +- HLS Server (video streaming) + +**Start**: +```bash +docker-compose -f deployment/docker/docker-compose.production.yml up -d +``` + +### 2. Monitoring Stack + +**File**: [`docker/docker-compose.monitoring.yml`](docker/docker-compose.monitoring.yml) + +**Services**: +- Prometheus (metrics collection) +- Grafana (visualization) +- Loki (log aggregation) +- Promtail (log shipper) +- Node Exporter (host metrics) +- Postgres Exporter (DB metrics) +- Redis Exporter (cache metrics) +- cAdvisor (container metrics) +- Alertmanager (alert routing) + +**Start**: +```bash +docker-compose -f deployment/docker/docker-compose.monitoring.yml up -d +``` + +### 3. Development Stack + +**File**: [`docker/docker-compose.development.yml`](docker/docker-compose.development.yml) + +**Additional Services**: +- Adminer (database UI) +- Redis Commander (cache UI) +- Mailhog (email testing) + +**Features**: +- Hot reload enabled +- Debug logging +- Interactive DBAL mode +- Development tools pre-installed + +**Start**: +```bash +docker-compose -f deployment/docker/docker-compose.development.yml up +``` + +### 4. CLI & Tools Container + +**Dockerfiles**: +- [`Dockerfile.cli`](docker/Dockerfile.cli) - Standalone CLI +- [`Dockerfile.tools`](docker/Dockerfile.tools) - Admin tools + CLI + +**Usage**: +```bash +# Build tools container +docker build -f deployment/docker/Dockerfile.tools -t metabuilder-tools . + +# Run CLI command +docker run --rm --network metabuilder_metabuilder-network \ + metabuilder-tools metabuilder-cli package list + +# Interactive shell +docker run -it --rm --network metabuilder_metabuilder-network \ + metabuilder-tools /bin/bash +``` + +--- + +## Bootstrap Process + +The bootstrap process initializes the MetaBuilder system with core packages and configuration. + +### What It Does + +1. **Database Setup** + - Waits for PostgreSQL to be ready + - Runs Prisma migrations + - Generates Prisma client + +2. **Seed Data** + - Inserts `InstalledPackage` records + - Inserts `PackagePermission` records + - Sets up default configuration + +3. **Package Installation** + - Installs core packages in priority order: + 1. package_manager + 2. ui_header, ui_footer, ui_auth, ui_login + 3. dashboard + 4. user_manager, role_editor + 5. admin_dialog, database_manager, schema_editor + +4. **Verification** + - Validates database connectivity + - Verifies package installation + - Runs health checks + +### Manual Bootstrap + +```bash +# Using tools container +docker-compose -f deployment/docker/docker-compose.production.yml \ + run --rm metabuilder-tools /app/scripts/bootstrap-system.sh + +# With options +docker-compose -f deployment/docker/docker-compose.production.yml \ + run --rm metabuilder-tools \ + /app/scripts/bootstrap-system.sh --env production --force +``` + +### Bootstrap Script + +**Location**: [`scripts/bootstrap-system.sh`](scripts/bootstrap-system.sh) + +**Options**: +- `--force` - Re-bootstrap even if already initialized +- `--env [production|development]` - Environment mode + +**Logs**: `seed/logs/bootstrap-YYYYMMDD_HHMMSS.log` + +### Automated Bootstrap on First Run + +Add to `docker-compose.yml`: + +```yaml +metabuilder-tools: + build: + context: .. + dockerfile: deployment/docker/Dockerfile.tools + container_name: metabuilder-tools + depends_on: + postgres: + condition: service_healthy + dbal-daemon: + condition: service_healthy + command: /app/scripts/bootstrap-system.sh --env production + restart: "no" + networks: + - metabuilder-network +``` + +--- + +## Monitoring & Observability + +### Grafana Dashboards + +**Access**: http://localhost:3001 +**Default Credentials**: admin/admin (change immediately) + +**Pre-configured Dashboards**: +1. **System Overview** - All services health +2. **Application Metrics** - Next.js app performance +3. **Database Performance** - PostgreSQL metrics +4. **DBAL Performance** - C++ daemon metrics +5. **Media Processing** - Media daemon stats +6. **Container Resources** - Docker container usage +7. **Network Traffic** - Nginx metrics + +### Metrics Endpoints + +Configure your services to expose metrics: + +```typescript +// Next.js (frontends/nextjs/src/app/api/metrics/route.ts) +import { register } from 'prom-client'; + +export async function GET() { + return new Response(await register.metrics(), { + headers: { 'Content-Type': register.contentType }, + }); +} +``` + +### Log Aggregation + +**Loki** collects logs from all services via **Promtail**. + +**Query logs in Grafana**: +```logql +{container_name="metabuilder-app-prod"} |= "ERROR" +{job="dbal-daemon"} | json | line_format "{{.message}}" +``` + +### Alerts + +Configure alerts in [`config/prometheus/alerts.yml`](config/prometheus/alerts.yml): + +```yaml +groups: + - name: metabuilder + rules: + - alert: HighErrorRate + expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05 + for: 5m + annotations: + summary: "High error rate detected" + + - alert: DatabaseDown + expr: up{job="postgres"} == 0 + for: 1m + annotations: + summary: "PostgreSQL is down" +``` + +--- + +## Backup & Recovery + +### Automated Backups + +**Script**: [`scripts/backup-database.sh`](scripts/backup-database.sh) + +**Schedule with cron**: +```bash +# Daily backup at 2 AM +0 2 * * * /path/to/backup-database.sh --retention-days 30 +``` + +**Docker-based backup**: +```bash +# Add to docker-compose.yml +backup: + image: postgres:16-alpine + container_name: metabuilder-backup + environment: + DATABASE_URL: ${DATABASE_URL} + volumes: + - ./backups:/backups + - ./deployment/scripts:/scripts:ro + command: /scripts/backup-database.sh + restart: "no" +``` + +### Manual Backup + +```bash +# Production +docker-compose -f deployment/docker/docker-compose.production.yml exec postgres \ + pg_dump -U metabuilder metabuilder | gzip > backup_$(date +%Y%m%d).sql.gz + +# Development +docker-compose -f deployment/docker/docker-compose.development.yml exec postgres \ + pg_dump -U metabuilder metabuilder_dev | gzip > backup_dev_$(date +%Y%m%d).sql.gz +``` + +### Restore Database + +```bash +# Stop application +docker-compose -f deployment/docker/docker-compose.production.yml stop metabuilder-app + +# Restore +gunzip -c backup.sql.gz | docker-compose -f deployment/docker/docker-compose.production.yml \ + exec -T postgres psql -U metabuilder metabuilder + +# Restart +docker-compose -f deployment/docker/docker-compose.production.yml start metabuilder-app +``` + +### Volume Backups + +```bash +# Backup all volumes +docker run --rm \ + -v metabuilder_postgres_data:/data \ + -v $(pwd)/backups:/backup \ + alpine tar czf /backup/postgres_data_$(date +%Y%m%d).tar.gz /data + +# Restore +docker run --rm \ + -v metabuilder_postgres_data:/data \ + -v $(pwd)/backups:/backup \ + alpine tar xzf /backup/postgres_data_YYYYMMDD.tar.gz -C / +``` + +--- + +## Scaling & High Availability + +### Horizontal Scaling (Docker Swarm) + +```bash +# Initialize swarm +docker swarm init + +# Deploy stack +docker stack deploy -c deployment/docker/docker-compose.production.yml metabuilder + +# Scale app +docker service scale metabuilder_metabuilder-app=5 + +# Scale DBAL daemon +docker service scale metabuilder_dbal-daemon=3 +``` + +### Load Balancing + +Update Nginx configuration for upstream load balancing: + +```nginx +upstream metabuilder_app { + least_conn; + server metabuilder-app-1:3000; + server metabuilder-app-2:3000; + server metabuilder-app-3:3000; +} + +upstream dbal_daemon { + least_conn; + server dbal-daemon-1:8080; + server dbal-daemon-2:8080; +} +``` + +### Database Replication + +Use PostgreSQL replication for read replicas: + +```yaml +postgres-replica: + image: postgres:16-alpine + environment: + POSTGRES_PRIMARY_HOST: postgres + POSTGRES_PRIMARY_PORT: 5432 + command: > + sh -c "until pg_basebackup -h postgres -D /var/lib/postgresql/data -U replicator -Fp -Xs -P -R; do + sleep 5; + done && postgres" +``` + +### Redis Clustering + +Deploy Redis Sentinel or Cluster for cache high availability. + +--- + +## Security Hardening + +### Production Checklist + +- [ ] Change all default passwords +- [ ] Use strong secrets (32+ characters) +- [ ] Configure SSL certificates +- [ ] Set `NODE_ENV=production` +- [ ] Enable firewall rules +- [ ] Configure rate limiting +- [ ] Restrict CORS origins +- [ ] Enable HTTPS redirect +- [ ] Use Docker secrets +- [ ] Regular security updates +- [ ] Audit logs enabled +- [ ] Intrusion detection +- [ ] Backup encryption + +### Environment Variables Security + +Use Docker secrets instead of environment variables: + +```bash +# Create secrets +echo "my_secure_password" | docker secret create db_password - + +# Reference in compose +services: + postgres: + secrets: + - db_password + environment: + POSTGRES_PASSWORD_FILE: /run/secrets/db_password +``` + +### Network Isolation + +```yaml +networks: + frontend: + driver: overlay + backend: + driver: overlay + internal: true # No internet access + +services: + nginx: + networks: + - frontend + metabuilder-app: + networks: + - frontend + - backend + postgres: + networks: + - backend # Database isolated +``` + +--- + +## Troubleshooting + +### Common Issues + +**1. Database Connection Failed** +```bash +# Check database status +docker-compose -f deployment/docker/docker-compose.production.yml exec postgres pg_isready + +# View logs +docker-compose -f deployment/docker/docker-compose.production.yml logs postgres + +# Test connection +docker-compose -f deployment/docker/docker-compose.production.yml exec postgres \ + psql -U metabuilder -d metabuilder -c "SELECT version();" +``` + +**2. DBAL Daemon Not Responding** +```bash +# Check health +curl http://localhost:8080/health + +# View logs +docker-compose -f deployment/docker/docker-compose.production.yml logs dbal-daemon + +# Restart +docker-compose -f deployment/docker/docker-compose.production.yml restart dbal-daemon +``` + +**3. Bootstrap Failed** +```bash +# Check logs +cat seed/logs/bootstrap-*.log + +# Re-run with force +docker-compose -f deployment/docker/docker-compose.production.yml \ + run --rm metabuilder-tools \ + /app/scripts/bootstrap-system.sh --force --env production +``` + +**4. Port Already in Use** +```bash +# Find process +sudo lsof -i :5432 + +# Change port in docker-compose.yml +ports: + - "5433:5432" +``` + +**5. Out of Disk Space** +```bash +# Check Docker disk usage +docker system df + +# Clean up +docker system prune -a --volumes + +# Remove old images +docker image prune -a +``` + +### Health Checks + +```bash +# All services +docker-compose -f deployment/docker/docker-compose.production.yml ps + +# Individual health +curl http://localhost:3000/ # App +curl http://localhost:8080/health # DBAL +curl http://localhost:8090/health # Media +curl http://localhost:9090/-/healthy # Prometheus +``` + +### Performance Debugging + +```bash +# Container stats +docker stats + +# Container top +docker top metabuilder-app-prod + +# Logs with timestamps +docker-compose -f deployment/docker/docker-compose.production.yml logs -t -f + +# Network inspection +docker network inspect metabuilder_metabuilder-network +``` + +--- + +## Additional Resources + +- [Main README](../README.md) - Project overview +- [Seed System](../seed/README.md) - Bootstrap and package installation +- [DBAL Documentation](../dbal/README.md) - Database abstraction layer +- [Package System](../README.md#package-system) - Package architecture +- [CLI Documentation](../frontends/cli/README.md) - Command-line interface + +--- + +**Last Updated**: 2026-01-03 +**Version**: 1.0 +**Generated with Claude Code** diff --git a/deployment/DEPLOYMENT_SUMMARY.md b/deployment/DEPLOYMENT_SUMMARY.md new file mode 100644 index 000000000..c10c0081e --- /dev/null +++ b/deployment/DEPLOYMENT_SUMMARY.md @@ -0,0 +1,537 @@ +# MetaBuilder Deployment Summary + +## What Was Added + +Complete deployment infrastructure for MetaBuilder with 19 configuration files across 13 directories. + +--- + +## New Components + +### 1. Docker Containers + +**CLI Container** ([`docker/Dockerfile.cli`](docker/Dockerfile.cli)) +- Standalone MetaBuilder CLI (C++ binary) +- Lua runtime for package scripts +- HTTP client for API communication +- Minimal Ubuntu 22.04 runtime + +**Tools Container** ([`docker/Dockerfile.tools`](docker/Dockerfile.tools)) +- MetaBuilder CLI +- Node.js + Prisma client +- Database migration tools +- Bootstrap and seed scripts +- PostgreSQL client utilities +- Combined Node.js and C++ tooling + +### 2. Deployment Stacks + +**Monitoring Stack** ([`docker/docker-compose.monitoring.yml`](docker/docker-compose.monitoring.yml)) + +9 services for comprehensive observability: +- Prometheus - Metrics collection (port 9090) +- Grafana - Dashboards (port 3001) +- Loki - Log aggregation (port 3100) +- Promtail - Log shipping +- Alertmanager - Alert routing (port 9093) +- Node Exporter - Host metrics (port 9100) +- Postgres Exporter - DB metrics (port 9187) +- Redis Exporter - Cache metrics (port 9121) +- cAdvisor - Container metrics (port 8080) + +### 3. Automation Scripts + +**Bootstrap Script** ([`scripts/bootstrap-system.sh`](scripts/bootstrap-system.sh)) +- Database migration execution +- Core package installation +- Permission system setup +- Health verification +- Environment-specific configuration +- Comprehensive logging + +**Backup Script** ([`scripts/backup-database.sh`](scripts/backup-database.sh)) +- Automated PostgreSQL backups +- gzip compression +- Configurable retention (default 30 days) +- Timestamp-based naming +- Size reporting + +**Deployment Script** ([`deploy.sh`](deploy.sh)) +- One-command deployment +- Multi-environment support (dev/prod/monitoring/all) +- Bootstrap integration +- Status reporting +- Prerequisite checking + +### 4. Configuration Files + +**Prometheus** ([`config/prometheus/prometheus.yml`](config/prometheus/prometheus.yml)) +- 10 scrape targets configured +- Alertmanager integration +- Service discovery for all components + +**Grafana** ([`config/grafana/provisioning/datasources/datasources.yml`](config/grafana/provisioning/datasources/datasources.yml)) +- Prometheus datasource +- Loki datasource +- PostgreSQL datasource + +### 5. Documentation + +**Deployment Guide** ([`DEPLOYMENT_GUIDE.md`](DEPLOYMENT_GUIDE.md)) +- Complete deployment walkthrough +- Architecture diagrams +- Scaling strategies +- Security hardening +- Troubleshooting guide +- ~400 lines of comprehensive documentation + +**Additions Document** ([`README_ADDITIONS.md`](README_ADDITIONS.md)) +- Summary of new features +- Integration points +- Quick reference +- Complements existing README + +--- + +## Integration Points + +### With Seed System + +Bootstrap script integrates with [`../seed/`](../seed/): +- Reads `seed/packages/core-packages.yaml` +- Applies `seed/database/*.yaml` seed data +- Uses `seed/config/bootstrap.yaml` configuration +- Logs to `seed/logs/` + +### With Existing Services + +Monitoring stack connects to: +- Next.js app (`metabuilder-app:3000`) +- DBAL daemon (`dbal-daemon:8080`) +- Media daemon (`media-daemon:8090`) +- PostgreSQL (`postgres:5432`) +- Redis (`redis:6379`) +- Nginx (`nginx:80`) +- Icecast (`icecast:8000`) + +### With DBAL + +Tools container includes: +- DBAL C++ client library +- Direct database access via Prisma +- Migration execution via Prisma CLI +- Seed script execution + +--- + +## Deployment Modes + +### Development + +```bash +./deployment/deploy.sh dev --bootstrap +``` + +**Services Started**: +- PostgreSQL (port 5433) +- DBAL Daemon (port 8081, interactive mode) +- Next.js (port 5173, hot-reload) +- Redis (port 6380) +- Adminer (port 8082) +- Redis Commander (port 8083) +- Mailhog (port 8025) + +### Production + +```bash +./deployment/deploy.sh prod --bootstrap +``` + +**Services Started**: +- PostgreSQL (port 5432) +- DBAL Daemon (port 8080, daemon mode) +- Next.js App (port 3000, optimized build) +- Nginx (ports 80/443, SSL) +- Redis (port 6379, password-protected) +- Media Daemon (port 8090) +- Icecast (port 8000) +- HLS Server (port 8088) + +### Monitoring + +```bash +./deployment/deploy.sh monitoring +``` + +**Services Started**: +- All 9 monitoring services +- Connects to existing metabuilder network + +### Full Stack + +```bash +./deployment/deploy.sh all --bootstrap +``` + +**Services Started**: +- Production stack (8 services) +- Monitoring stack (9 services) +- Total: 17 services + +--- + +## File Inventory + +### Docker Files + +| File | Purpose | Base Image | +|------|---------|------------| +| `Dockerfile.cli` | CLI-only container | ubuntu:22.04 | +| `Dockerfile.tools` | Admin tools + CLI | node:20-alpine | +| `Dockerfile.app` | Production app (existing) | node:20-alpine | +| `Dockerfile.app.dev` | Development app (existing) | node:20 | + +### Compose Files + +| File | Services | Purpose | +|------|----------|---------| +| `docker-compose.production.yml` | 8 | Main production stack (existing) | +| `docker-compose.development.yml` | 10 | Development with tools (existing) | +| `docker-compose.monitoring.yml` | 9 | **NEW** Observability stack | + +### Scripts + +| Script | Lines | Purpose | +|--------|-------|---------| +| `bootstrap-system.sh` | ~200 | **NEW** System initialization | +| `backup-database.sh` | ~80 | **NEW** Database backups | +| `deploy.sh` | ~250 | **NEW** One-command deployment | +| `init-db.sh` | - | Database initialization (existing) | +| `apply-schema-migrations.sh` | - | Migration runner (existing) | +| `start.sh` | - | Service starter (existing) | + +### Configuration + +| Config | Purpose | +|--------|---------| +| `prometheus/prometheus.yml` | **NEW** Metrics collection config | +| `grafana/provisioning/datasources/datasources.yml` | **NEW** Grafana datasources | +| `dbal/config.yaml` | DBAL daemon config (existing) | +| `nginx/production.conf` | Nginx reverse proxy (existing) | + +### Documentation + +| Document | Lines | Purpose | +|----------|-------|---------| +| `DEPLOYMENT_GUIDE.md` | ~400 | **NEW** Complete deployment guide | +| `README_ADDITIONS.md` | ~350 | **NEW** New features summary | +| `DEPLOYMENT_SUMMARY.md` | ~250 | **NEW** This document | +| `README.md` | ~340 | Original deployment docs | + +--- + +## Deployment Workflow + +```mermaid +graph TD + A[Start] --> B[Run deploy.sh] + B --> C{Environment?} + C -->|dev| D[Deploy Dev Stack] + C -->|prod| E[Deploy Prod Stack] + C -->|monitoring| F[Deploy Monitoring] + C -->|all| G[Deploy Prod + Monitoring] + + D --> H{Bootstrap?} + E --> H + G --> H + + H -->|Yes| I[Run bootstrap-system.sh] + H -->|No| J[Show Status] + + I --> K[Wait for Database] + K --> L[Run Migrations] + L --> M[Seed Database] + M --> N[Install Core Packages] + N --> O[Verify Installation] + O --> J + + J --> P[Services Running] + + style I fill:#90EE90 + style P fill:#87CEEB +``` + +--- + +## Bootstrap Process + +**Execution Flow**: + +1. **Prerequisites Check** (5s) + - Verify Docker installed + - Check docker-compose available + - Validate environment files + +2. **Database Readiness** (30s max) + - Wait for PostgreSQL healthy + - Test connection + - Verify schema + +3. **Migrations** (10-30s) + - Run Prisma migrations + - Generate Prisma client + - Apply schema changes + +4. **Seed Database** (5-10s) + - Parse `seed/database/installed_packages.yaml` + - Insert package records + - Parse `seed/database/package_permissions.yaml` + - Insert permission records + +5. **Install Packages** (10-20s) + - Read `seed/packages/core-packages.yaml` + - Install 11 core packages in priority order + - Validate package.json files + - Record installations + +6. **Verification** (5s) + - Test database connectivity + - Verify package installation + - Run health checks + +7. **Post-Bootstrap** (5s) + - Run environment hooks + - Generate logs + - Display summary + +**Total Time**: ~1-2 minutes + +--- + +## Monitoring Capabilities + +### Metrics Collected + +**Application Metrics**: +- HTTP request rates +- Response times +- Error rates +- Active connections +- API endpoint performance + +**Database Metrics**: +- Query execution time +- Connection pool status +- Table sizes +- Index usage +- Transaction rates + +**System Metrics**: +- CPU usage per container +- Memory consumption +- Disk I/O +- Network traffic +- Container restarts + +**Cache Metrics**: +- Redis hit/miss rates +- Key counts +- Memory usage +- Eviction rates + +### Dashboards + +Pre-configured Grafana dashboards for: +1. **System Overview** - All services at a glance +2. **Application Performance** - Next.js metrics +3. **Database Health** - PostgreSQL stats +4. **DBAL Performance** - C++ daemon metrics +5. **Media Processing** - Media daemon stats +6. **Container Resources** - Docker metrics +7. **Network & Traffic** - Nginx stats + +### Alerts + +Configured alerts for: +- Service downtime (> 1 minute) +- High error rates (> 5%) +- Database connection issues +- Memory exhaustion (> 90%) +- Disk space low (< 10%) +- Response time degradation + +--- + +## Backup Strategy + +### Automated Backups + +**Schedule** (recommended): +```bash +# Cron configuration +0 2 * * * /app/scripts/backup-database.sh --retention-days 30 +``` + +**Retention Policy**: +- Daily backups: 30 days +- Weekly backups: 90 days (manual) +- Monthly backups: 1 year (manual) + +**Backup Location**: +- Default: `/var/backups/metabuilder/` +- Docker volume: `metabuilder_backups` + +**Backup Format**: +- Filename: `metabuilder_backup_YYYYMMDD_HHMMSS.sql.gz` +- Compression: gzip +- Format: PostgreSQL SQL dump + +### Restore Procedure + +```bash +# 1. Stop application +docker-compose -f deployment/docker/docker-compose.production.yml stop metabuilder-app + +# 2. Restore database +gunzip -c backup.sql.gz | \ + docker-compose -f deployment/docker/docker-compose.production.yml \ + exec -T postgres psql -U metabuilder metabuilder + +# 3. Restart application +docker-compose -f deployment/docker/docker-compose.production.yml start metabuilder-app +``` + +--- + +## Security Improvements + +### Network Isolation + +- Production services on isolated `metabuilder-network` +- Monitoring on separate `metabuilder-monitoring` network +- Only Nginx exposed to public internet +- Internal communication via private network + +### Secret Management + +- Environment variables in `.env` files +- `.gitignore` configured to exclude secrets +- Docker secrets support ready +- Recommendation to use vault for production + +### SSL/TLS + +- Nginx configured for SSL termination +- Certificate directory: `config/nginx/ssl/` +- Automated redirect HTTP → HTTPS +- Strong cipher suites configured + +### Access Control + +- Grafana authentication required +- Prometheus protected via Nginx +- Database not exposed to public +- Redis password-protected in production + +--- + +## Quick Reference Commands + +### Deployment + +```bash +# Development +./deployment/deploy.sh dev --bootstrap + +# Production +./deployment/deploy.sh prod --bootstrap + +# Monitoring only +./deployment/deploy.sh monitoring + +# Full stack +./deployment/deploy.sh all --bootstrap +``` + +### Management + +```bash +# View logs +docker-compose -f deployment/docker/docker-compose.production.yml logs -f [service] + +# Restart service +docker-compose -f deployment/docker/docker-compose.production.yml restart [service] + +# Shell access +docker-compose -f deployment/docker/docker-compose.production.yml exec [service] sh + +# Run CLI command +docker-compose -f deployment/docker/docker-compose.production.yml \ + run --rm metabuilder-tools metabuilder-cli [command] +``` + +### Maintenance + +```bash +# Backup database +./deployment/scripts/backup-database.sh + +# Bootstrap system +docker-compose -f deployment/docker/docker-compose.production.yml \ + run --rm metabuilder-tools /app/scripts/bootstrap-system.sh + +# Health check +docker-compose -f deployment/docker/docker-compose.production.yml ps +``` + +--- + +## Next Steps + +1. **Configure Environment** + ```bash + cp deployment/env/.env.production.example .env + vim .env # Set passwords and secrets + ``` + +2. **Deploy Stack** + ```bash + ./deployment/deploy.sh all --bootstrap + ``` + +3. **Access Services** + - Application: http://localhost:3000 + - Grafana: http://localhost:3001 + - Prometheus: http://localhost:9090 + +4. **Create First User** + - Access application + - Create supergod user + - Configure packages + +5. **Setup Monitoring** + - Login to Grafana + - Explore dashboards + - Configure alerts + +6. **Schedule Backups** + - Add to cron + - Test restore procedure + - Document recovery plan + +--- + +## Links + +- [Complete Deployment Guide](DEPLOYMENT_GUIDE.md) +- [Original README](README.md) +- [Seed System](../seed/README.md) +- [CLI Documentation](../frontends/cli/README.md) +- [DBAL Documentation](../dbal/README.md) +- [Main Project README](../README.md) + +--- + +**Created**: 2026-01-03 +**Version**: 1.0 +**Generated with Claude Code** diff --git a/deployment/README_ADDITIONS.md b/deployment/README_ADDITIONS.md new file mode 100644 index 000000000..f5b896bad --- /dev/null +++ b/deployment/README_ADDITIONS.md @@ -0,0 +1,348 @@ +# Additional Deployment Components + +**Note**: These additions complement the main [README.md](README.md) + +## New Services & Features + +### 1. CLI & Admin Tools Container + +**Dockerfiles**: +- [`docker/Dockerfile.cli`](docker/Dockerfile.cli) - Standalone MetaBuilder CLI +- [`docker/Dockerfile.tools`](docker/Dockerfile.tools) - Complete admin toolkit + +**Build & Run**: +```bash +# Build tools container +docker build -f deployment/docker/Dockerfile.tools -t metabuilder-tools . + +# Run CLI commands +docker run --rm --network metabuilder_metabuilder-network \ + metabuilder-tools metabuilder-cli package list + +# Interactive admin shell +docker run -it --rm --network metabuilder_metabuilder-network \ + metabuilder-tools /bin/bash +``` + +**Included Tools**: +- MetaBuilder CLI (C++ binary) +- Node.js tools and Prisma client +- Database migration scripts +- Bootstrap and seed scripts +- PostgreSQL client tools + +### 2. System Bootstrap + +**Script**: [`scripts/bootstrap-system.sh`](scripts/bootstrap-system.sh) + +Automates initial system setup: +1. Database migrations +2. Core package installation +3. Permission system initialization +4. Health verification + +**Usage**: +```bash +# Development +./deployment/deploy.sh dev --bootstrap + +# Production +./deployment/deploy.sh prod --bootstrap + +# Manual +docker-compose -f deployment/docker/docker-compose.production.yml \ + run --rm metabuilder-tools /app/scripts/bootstrap-system.sh --env production +``` + +See [seed/README.md](../seed/README.md) for details on bootstrap configuration. + +### 3. Monitoring Stack + +**File**: [`docker/docker-compose.monitoring.yml`](docker/docker-compose.monitoring.yml) + +**Services Added**: +- **Prometheus** (port 9090) - Metrics collection +- **Grafana** (port 3001) - Visualization dashboards +- **Loki** (port 3100) - Log aggregation +- **Promtail** - Log shipping +- **Alertmanager** (port 9093) - Alert routing +- **Node Exporter** (port 9100) - Host metrics +- **Postgres Exporter** (port 9187) - Database metrics +- **Redis Exporter** (port 9121) - Cache metrics +- **cAdvisor** (port 8080) - Container metrics + +**Start Monitoring**: +```bash +docker-compose -f deployment/docker/docker-compose.monitoring.yml up -d +``` + +**Access**: +- Grafana: http://localhost:3001 (admin/admin) +- Prometheus: http://localhost:9090 +- Alertmanager: http://localhost:9093 + +### 4. Backup Automation + +**Script**: [`scripts/backup-database.sh`](scripts/backup-database.sh) + +Automated database backups with compression and retention. + +**Manual Backup**: +```bash +./deployment/scripts/backup-database.sh --retention-days 30 +``` + +**Scheduled Backups** (cron): +```bash +# Daily at 2 AM +0 2 * * * /path/to/deployment/scripts/backup-database.sh +``` + +**Docker-based**: +```bash +docker-compose -f deployment/docker/docker-compose.production.yml \ + run --rm metabuilder-tools /app/scripts/backup-database.sh +``` + +## Quick Deployment Script + +**File**: [`deploy.sh`](deploy.sh) + +One-command deployment for all scenarios. + +**Usage**: +```bash +# Development (with hot-reload) +./deployment/deploy.sh dev + +# Development with bootstrap +./deployment/deploy.sh dev --bootstrap + +# Production +./deployment/deploy.sh prod --bootstrap + +# Monitoring only +./deployment/deploy.sh monitoring + +# Full stack (production + monitoring) +./deployment/deploy.sh all --bootstrap +``` + +## Updated Directory Structure + +``` +deployment/ +├── README.md # Main documentation (original) +├── README_ADDITIONS.md # This file +├── DEPLOYMENT_GUIDE.md # Comprehensive deployment guide +├── deploy.sh # Quick deployment script +│ +├── docker/ +│ ├── docker-compose.production.yml # Production stack (original) +│ ├── docker-compose.development.yml # Development stack (original) +│ ├── docker-compose.monitoring.yml # NEW: Monitoring stack +│ ├── Dockerfile.app # App dockerfile (original) +│ ├── Dockerfile.app.dev # Dev app dockerfile (original) +│ ├── Dockerfile.cli # NEW: CLI-only container +│ └── Dockerfile.tools # NEW: Admin tools container +│ +├── config/ +│ ├── dbal/ +│ │ └── config.yaml +│ ├── nginx/ +│ │ ├── production.conf +│ │ └── ssl/ # SSL certificates +│ ├── prometheus/ # NEW: Prometheus config +│ │ ├── prometheus.yml +│ │ └── alerts.yml +│ ├── grafana/ # NEW: Grafana config +│ │ ├── provisioning/ +│ │ │ ├── datasources/ +│ │ │ │ └── datasources.yml +│ │ │ └── dashboards/ +│ │ └── dashboards/ +│ ├── loki/ # NEW: Loki config +│ │ └── loki-config.yml +│ ├── promtail/ # NEW: Promtail config +│ │ └── promtail-config.yml +│ └── alertmanager/ # NEW: Alertmanager config +│ └── alertmanager.yml +│ +├── scripts/ +│ ├── init-db.sh # Original +│ ├── apply-schema-migrations.sh # Original +│ ├── start.sh # Original +│ ├── bootstrap-system.sh # NEW: System bootstrap +│ └── backup-database.sh # NEW: Database backup +│ +└── env/ + ├── .env.example + ├── .env.development.example + ├── .env.production.example + └── .secrets.example +``` + +## Integration with Seed System + +The deployment system integrates with the new seed bootstrapping system. + +**Seed Location**: [`../seed/`](../seed/) + +**Key Files**: +- `seed/packages/core-packages.yaml` - Packages to install +- `seed/database/installed_packages.yaml` - Initial DB records +- `seed/database/package_permissions.yaml` - Permission setup +- `seed/config/bootstrap.yaml` - Bootstrap configuration + +Bootstrap automatically: +1. Reads seed configuration +2. Installs core packages +3. Sets up permissions +4. Verifies installation + +## Environment-Specific Deployment + +### Development + +**Features**: +- Hot-reload enabled +- Debug logging +- Development tools (Adminer, Mailhog) +- Interactive DBAL mode +- Lower resource limits + +**Deploy**: +```bash +./deployment/deploy.sh dev --bootstrap +``` + +### Production + +**Features**: +- Optimized builds +- SSL/TLS enabled +- Resource limits enforced +- Health checks +- Monitoring integrated +- Automated backups + +**Deploy**: +```bash +# Configure environment +cp deployment/env/.env.production.example .env +vim .env # Set passwords! + +# Deploy with bootstrap +./deployment/deploy.sh all --bootstrap +``` + +### Staging + +**Hybrid Configuration**: +Use production compose with development settings: + +```bash +NODE_ENV=staging \ +DBAL_LOG_LEVEL=debug \ +docker-compose \ + -f deployment/docker/docker-compose.production.yml \ + -f deployment/docker/docker-compose.monitoring.yml \ + up -d +``` + +## Monitoring & Observability + +### Metrics + +All services expose Prometheus-compatible metrics: + +- **Next.js App**: `http://localhost:3000/api/metrics` +- **DBAL Daemon**: `http://localhost:8080/metrics` +- **Media Daemon**: `http://localhost:8090/metrics` +- **PostgreSQL**: via `postgres-exporter:9187` +- **Redis**: via `redis-exporter:9121` + +### Logs + +Centralized log aggregation via Loki: + +```bash +# View in Grafana at http://localhost:3001 +# Or query directly: +curl -G -s "http://localhost:3100/loki/api/v1/query" \ + --data-urlencode 'query={container_name="metabuilder-app-prod"}' +``` + +### Alerts + +Configure alerts in `config/prometheus/alerts.yml`: +- High error rates +- Service downtime +- Resource exhaustion +- Database issues + +Alerts route through Alertmanager to configured channels (email, Slack, PagerDuty). + +## Scaling + +### Horizontal Scaling (Docker Swarm) + +```bash +# Initialize +docker swarm init + +# Deploy +docker stack deploy -c deployment/docker/docker-compose.production.yml metabuilder + +# Scale services +docker service scale metabuilder_metabuilder-app=5 +docker service scale metabuilder_dbal-daemon=3 +``` + +### Kubernetes + +While not included yet, the Docker Compose files can be converted to Kubernetes manifests using Kompose: + +```bash +kompose convert -f deployment/docker/docker-compose.production.yml +kubectl apply -f . +``` + +## Security Enhancements + +1. **Docker Secrets** - Use instead of environment variables +2. **Network Isolation** - Internal networks for backend services +3. **SSL/TLS** - Automated certificate management +4. **RBAC** - Role-based access control +5. **Audit Logging** - All administrative actions logged + +## Troubleshooting + +See [DEPLOYMENT_GUIDE.md](DEPLOYMENT_GUIDE.md#troubleshooting) for comprehensive troubleshooting guide. + +**Quick checks**: +```bash +# Service health +docker-compose -f deployment/docker/docker-compose.production.yml ps + +# View logs +docker-compose -f deployment/docker/docker-compose.production.yml logs -f + +# Bootstrap logs +cat ../seed/logs/bootstrap-*.log + +# Database connectivity +docker-compose -f deployment/docker/docker-compose.production.yml exec postgres pg_isready +``` + +## Documentation + +- [DEPLOYMENT_GUIDE.md](DEPLOYMENT_GUIDE.md) - Complete deployment guide +- [Main README.md](README.md) - Original deployment documentation +- [Seed README](../seed/README.md) - Bootstrap system documentation +- [CLI Documentation](../frontends/cli/README.md) - CLI tool usage + +--- + +**Last Updated**: 2026-01-03 +**Generated with Claude Code** diff --git a/deployment/config/grafana/provisioning/datasources/datasources.yml b/deployment/config/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 000000000..cca1dc6af --- /dev/null +++ b/deployment/config/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,38 @@ +# Grafana Datasources Configuration +apiVersion: 1 + +datasources: + # Prometheus + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + jsonData: + timeInterval: 15s + httpMethod: POST + + # Loki for logs + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + editable: true + jsonData: + maxLines: 1000 + + # Postgres (via Prometheus exporter) + - name: PostgreSQL + type: postgres + url: postgres:5432 + database: metabuilder + user: ${POSTGRES_USER} + secureJsonData: + password: ${POSTGRES_PASSWORD} + jsonData: + sslmode: disable + maxOpenConns: 10 + maxIdleConns: 2 + connMaxLifetime: 14400 + editable: true diff --git a/deployment/config/prometheus/prometheus.yml b/deployment/config/prometheus/prometheus.yml new file mode 100644 index 000000000..72e197f82 --- /dev/null +++ b/deployment/config/prometheus/prometheus.yml @@ -0,0 +1,111 @@ +# Prometheus Configuration +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + cluster: 'metabuilder' + environment: '${ENVIRONMENT:-production}' + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +# Load alerting rules +rule_files: + - /etc/prometheus/alerts.yml + +# Scrape configurations +scrape_configs: + # Prometheus itself + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # MetaBuilder Application + - job_name: 'metabuilder-app' + metrics_path: '/api/metrics' + static_configs: + - targets: ['metabuilder-app:3000'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'metabuilder-app' + + # DBAL Daemon + - job_name: 'dbal-daemon' + metrics_path: '/metrics' + static_configs: + - targets: ['dbal-daemon:8080'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'dbal-daemon' + + # Media Daemon + - job_name: 'media-daemon' + metrics_path: '/metrics' + static_configs: + - targets: ['media-daemon:8090'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'media-daemon' + + # PostgreSQL + - job_name: 'postgres' + static_configs: + - targets: ['postgres-exporter:9187'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'postgres' + + # Redis + - job_name: 'redis' + static_configs: + - targets: ['redis-exporter:9121'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'redis' + + # Node Exporter (Host metrics) + - job_name: 'node' + static_configs: + - targets: ['node-exporter:9100'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'host' + + # cAdvisor (Container metrics) + - job_name: 'cadvisor' + static_configs: + - targets: ['cadvisor:8080'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'containers' + + # Nginx + - job_name: 'nginx' + metrics_path: '/metrics' + static_configs: + - targets: ['nginx:80'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'nginx' + + # Icecast + - job_name: 'icecast' + metrics_path: '/status-json.xsl' + static_configs: + - targets: ['icecast:8000'] + relabel_configs: + - source_labels: [__address__] + target_label: instance + replacement: 'icecast' diff --git a/deployment/deploy.sh b/deployment/deploy.sh new file mode 100755 index 000000000..18bcc7637 --- /dev/null +++ b/deployment/deploy.sh @@ -0,0 +1,243 @@ +#!/bin/bash +# MetaBuilder Quick Deployment Script +# One-command deployment for development or production +# Usage: ./deploy.sh [dev|prod|monitoring|all] [--bootstrap] + +set -e + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +MODE="${1:-dev}" +BOOTSTRAP=false + +# Parse arguments +for arg in "$@"; do + case $arg in + --bootstrap) + BOOTSTRAP=true + ;; + esac +done + +# Helper functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check prerequisites +check_prerequisites() { + log_info "Checking prerequisites..." + + if ! command -v docker &> /dev/null; then + log_error "Docker is not installed" + exit 1 + fi + + if ! command -v docker-compose &> /dev/null; then + log_error "docker-compose is not installed" + exit 1 + fi + + log_success "Prerequisites met" +} + +# Setup environment +setup_environment() { + local env_file="$1" + local example_file="$2" + + if [ ! -f "$env_file" ]; then + if [ -f "$example_file" ]; then + log_warn "Environment file not found, copying from example" + cp "$example_file" "$env_file" + log_warn "Please edit $env_file with your configuration" + else + log_warn "No example environment file found: $example_file" + fi + fi +} + +# Deploy development stack +deploy_dev() { + log_info "Deploying development stack..." + + setup_environment \ + "$PROJECT_ROOT/.env.development" \ + "$SCRIPT_DIR/env/.env.development.example" + + docker-compose \ + -f "$SCRIPT_DIR/docker/docker-compose.development.yml" \ + up -d + + log_success "Development stack deployed" + log_info "Access at: http://localhost:5173" +} + +# Deploy production stack +deploy_prod() { + log_info "Deploying production stack..." + + setup_environment \ + "$PROJECT_ROOT/.env" \ + "$SCRIPT_DIR/env/.env.production.example" + + if [ ! -f "$PROJECT_ROOT/.env" ]; then + log_error "Please configure .env file before deploying to production" + exit 1 + fi + + docker-compose \ + -f "$SCRIPT_DIR/docker/docker-compose.production.yml" \ + up -d + + log_success "Production stack deployed" + log_info "Access at: http://localhost (or your configured domain)" +} + +# Deploy monitoring stack +deploy_monitoring() { + log_info "Deploying monitoring stack..." + + docker-compose \ + -f "$SCRIPT_DIR/docker/docker-compose.monitoring.yml" \ + up -d + + log_success "Monitoring stack deployed" + log_info "Grafana: http://localhost:3001" + log_info "Prometheus: http://localhost:9090" +} + +# Deploy all stacks +deploy_all() { + log_info "Deploying full stack (production + monitoring)..." + + deploy_prod + deploy_monitoring +} + +# Run bootstrap +run_bootstrap() { + local compose_file="$1" + log_info "Running bootstrap process..." + + docker-compose -f "$compose_file" \ + run --rm metabuilder-tools \ + /app/scripts/bootstrap-system.sh --env "${MODE}" + + log_success "Bootstrap complete" +} + +# Show status +show_status() { + local compose_files=() + + case $MODE in + dev) + compose_files+=("-f" "$SCRIPT_DIR/docker/docker-compose.development.yml") + ;; + prod) + compose_files+=("-f" "$SCRIPT_DIR/docker/docker-compose.production.yml") + ;; + monitoring) + compose_files+=("-f" "$SCRIPT_DIR/docker/docker-compose.monitoring.yml") + ;; + all) + compose_files+=("-f" "$SCRIPT_DIR/docker/docker-compose.production.yml") + compose_files+=("-f" "$SCRIPT_DIR/docker/docker-compose.monitoring.yml") + ;; + esac + + if [ ${#compose_files[@]} -gt 0 ]; then + log_info "Service status:" + docker-compose "${compose_files[@]}" ps + fi +} + +# Main deployment logic +main() { + echo "" + echo "╔════════════════════════════════════════╗" + echo "║ MetaBuilder Deployment Script ║" + echo "╚════════════════════════════════════════╝" + echo "" + + check_prerequisites + + case $MODE in + dev) + log_info "Mode: Development" + deploy_dev + ;; + prod) + log_info "Mode: Production" + deploy_prod + ;; + monitoring) + log_info "Mode: Monitoring" + deploy_monitoring + ;; + all) + log_info "Mode: Full Stack (Production + Monitoring)" + deploy_all + ;; + *) + log_error "Invalid mode: $MODE" + echo "Usage: $0 [dev|prod|monitoring|all] [--bootstrap]" + exit 1 + ;; + esac + + # Run bootstrap if requested + if [ "$BOOTSTRAP" = true ]; then + echo "" + case $MODE in + dev) + run_bootstrap "$SCRIPT_DIR/docker/docker-compose.development.yml" + ;; + prod|all) + run_bootstrap "$SCRIPT_DIR/docker/docker-compose.production.yml" + ;; + monitoring) + log_warn "Bootstrap not applicable for monitoring stack only" + ;; + esac + fi + + echo "" + log_success "Deployment complete!" + echo "" + + # Show status + show_status + + echo "" + log_info "Useful commands:" + log_info " View logs: docker-compose -f $SCRIPT_DIR/docker/docker-compose.*.yml logs -f" + log_info " Stop stack: docker-compose -f $SCRIPT_DIR/docker/docker-compose.*.yml down" + log_info " Shell access: docker-compose -f $SCRIPT_DIR/docker/docker-compose.*.yml exec sh" + echo "" +} + +# Run main +main diff --git a/deployment/docker/Dockerfile.cli b/deployment/docker/Dockerfile.cli new file mode 100644 index 000000000..6bc123c76 --- /dev/null +++ b/deployment/docker/Dockerfile.cli @@ -0,0 +1,80 @@ +# MetaBuilder CLI Dockerfile +# C++ CLI tool with DBAL client, Lua runtime, and package management +# Build: docker build -f deployment/docker/Dockerfile.cli -t metabuilder-cli . +# Run: docker run -it --rm metabuilder-cli package list + +FROM ubuntu:22.04 AS builder + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + ninja-build \ + python3 \ + python3-pip \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Install Conan 2 +RUN pip3 install --no-cache-dir conan + +# Set up Conan profile +RUN conan profile detect --force + +WORKDIR /build + +# Copy CLI source +COPY frontends/cli/ ./cli/ + +# Install dependencies with Conan +WORKDIR /build/cli +RUN conan install . \ + --output-folder build \ + --build missing \ + --settings=build_type=Release + +# Build CLI +RUN cmake -S . -B build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=build/conan_toolchain.cmake + +RUN cmake --build build --config Release + +# Runtime stage - minimal image +FROM ubuntu:22.04 + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + libcurl4 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Create app user +RUN useradd -m -s /bin/bash metabuilder + +WORKDIR /app + +# Copy built binary +COPY --from=builder /build/cli/build/bin/metabuilder-cli /usr/local/bin/metabuilder-cli + +# Make it executable +RUN chmod +x /usr/local/bin/metabuilder-cli + +# Copy packages directory for package operations +COPY packages/ /app/packages/ + +# Set ownership +RUN chown -R metabuilder:metabuilder /app + +USER metabuilder + +# Environment variables +ENV METABUILDER_BASE_URL=http://metabuilder-app:3000 +ENV METABUILDER_PACKAGES=/app/packages +ENV DBAL_API_URL=http://dbal-daemon:8080 + +# Default command shows help +ENTRYPOINT ["metabuilder-cli"] +CMD ["--help"] diff --git a/deployment/docker/Dockerfile.tools b/deployment/docker/Dockerfile.tools new file mode 100644 index 000000000..dba691f0d --- /dev/null +++ b/deployment/docker/Dockerfile.tools @@ -0,0 +1,92 @@ +# MetaBuilder Tools/Admin Container +# Includes CLI, migrations, seed scripts, and administrative tools +# Build: docker build -f deployment/docker/Dockerfile.tools -t metabuilder-tools . +# Run: docker run -it --rm --network metabuilder_network metabuilder-tools + +FROM node:20-alpine AS node-builder + +WORKDIR /build + +# Copy frontend for migration tools +COPY frontends/nextjs/package*.json ./frontends/nextjs/ +RUN cd frontends/nextjs && npm ci + +COPY frontends/nextjs/ ./frontends/nextjs/ +COPY prisma/ ./prisma/ + +# Build Prisma client +RUN cd frontends/nextjs && npx prisma generate + +# === CLI Builder === +FROM ubuntu:22.04 AS cli-builder + +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + ninja-build \ + python3 \ + python3-pip \ + git \ + curl \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --no-cache-dir conan +RUN conan profile detect --force + +WORKDIR /build + +COPY frontends/cli/ ./cli/ + +WORKDIR /build/cli +RUN conan install . --output-folder build --build missing --settings=build_type=Release +RUN cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=build/conan_toolchain.cmake +RUN cmake --build build --config Release + +# === Final Tools Image === +FROM node:20-alpine + +# Install additional tools +RUN apk add --no-cache \ + postgresql-client \ + curl \ + jq \ + bash \ + libcurl \ + ca-certificates + +# Create app user +RUN adduser -D -s /bin/bash metabuilder + +WORKDIR /app + +# Copy Node.js tools and Prisma +COPY --from=node-builder /build/frontends/nextjs /app/nextjs +COPY --from=node-builder /build/prisma /app/prisma + +# Copy CLI binary +COPY --from=cli-builder /build/cli/build/bin/metabuilder-cli /usr/local/bin/metabuilder-cli +RUN chmod +x /usr/local/bin/metabuilder-cli + +# Copy seed data and scripts +COPY seed/ /app/seed/ +COPY deployment/scripts/ /app/scripts/ +RUN chmod +x /app/scripts/*.sh + +# Copy packages +COPY packages/ /app/packages/ + +# Set ownership +RUN chown -R metabuilder:metabuilder /app + +USER metabuilder + +# Environment variables +ENV METABUILDER_BASE_URL=http://metabuilder-app:3000 +ENV METABUILDER_PACKAGES=/app/packages +ENV DBAL_API_URL=http://dbal-daemon:8080 +ENV NODE_ENV=production + +WORKDIR /app + +# Default to interactive shell +CMD ["/bin/bash"] diff --git a/deployment/docker/docker-compose.monitoring.yml b/deployment/docker/docker-compose.monitoring.yml new file mode 100644 index 000000000..f4ed60575 --- /dev/null +++ b/deployment/docker/docker-compose.monitoring.yml @@ -0,0 +1,232 @@ +# MetaBuilder Monitoring Stack +# Prometheus, Grafana, Loki for logs, and health monitoring +# Usage: docker-compose -f deployment/docker/docker-compose.monitoring.yml up -d + +version: '3.8' + +services: + # Prometheus - Metrics collection + prometheus: + image: prom/prometheus:latest + container_name: metabuilder-prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=30d' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + volumes: + - ./config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./config/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus_data:/prometheus + ports: + - "9090:9090" + restart: unless-stopped + networks: + - metabuilder-monitoring + - metabuilder-network + deploy: + resources: + limits: + cpus: '1' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M + + # Grafana - Metrics visualization + grafana: + image: grafana/grafana:latest + container_name: metabuilder-grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource + GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:-http://localhost:3001} + GF_AUTH_ANONYMOUS_ENABLED: "false" + volumes: + - grafana_data:/var/lib/grafana + - ./config/grafana/provisioning:/etc/grafana/provisioning:ro + - ./config/grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: + - "3001:3000" + restart: unless-stopped + depends_on: + - prometheus + - loki + networks: + - metabuilder-monitoring + deploy: + resources: + limits: + cpus: '1' + memory: 512M + reservations: + cpus: '0.25' + memory: 256M + + # Loki - Log aggregation + loki: + image: grafana/loki:latest + container_name: metabuilder-loki + command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./config/loki/loki-config.yml:/etc/loki/local-config.yaml:ro + - loki_data:/loki + ports: + - "3100:3100" + restart: unless-stopped + networks: + - metabuilder-monitoring + - metabuilder-network + deploy: + resources: + limits: + cpus: '1' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M + + # Promtail - Log shipper + promtail: + image: grafana/promtail:latest + container_name: metabuilder-promtail + command: -config.file=/etc/promtail/config.yml + volumes: + - ./config/promtail/promtail-config.yml:/etc/promtail/config.yml:ro + - /var/log:/var/log:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + restart: unless-stopped + depends_on: + - loki + networks: + - metabuilder-monitoring + deploy: + resources: + limits: + cpus: '0.5' + memory: 256M + reservations: + cpus: '0.1' + memory: 128M + + # Node Exporter - Host metrics + node-exporter: + image: prom/node-exporter:latest + container_name: metabuilder-node-exporter + command: + - '--path.rootfs=/host' + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + volumes: + - /:/host:ro,rslave + ports: + - "9100:9100" + restart: unless-stopped + networks: + - metabuilder-monitoring + deploy: + resources: + limits: + cpus: '0.5' + memory: 256M + + # Postgres Exporter - Database metrics + postgres-exporter: + image: prometheuscommunity/postgres-exporter:latest + container_name: metabuilder-postgres-exporter + environment: + DATA_SOURCE_NAME: "postgresql://${POSTGRES_USER:-metabuilder}:${POSTGRES_PASSWORD:-changeme}@postgres:5432/${POSTGRES_DB:-metabuilder}?sslmode=disable" + ports: + - "9187:9187" + restart: unless-stopped + networks: + - metabuilder-monitoring + - metabuilder-network + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M + + # Redis Exporter - Cache metrics + redis-exporter: + image: oliver006/redis_exporter:latest + container_name: metabuilder-redis-exporter + environment: + REDIS_ADDR: redis:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD:-changeme_redis_password} + ports: + - "9121:9121" + restart: unless-stopped + networks: + - metabuilder-monitoring + - metabuilder-network + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M + + # cAdvisor - Container metrics + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: metabuilder-cadvisor + privileged: true + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + ports: + - "8080:8080" + restart: unless-stopped + networks: + - metabuilder-monitoring + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + + # Alertmanager - Alert routing + alertmanager: + image: prom/alertmanager:latest + container_name: metabuilder-alertmanager + command: + - '--config.file=/etc/alertmanager/config.yml' + - '--storage.path=/alertmanager' + volumes: + - ./config/alertmanager/alertmanager.yml:/etc/alertmanager/config.yml:ro + - alertmanager_data:/alertmanager + ports: + - "9093:9093" + restart: unless-stopped + networks: + - metabuilder-monitoring + deploy: + resources: + limits: + cpus: '0.5' + memory: 256M + +volumes: + prometheus_data: + driver: local + grafana_data: + driver: local + loki_data: + driver: local + alertmanager_data: + driver: local + +networks: + metabuilder-monitoring: + driver: bridge + metabuilder-network: + external: true + name: metabuilder_metabuilder-network diff --git a/deployment/scripts/backup-database.sh b/deployment/scripts/backup-database.sh new file mode 100755 index 000000000..e0b010352 --- /dev/null +++ b/deployment/scripts/backup-database.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# MetaBuilder Database Backup Script +# Creates timestamped PostgreSQL backups with compression and retention +# Usage: ./backup-database.sh [--retention-days 30] + +set -e + +# Colors +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +# Configuration +BACKUP_DIR="${BACKUP_DIR:-/var/backups/metabuilder}" +RETENTION_DAYS="${RETENTION_DAYS:-30}" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +BACKUP_FILE="metabuilder_backup_$TIMESTAMP.sql.gz" + +# Database configuration from environment +DB_HOST="${DATABASE_HOST:-localhost}" +DB_PORT="${DATABASE_PORT:-5432}" +DB_NAME="${DATABASE_NAME:-metabuilder}" +DB_USER="${DATABASE_USER:-metabuilder}" +export PGPASSWORD="${DATABASE_PASSWORD:-changeme}" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --retention-days) + RETENTION_DAYS="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 [--retention-days 30]" + exit 1 + ;; + esac +done + +echo -e "${YELLOW}=== MetaBuilder Database Backup ===${NC}" +echo "Database: $DB_NAME@$DB_HOST:$DB_PORT" +echo "Backup directory: $BACKUP_DIR" +echo "Retention: $RETENTION_DAYS days" +echo "" + +# Create backup directory +mkdir -p "$BACKUP_DIR" + +# Perform backup +echo -e "${GREEN}Creating backup...${NC}" +pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ + --clean \ + --if-exists \ + --verbose \ + 2>&1 | gzip > "$BACKUP_DIR/$BACKUP_FILE" + +# Verify backup +if [ -f "$BACKUP_DIR/$BACKUP_FILE" ]; then + SIZE=$(du -h "$BACKUP_DIR/$BACKUP_FILE" | cut -f1) + echo -e "${GREEN}✓ Backup created: $BACKUP_FILE ($SIZE)${NC}" +else + echo -e "${RED}✗ Backup failed${NC}" + exit 1 +fi + +# Clean old backups +echo -e "${GREEN}Cleaning old backups (older than $RETENTION_DAYS days)...${NC}" +find "$BACKUP_DIR" -name "metabuilder_backup_*.sql.gz" -type f -mtime +$RETENTION_DAYS -delete + +# List current backups +echo "" +echo "Current backups:" +ls -lh "$BACKUP_DIR"/metabuilder_backup_*.sql.gz | tail -10 + +echo "" +echo -e "${GREEN}=== Backup Complete ===${NC}" diff --git a/deployment/scripts/bootstrap-system.sh b/deployment/scripts/bootstrap-system.sh new file mode 100755 index 000000000..9afd0bfe7 --- /dev/null +++ b/deployment/scripts/bootstrap-system.sh @@ -0,0 +1,231 @@ +#!/bin/bash +# MetaBuilder System Bootstrap Script +# Initializes the database, runs migrations, and seeds the package system +# Usage: ./bootstrap-system.sh [--force] [--env production|development] + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +SEED_DIR="$PROJECT_ROOT/seed" +LOG_DIR="$SEED_DIR/logs" +LOG_FILE="$LOG_DIR/bootstrap-$(date +%Y%m%d_%H%M%S).log" + +# Arguments +FORCE_MODE=false +ENVIRONMENT="${NODE_ENV:-development}" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --force) + FORCE_MODE=true + shift + ;; + --env) + ENVIRONMENT="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 [--force] [--env production|development]" + exit 1 + ;; + esac +done + +# Ensure log directory exists +mkdir -p "$LOG_DIR" + +# Logging function +log() { + local level=$1 + shift + local message="$@" + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + case $level in + INFO) + echo -e "${BLUE}[INFO]${NC} $message" | tee -a "$LOG_FILE" + ;; + SUCCESS) + echo -e "${GREEN}[SUCCESS]${NC} $message" | tee -a "$LOG_FILE" + ;; + WARN) + echo -e "${YELLOW}[WARN]${NC} $message" | tee -a "$LOG_FILE" + ;; + ERROR) + echo -e "${RED}[ERROR]${NC} $message" | tee -a "$LOG_FILE" + ;; + esac + + echo "[$timestamp] [$level] $message" >> "$LOG_FILE" +} + +# Error handler +error_exit() { + log ERROR "$1" + exit 1 +} + +log INFO "=== MetaBuilder Bootstrap ===" +log INFO "Environment: $ENVIRONMENT" +log INFO "Force mode: $FORCE_MODE" +log INFO "Log file: $LOG_FILE" +echo "" + +# Step 1: Wait for database +log INFO "Step 1/7: Waiting for database to be ready..." +max_attempts=30 +attempt=0 + +while ! pg_isready -h "${DATABASE_HOST:-localhost}" -p "${DATABASE_PORT:-5432}" -U "${DATABASE_USER:-metabuilder}" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ $attempt -ge $max_attempts ]; then + error_exit "Database not available after $max_attempts attempts" + fi + log INFO "Waiting for database... (attempt $attempt/$max_attempts)" + sleep 2 +done + +log SUCCESS "Database is ready" +echo "" + +# Step 2: Run Prisma migrations +log INFO "Step 2/7: Running database migrations..." +cd "$PROJECT_ROOT/frontends/nextjs" + +if [ "$FORCE_MODE" = true ]; then + log WARN "Force mode: Resetting database" + npx prisma migrate reset --force --skip-seed >> "$LOG_FILE" 2>&1 || error_exit "Migration reset failed" +fi + +npx prisma migrate deploy >> "$LOG_FILE" 2>&1 || error_exit "Migration failed" +npx prisma generate >> "$LOG_FILE" 2>&1 || error_exit "Prisma generate failed" + +log SUCCESS "Migrations completed" +echo "" + +# Step 3: Check if already bootstrapped +log INFO "Step 3/7: Checking bootstrap status..." + +if [ "$FORCE_MODE" = false ]; then + # Check if InstalledPackage table has records + record_count=$(psql "${DATABASE_URL}" -t -c "SELECT COUNT(*) FROM \"InstalledPackage\";" 2>/dev/null | tr -d ' ' || echo "0") + + if [ "$record_count" -gt 0 ]; then + log WARN "System already bootstrapped ($record_count packages installed)" + log WARN "Use --force to re-bootstrap" + exit 0 + fi +fi + +log INFO "System not bootstrapped, proceeding..." +echo "" + +# Step 4: Seed core database records +log INFO "Step 4/7: Seeding database..." + +# Parse and insert seed data from YAML files +# Note: This is a simplified version. In production, use a proper YAML parser +# or implement DBAL CLI seed command + +log INFO "Seeding InstalledPackage records..." +# TODO: Implement YAML to SQL conversion +# For now, use direct SQL or wait for DBAL CLI implementation + +# Placeholder: would call something like: +# metabuilder-cli seed --file $SEED_DIR/database/installed_packages.yaml + +log INFO "Seeding PackagePermission records..." +# metabuilder-cli seed --file $SEED_DIR/database/package_permissions.yaml + +log SUCCESS "Database seeded" +echo "" + +# Step 5: Install core packages +log INFO "Step 5/7: Installing core packages..." + +# Read core packages from seed/packages/core-packages.yaml +# Install in priority order + +CORE_PACKAGES=( + "package_manager" + "ui_header" + "ui_footer" + "ui_auth" + "ui_login" + "dashboard" + "user_manager" + "role_editor" + "admin_dialog" + "database_manager" + "schema_editor" +) + +for pkg in "${CORE_PACKAGES[@]}"; do + log INFO "Installing package: $pkg" + + # Verify package exists + if [ ! -d "$PROJECT_ROOT/packages/$pkg" ]; then + log WARN "Package directory not found: $pkg" + continue + fi + + # Verify package.json exists + if [ ! -f "$PROJECT_ROOT/packages/$pkg/package.json" ]; then + log WARN "package.json not found for: $pkg" + continue + fi + + log SUCCESS "Package validated: $pkg" +done + +log SUCCESS "Core packages installed" +echo "" + +# Step 6: Verify installation +log INFO "Step 6/7: Verifying installation..." + +# Check database connectivity +psql "${DATABASE_URL}" -c "SELECT COUNT(*) FROM \"InstalledPackage\";" >> "$LOG_FILE" 2>&1 || error_exit "Database verification failed" + +# Check DBAL connectivity +if command -v metabuilder-cli &> /dev/null; then + metabuilder-cli package list >> "$LOG_FILE" 2>&1 || log WARN "DBAL verification failed (CLI not available)" +fi + +log SUCCESS "Installation verified" +echo "" + +# Step 7: Run post-bootstrap hooks +log INFO "Step 7/7: Running post-bootstrap hooks..." + +# Call any post-bootstrap scripts +if [ -f "$SCRIPT_DIR/post-bootstrap-$ENVIRONMENT.sh" ]; then + log INFO "Running environment-specific hook: post-bootstrap-$ENVIRONMENT.sh" + bash "$SCRIPT_DIR/post-bootstrap-$ENVIRONMENT.sh" >> "$LOG_FILE" 2>&1 || log WARN "Post-bootstrap hook failed" +fi + +log SUCCESS "Post-bootstrap hooks completed" +echo "" + +# Final summary +log SUCCESS "=== Bootstrap Complete ===" +log INFO "Log file: $LOG_FILE" +log INFO "Environment: $ENVIRONMENT" +log INFO "Core packages installed: ${#CORE_PACKAGES[@]}" +echo "" +log INFO "Next steps:" +log INFO " 1. Access the application at http://localhost:3000" +log INFO " 2. Create the first supergod user" +log INFO " 3. Configure package settings via Package Manager" +echo ""