diff --git a/AGENTS.md b/AGENTS.md index 09113a967..0df4057e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,7 +50,7 @@ POST /pastebin/pastebin/User | `dbal/production/src/daemon/server_routes.cpp` | Route registration + auto-seed startup | | `frontends/pastebin/backend/app.py` | Flask JWT auth + Python runner | | `frontends/pastebin/src/` | Next.js React app | -| `deployment/docker-compose.stack.yml` | Full stack compose | +| `deployment/compose.yml` | Full stack compose | | `deployment/deployment.py` | Python CLI for all build/deploy/stack commands | --- @@ -80,15 +80,15 @@ cd deployment # Full rebuild + restart python3 deployment.py build apps --force dbal pastebin -docker compose -f docker-compose.stack.yml up -d +docker compose -f compose.yml up -d # Flask backend (separate from Next.js) -docker compose -f docker-compose.stack.yml build pastebin-backend -docker compose -f docker-compose.stack.yml up -d pastebin-backend +docker compose -f compose.yml build pastebin-backend +docker compose -f compose.yml up -d pastebin-backend # dbal-init volume (schema volume container — rebuild when entity JSON changes) -docker compose -f docker-compose.stack.yml build dbal-init -docker compose -f docker-compose.stack.yml up dbal-init +docker compose -f compose.yml build dbal-init +docker compose -f compose.yml up dbal-init ``` --- diff --git a/CLAUDE.md b/CLAUDE.md index 40bd35e30..561ef4aa1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -230,11 +230,11 @@ npm run build --workspaces cd deployment && python3 deployment.py build base # Build Docker base images # Deploy full stack -cd deployment && docker compose -f docker-compose.stack.yml up -d +cd deployment && docker compose -f compose.yml up -d # Build & deploy specific apps python3 deployment.py build apps --force dbal pastebin # Next.js frontend only -docker compose -f docker-compose.stack.yml build pastebin-backend # Flask backend +docker compose -f compose.yml build pastebin-backend # Flask backend # DBAL logs / seed verification docker logs -f metabuilder-dbal diff --git a/README.md b/README.md index e84dd1676..0aaf4b364 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ http://localhost:8080 # DBAL C++ REST API (entities) ```bash # Deploy full stack cd deployment -docker compose -f docker-compose.stack.yml up -d +docker compose -f compose.yml up -d # Build & deploy a specific app python3 deployment.py build apps --force dbal pastebin diff --git a/deployment/README.md b/deployment/README.md index f4913bab7..fda55d550 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -105,7 +105,7 @@ deployment.py artifactory init | File | Purpose | |------|---------| | `docker-compose.nexus.yml` | Local registries (Nexus + Artifactory) | -| `docker-compose.stack.yml` | Full application stack | +| `compose.yml` | Full application stack | | `docker-compose.test.yml` | Integration test services | | `docker-compose.smoke.yml` | Smoke test environment | diff --git a/deployment/cli/build_apps.py b/deployment/cli/build_apps.py index dc3f769e9..d603aef0e 100644 --- a/deployment/cli/build_apps.py +++ b/deployment/cli/build_apps.py @@ -4,7 +4,7 @@ import argparse import time from cli.helpers import ( BASE_DIR, PROJECT_ROOT, GREEN, YELLOW, NC, - docker_compose, docker_image_exists, log_err, log_info, log_ok, log_warn, + docker_compose, docker_image_exists, get_buildable_services, log_err, log_info, log_ok, log_warn, pull_with_retry, resolve_services, run as run_proc, ) @@ -37,8 +37,8 @@ def run_cmd(args: argparse.Namespace, config: dict) -> int: print(f" - {img}") print(f"{YELLOW}Build with:{NC} python3 deployment.py build base\n") - all_apps = defs["all_apps"] - targets = args.apps if args.apps else list(all_apps) + buildable = get_buildable_services() + targets = args.apps if args.apps else buildable services = resolve_services(targets, config) if services is None: return 1 diff --git a/deployment/cli/commands.json b/deployment/cli/commands.json index a5f3bd136..d901ad6aa 100644 --- a/deployment/cli/commands.json +++ b/deployment/cli/commands.json @@ -9,22 +9,9 @@ }, "definitions": { - "all_apps": ["workflowui", "codegen", "pastebin", "postgres", "emailclient", "exploded-diagrams", "storybook", "frontend-app", "dbal"], "base_build_order": ["apt", "conan-deps", "android-sdk", "node-deps", "pip-deps", "devcontainer"], "stack_commands": ["up", "start", "down", "stop", "build", "restart", "logs", "ps", "status", "clean"], - "service_map": { - "workflowui": "workflowui", - "codegen": "codegen", - "pastebin": "pastebin", - "postgres": "postgres-dashboard", - "emailclient": "emailclient-app", - "exploded-diagrams": "exploded-diagrams", - "storybook": "storybook", - "frontend-app": "frontend-app", - "dbal": "dbal" - }, - "base_images": { "apt": { "dockerfile": "Dockerfile.apt", "tag": "metabuilder/base-apt:latest" }, "conan-deps": { "dockerfile": "Dockerfile.conan-deps", "tag": "metabuilder/base-conan-deps:latest" }, diff --git a/deployment/cli/deploy.py b/deployment/cli/deploy.py index 9369e226c..8627e3ed6 100644 --- a/deployment/cli/deploy.py +++ b/deployment/cli/deploy.py @@ -6,16 +6,16 @@ import sys import time from cli.helpers import ( COMPOSE_FILE, GREEN, RED, YELLOW, BLUE, NC, - docker_compose, log_err, log_warn, resolve_services, run as run_proc, + docker_compose, get_buildable_services, log_err, log_warn, resolve_services, run as run_proc, ) def run_cmd(args: argparse.Namespace, config: dict) -> int: - all_apps = config["definitions"]["all_apps"] - targets = list(all_apps) if args.all else args.apps + buildable = get_buildable_services() + targets = buildable if args.all else args.apps if not targets: log_err("Specify app(s) to deploy, or use --all") - print(f"Available: {', '.join(all_apps)}") + print(f"Available: {', '.join(buildable)}") return 1 services = resolve_services(targets, config) diff --git a/deployment/cli/helpers.py b/deployment/cli/helpers.py index fc0f9ae00..d55956189 100644 --- a/deployment/cli/helpers.py +++ b/deployment/cli/helpers.py @@ -13,7 +13,7 @@ from pathlib import Path SCRIPT_DIR = Path(__file__).resolve().parent.parent # deployment/ PROJECT_ROOT = SCRIPT_DIR.parent BASE_DIR = SCRIPT_DIR / "base-images" -COMPOSE_FILE = SCRIPT_DIR / "docker-compose.stack.yml" +COMPOSE_FILE = SCRIPT_DIR / "compose.yml" # ── Colors ─────────────────────────────────────────────────────────────────── @@ -121,17 +121,27 @@ def build_with_retry(tag: str, dockerfile: str, context: str, max_attempts: int return False +def get_buildable_services() -> list[str]: + """Return all service names that have a build: section in the compose file.""" + import yaml + with open(COMPOSE_FILE) as f: + compose = yaml.safe_load(f) + return [ + name for name, svc in compose.get("services", {}).items() + if isinstance(svc, dict) and "build" in svc + ] + + def resolve_services(targets: list[str], config: dict) -> list[str] | None: - """Map friendly app names to compose service names. Returns None on error.""" - svc_map = config["definitions"]["service_map"] + """Validate compose service names against the compose file. Returns None on error.""" + buildable = get_buildable_services() services = [] for t in targets: - svc = svc_map.get(t) - if not svc: - log_err(f"Unknown app: {t}") - print(f"Available: {', '.join(config['definitions']['all_apps'])}") + if t not in buildable: + log_err(f"Unknown or non-buildable service: {t}") + print(f"Available: {', '.join(buildable)}") return None - services.append(svc) + services.append(t) return services diff --git a/deployment/cli/nexus_populate.py b/deployment/cli/nexus_populate.py index 67d567e03..fa832840c 100644 --- a/deployment/cli/nexus_populate.py +++ b/deployment/cli/nexus_populate.py @@ -1,11 +1,31 @@ """Push all locally-built images to Nexus with :main + :latest tags.""" import argparse +import pathlib +import yaml + from cli.helpers import ( BLUE, GREEN, NC, docker_image_exists, log_err, log_info, log_ok, log_warn, run as run_proc, ) +COMPOSE_FILE = pathlib.Path(__file__).parent.parent / "compose.yml" + + +def _load_built_images() -> list[dict]: + """Return [{local, name}] for every service with a build: directive.""" + with open(COMPOSE_FILE) as f: + dc = yaml.safe_load(f) + images = [] + for svc_name, svc in dc.get("services", {}).items(): + if "build" not in svc: + continue + local = svc.get("image", f"deployment-{svc_name}:latest") + # derive a short push name from the image tag (strip prefix/tag) + name = local.split("/")[-1].split(":")[0].removeprefix("deployment-") + images.append({"local": local, "name": name}) + return images + def run_cmd(args: argparse.Namespace, config: dict) -> int: nexus = "localhost:5050" @@ -16,23 +36,23 @@ def run_cmd(args: argparse.Namespace, config: dict) -> int: run_proc(["docker", "login", nexus, "-u", nexus_user, "--password-stdin"], input=nexus_pass.encode()) - images_def = config["definitions"]["nexus_images"] + images = _load_built_images() pushed = skipped = failed = 0 - def push_image(src: str, name: str, size: str) -> None: + def push_image(local: str, name: str) -> None: nonlocal pushed, skipped, failed - if not docker_image_exists(src): - log_warn(f"SKIP {name} — {src} not found locally") + if not docker_image_exists(local): + log_warn(f"SKIP {name} — {local} not found locally") skipped += 1 return dst_main = f"{nexus}/{slug}/{name}:main" dst_latest = f"{nexus}/{slug}/{name}:latest" - log_info(f"Pushing {name} ({size})...") - run_proc(["docker", "tag", src, dst_main]) - run_proc(["docker", "tag", src, dst_latest]) + log_info(f"Pushing {name}...") + run_proc(["docker", "tag", local, dst_main]) + run_proc(["docker", "tag", local, dst_latest]) r1 = run_proc(["docker", "push", dst_main]) r2 = run_proc(["docker", "push", dst_latest]) @@ -43,21 +63,12 @@ def run_cmd(args: argparse.Namespace, config: dict) -> int: log_err(f" {name} FAILED") failed += 1 - print(f"\n{BLUE}Registry : {nexus}{NC}") - print(f"{BLUE}Slug : {slug}{NC}") - print(f"{BLUE}Skip heavy: {args.skip_heavy}{NC}\n") + print(f"\n{BLUE}Registry : {nexus}{NC}") + print(f"{BLUE}Slug : {slug}{NC}") + print(f"{BLUE}Images : {len(images)} (parsed from compose.yml){NC}\n") - for entry in images_def["base"] + images_def["apps"]: - push_image(entry["local"], entry["name"], entry["size"]) - - if args.skip_heavy: - log_warn("Skipping heavy images (--skip-heavy set):") - for entry in images_def["heavy"] + images_def["heavy_apps"]: - log_warn(f" {entry['name']} ({entry['size']})") - else: - log_info("--- Heavy images (this will take a while) ---") - for entry in images_def["heavy_apps"] + images_def["heavy"]: - push_image(entry["local"], entry["name"], entry["size"]) + for entry in images: + push_image(entry["local"], entry["name"]) print(f"\n{GREEN}{'=' * 46}{NC}") print(f"{GREEN} Done. pushed={pushed} skipped={skipped} failed={failed}{NC}") diff --git a/deployment/cli/nexus_push.py b/deployment/cli/nexus_push.py index 8471f32e1..41c35f320 100644 --- a/deployment/cli/nexus_push.py +++ b/deployment/cli/nexus_push.py @@ -11,7 +11,7 @@ from cli.helpers import ( docker_image_exists, log_info, run as run_proc, ) -COMPOSE_FILE = pathlib.Path(__file__).parent.parent / "docker-compose.stack.yml" +COMPOSE_FILE = pathlib.Path(__file__).parent.parent / "compose.yml" BASE_IMAGES_DIR = pathlib.Path(__file__).parent.parent / "base-images" # Dockerfile.apt -> base-apt, Dockerfile.node-deps -> base-node-deps, etc. @@ -70,7 +70,7 @@ def run_cmd(args: argparse.Namespace, config: dict) -> int: print(f"{YELLOW}Registry:{NC} {local_registry}") print(f"{YELLOW}Slug:{NC} {slug}") print(f"{YELLOW}Tag:{NC} {tag}") - print(f"{YELLOW}Images:{NC} {len(images)} (base-images/ + docker-compose.stack.yml)\n") + print(f"{YELLOW}Images:{NC} {len(images)} (base-images/ + compose.yml)\n") log_info(f"Logging in to {local_registry}...") run_proc(["docker", "login", local_registry, "-u", nexus_user, "--password-stdin"], diff --git a/deployment/docker-compose.stack.yml b/deployment/compose.yml similarity index 92% rename from deployment/docker-compose.stack.yml rename to deployment/compose.yml index dd6042e55..c4fb1dba6 100644 --- a/deployment/docker-compose.stack.yml +++ b/deployment/compose.yml @@ -15,10 +15,10 @@ # --profile media Media daemon (FFmpeg/radio/retro), native HTTP streaming, HLS # # Usage: -# docker compose -f docker-compose.stack.yml up -d -# docker compose -f docker-compose.stack.yml --profile monitoring up -d -# docker compose -f docker-compose.stack.yml --profile media up -d -# docker compose -f docker-compose.stack.yml --profile monitoring --profile media up -d +# docker compose -f compose.yml up -d +# docker compose -f compose.yml --profile monitoring up -d +# docker compose -f compose.yml --profile media up -d +# docker compose -f compose.yml --profile monitoring --profile media up -d # # Access: # http://localhost Welcome portal @@ -918,6 +918,7 @@ services: build: context: .. dockerfile: deployment/config/prometheus/Dockerfile + image: deployment-prometheus:latest container_name: metabuilder-prometheus restart: unless-stopped profiles: [monitoring] @@ -949,6 +950,7 @@ services: build: context: .. dockerfile: deployment/config/grafana/Dockerfile + image: deployment-grafana:latest container_name: metabuilder-grafana restart: unless-stopped profiles: [monitoring] @@ -984,6 +986,7 @@ services: build: context: .. dockerfile: deployment/config/loki/Dockerfile + image: deployment-loki:latest container_name: metabuilder-loki restart: unless-stopped profiles: [monitoring] @@ -1008,6 +1011,7 @@ services: build: context: .. dockerfile: deployment/config/promtail/Dockerfile + image: deployment-promtail:latest container_name: metabuilder-promtail restart: unless-stopped profiles: [monitoring] @@ -1165,6 +1169,7 @@ services: build: context: .. dockerfile: deployment/config/alertmanager/Dockerfile + image: deployment-alertmanager:latest container_name: metabuilder-alertmanager restart: unless-stopped profiles: [monitoring] @@ -1197,6 +1202,7 @@ services: build: context: ../services/media_daemon dockerfile: Dockerfile + image: deployment-media-daemon:latest container_name: metabuilder-media-daemon restart: unless-stopped profiles: [media] @@ -1262,6 +1268,7 @@ services: build: context: .. dockerfile: deployment/config/nginx/Dockerfile.stream + image: deployment-nginx-stream:latest container_name: metabuilder-nginx-stream restart: unless-stopped profiles: [media] @@ -1280,6 +1287,101 @@ services: networks: - metabuilder +# ============================================================================ +# Package Registries (--profile registry) — Nexus + Artifactory +# +# Run with: +# docker compose -f compose.yml --profile registry up -d +# # Wait ~2 min for init containers, then: +# python3 deployment.py nexus push +# python3 deployment.py npm publish-patches +# +# Access: +# Nexus: http://localhost:8091 (admin / nexus) +# Artifactory: http://localhost:8092 (admin / password) +# ============================================================================ + + nexus: + image: sonatype/nexus3:3.75.0 + platform: linux/amd64 + profiles: [registry] + container_name: nexus-registry + restart: unless-stopped + ports: + - "8091:8081" + - "5050:5050" + volumes: + - nexus-data:/nexus-data + environment: + INSTALL4J_ADD_VM_PARAMS: "-Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8081/service/rest/v1/status || exit 1"] + interval: 30s + timeout: 10s + retries: 15 + start_period: 120s + networks: + - metabuilder + + nexus-init: + image: python:3.12-alpine + profiles: [registry] + container_name: nexus-init + depends_on: + nexus: + condition: service_healthy + volumes: + - nexus-data:/nexus-data:ro + - ./nexus-init.py:/nexus-init.py:ro + entrypoint: ["/bin/sh", "-c", "pip install -q requests && python3 /nexus-init.py"] + environment: + NEXUS_URL: "http://nexus:8081" + NEXUS_ADMIN_NEW_PASS: "nexus" + DOCKER_REPO_PORT: "5050" + restart: "no" + networks: + - metabuilder + + artifactory: + image: releases-docker.jfrog.io/jfrog/artifactory-cpp-ce:latest + profiles: [registry] + container_name: artifactory-ce + restart: unless-stopped + ports: + - "8092:8081" + - "8093:8082" + volumes: + - artifactory-data:/var/opt/jfrog/artifactory + environment: + JF_SHARED_DATABASE_TYPE: derby + JF_SHARED_DATABASE_ALLOWNONPOSTGRESQL: "true" + EXTRA_JAVA_OPTIONS: "-Xms512m -Xmx2g" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8082/router/api/v1/system/health || exit 1"] + interval: 30s + timeout: 10s + retries: 15 + start_period: 120s + networks: + - metabuilder + + artifactory-init: + image: python:3.12-alpine + profiles: [registry] + container_name: artifactory-init + depends_on: + artifactory: + condition: service_healthy + volumes: + - ./artifactory-init.py:/artifactory-init.py:ro + entrypoint: ["/bin/sh", "-c", "pip install -q requests && python3 /artifactory-init.py"] + environment: + ARTIFACTORY_URL: "http://artifactory:8081" + ARTIFACTORY_ADMIN_PASS: "password" + restart: "no" + networks: + - metabuilder + # ============================================================================ # Volumes # ============================================================================ @@ -1342,6 +1444,11 @@ volumes: driver: local packagerepo-data: driver: local + # Registry + nexus-data: + name: nexus-data + artifactory-data: + name: artifactory-data # ============================================================================ # Networks diff --git a/deployment/config/dbal/config.yaml b/deployment/config/dbal/config.yaml index 66d1688e3..7834b4d75 100644 --- a/deployment/config/dbal/config.yaml +++ b/deployment/config/dbal/config.yaml @@ -2,7 +2,7 @@ # # Primary adapter is configured here. Additional backends (cache, search, # secondary databases) are configured via environment variables in -# docker-compose.stack.yml: +# compose.yml: # # DBAL_ADAPTER Primary adapter type (postgres, mysql, sqlite, etc.) # DATABASE_URL Primary database connection string diff --git a/deployment/config/nginx-smoke/default.conf b/deployment/config/nginx-smoke/default.conf deleted file mode 100644 index 3bf12ca13..000000000 --- a/deployment/config/nginx-smoke/default.conf +++ /dev/null @@ -1,58 +0,0 @@ -## -## nginx-smoke — Gateway for deployment smoke tests in CI. -## -## Real apps (workflowui, pastebin) are proxied to playwright's webServer -## processes running on the host (reached via host.docker.internal). -## Remaining apps (codegen, emailclient, etc.) return stub 200 responses -## since they are not started as dev servers in CI. -## - -server { - listen 80; - server_name localhost; - - # ── Real apps — proxied to playwright webServer processes on host ───── - - location /workflowui { - proxy_pass http://host.docker.internal:3000; - proxy_set_header Host localhost; - proxy_set_header X-Real-IP $remote_addr; - proxy_read_timeout 120s; - } - - location /pastebin { - proxy_pass http://host.docker.internal:3001; - proxy_set_header Host localhost; - proxy_set_header X-Real-IP $remote_addr; - proxy_read_timeout 120s; - } - - # ── DBAL API — proxied to real C++ daemon ───────────────────────────── - - location /api/ { - proxy_pass http://dbal:8080/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_read_timeout 30s; - } - - # ── Portal — must contain "MetaBuilder" ─────────────────────────────── - - location = / { - add_header Content-Type text/html; - return 200 '

MetaBuilder Portal

'; - } - - # ── Postgres redirect ───────────────────────────────────────────────── - - location = /postgres { - return 307 /postgres/dashboard; - } - - # ── Remaining apps — stub (codegen, emailclient, diagrams, etc.) ────── - - location / { - add_header Content-Type text/html; - return 200 'MetaBuilder App'; - } -} diff --git a/deployment/docker-compose.nexus.yml b/deployment/docker-compose.nexus.yml deleted file mode 100644 index aa5e27247..000000000 --- a/deployment/docker-compose.nexus.yml +++ /dev/null @@ -1,124 +0,0 @@ -# Local Package Registries — Nexus (Docker + npm) + Artifactory CE (Conan2) -# -# Nexus (Sonatype): -# 8091 → Nexus web UI (admin / nexus) -# 5050 → Docker hosted repository (push + pull) -# Repos: Docker (local), npm (hosted + proxy + group) -# -# Artifactory CE (JFrog): -# 8092 → Artifactory web UI (admin / password) -# Repos: Conan2 (local + remote), generic -# -# Prerequisites — Docker Desktop must trust the insecure local registry: -# Docker Desktop → Settings → Docker Engine → add to "insecure-registries": -# "insecure-registries": ["localhost:5050"] -# Then restart Docker Desktop. -# -# Note: Nexus ships amd64-only. On Apple Silicon it runs via Rosetta (works fine). -# -# Usage: -# docker compose -f docker-compose.nexus.yml up -d -# # Wait ~2 min for init containers to finish, then: -# python3 deployment.py nexus push # Docker images → Nexus -# python3 deployment.py npm publish-patches # Patched npm packages → Nexus -# conan remote add artifactory http://localhost:8092/artifactory/api/conan/conan-local -# -# URLs: -# Nexus: http://localhost:8091 (admin / nexus) -# Artifactory: http://localhost:8092 (admin / password) -# npm group: http://localhost:8091/repository/npm-group/ -# Conan2: http://localhost:8092/artifactory/api/conan/conan-local - -services: - # ============================================================================ - # Sonatype Nexus — Docker registry + npm packages - # ============================================================================ - nexus: - image: sonatype/nexus3:3.75.0 - platform: linux/amd64 # Nexus has no ARM64 image; runs via Rosetta on Apple Silicon - container_name: nexus-registry - restart: unless-stopped - ports: - - "8091:8081" # Web UI + REST API - - "5050:5050" # Docker hosted repository - volumes: - - nexus-data:/nexus-data - environment: - INSTALL4J_ADD_VM_PARAMS: "-Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g" - healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8081/service/rest/v1/status || exit 1"] - interval: 30s - timeout: 10s - retries: 15 - start_period: 120s - networks: - - registry-net - - nexus-init: - image: python:3.12-alpine - container_name: nexus-init - depends_on: - nexus: - condition: service_healthy - volumes: - - nexus-data:/nexus-data:ro - - ./nexus-init.py:/nexus-init.py:ro - entrypoint: ["/bin/sh", "-c", "pip install -q requests && python3 /nexus-init.py"] - environment: - NEXUS_URL: "http://nexus:8081" - NEXUS_ADMIN_NEW_PASS: "nexus" - DOCKER_REPO_PORT: "5050" - restart: "no" - networks: - - registry-net - - # ============================================================================ - # JFrog Artifactory CE — Conan2 package repository - # ============================================================================ - artifactory: - image: releases-docker.jfrog.io/jfrog/artifactory-cpp-ce:latest - container_name: artifactory-ce - restart: unless-stopped - ports: - - "8092:8081" # Web UI + REST API - - "8093:8082" # Service port (router/health) - volumes: - - artifactory-data:/var/opt/jfrog/artifactory - environment: - JF_SHARED_DATABASE_TYPE: derby - JF_SHARED_DATABASE_ALLOWNONPOSTGRESQL: "true" - EXTRA_JAVA_OPTIONS: "-Xms512m -Xmx2g" - healthcheck: - test: ["CMD-SHELL", "curl -sf http://localhost:8082/router/api/v1/system/health || exit 1"] - interval: 30s - timeout: 10s - retries: 15 - start_period: 120s - networks: - - registry-net - - artifactory-init: - image: python:3.12-alpine - container_name: artifactory-init - depends_on: - artifactory: - condition: service_healthy - volumes: - - ./artifactory-init.py:/artifactory-init.py:ro - entrypoint: ["/bin/sh", "-c", "pip install -q requests && python3 /artifactory-init.py"] - environment: - ARTIFACTORY_URL: "http://artifactory:8081" - ARTIFACTORY_ADMIN_PASS: "password" - restart: "no" - networks: - - registry-net - -volumes: - nexus-data: - name: nexus-data - artifactory-data: - name: artifactory-data - -networks: - registry-net: - name: registry-net diff --git a/deployment/docker-compose.smoke.yml b/deployment/docker-compose.smoke.yml deleted file mode 100644 index 4b3bfe058..000000000 --- a/deployment/docker-compose.smoke.yml +++ /dev/null @@ -1,221 +0,0 @@ -# docker-compose.smoke.yml — Smoke test stack for CI. -# -# Includes a real DBAL daemon backed by PostgreSQL so E2E tests can seed -# and query data. Admin tools (phpMyAdmin, Mongo Express, RedisInsight) -# and an nginx gateway round out the stack. -# -# Usage: -# docker compose -f deployment/docker-compose.smoke.yml up -d --wait -# PLAYWRIGHT_BASE_URL=http://localhost/workflowui/ npx playwright test e2e/deployment-smoke.spec.ts -# docker compose -f deployment/docker-compose.smoke.yml down -v - -services: - - # ── Gateway stub ────────────────────────────────────────────────────────── - nginx: - image: nginx:alpine - ports: - - "80:80" - volumes: - - ./config/nginx-smoke/default.conf:/etc/nginx/conf.d/default.conf:ro - # host.docker.internal lets nginx proxy to playwright's webServer processes - # running on the host (workflowui :3000, pastebin :3001). - # On Linux (GitHub Actions) this requires the host-gateway extra_host. - extra_hosts: - - "host.docker.internal:host-gateway" - depends_on: - dbal: - condition: service_healthy - healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://127.0.0.1/"] - interval: 5s - timeout: 3s - retries: 10 - networks: - - smoke - - # ── DBAL + PostgreSQL ──────────────────────────────────────────────────── - postgres: - image: postgres:15-alpine - environment: - POSTGRES_USER: metabuilder - POSTGRES_PASSWORD: metabuilder - POSTGRES_DB: metabuilder - tmpfs: - - /var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U metabuilder"] - interval: 5s - timeout: 3s - retries: 10 - networks: - - smoke - - dbal-init: - image: ${DBAL_INIT_IMAGE:-ghcr.io/johndoe6345789/metabuilder/dbal-init:latest} - build: - context: .. - dockerfile: deployment/config/dbal/Dockerfile.init - volumes: - - dbal-schemas:/target/schemas/entities - - dbal-templates:/target/templates/sql - networks: - - smoke - - dbal: - image: ${DBAL_IMAGE:-ghcr.io/johndoe6345789/metabuilder/dbal:latest} - build: - context: ../dbal - dockerfile: production/build-config/Dockerfile - ports: - - "8080:8080" - environment: - DBAL_ADAPTER: postgres - DATABASE_URL: "postgresql://metabuilder:metabuilder@postgres:5432/metabuilder" - DBAL_SCHEMA_DIR: /app/schemas/entities - DBAL_TEMPLATE_DIR: /app/templates/sql - DBAL_SEED_DIR: /app/seeds/database - DBAL_SEED_ON_STARTUP: "true" - DBAL_BIND_ADDRESS: 0.0.0.0 - DBAL_PORT: 8080 - DBAL_MODE: production - DBAL_DAEMON: "true" - DBAL_LOG_LEVEL: info - DBAL_AUTO_CREATE_TABLES: "true" - DBAL_ENABLE_HEALTH_CHECK: "true" - DBAL_ADMIN_TOKEN: "smoke-test-admin-token" - DBAL_CORS_ORIGIN: "*" - JWT_SECRET_KEY: "test-secret" - volumes: - - dbal-schemas:/app/schemas/entities:ro - - dbal-templates:/app/templates/sql:ro - depends_on: - dbal-init: - condition: service_completed_successfully - postgres: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1:8080/health"] - interval: 5s - timeout: 3s - retries: 15 - start_period: 10s - networks: - - smoke - - # ── Infrastructure (stock images) ───────────────────────────────────────── - mysql: - image: mysql:8.0 - environment: - MYSQL_ROOT_PASSWORD: metabuilder - MYSQL_USER: metabuilder - MYSQL_PASSWORD: metabuilder - MYSQL_DATABASE: metabuilder - command: --default-authentication-plugin=mysql_native_password - tmpfs: - - /var/lib/mysql - healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-u", "root", "-pmetabuilder"] - interval: 5s - timeout: 5s - retries: 15 - start_period: 20s - networks: - - smoke - - mongodb: - image: mongo:7.0 - environment: - MONGO_INITDB_ROOT_USERNAME: metabuilder - MONGO_INITDB_ROOT_PASSWORD: metabuilder - tmpfs: - - /data/db - healthcheck: - test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')", "--quiet"] - interval: 5s - timeout: 5s - retries: 15 - networks: - - smoke - - redis: - image: redis:7-alpine - tmpfs: - - /data - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 3s - timeout: 3s - retries: 10 - networks: - - smoke - - # ── Admin tools (real containers, specific ports to match smoke tests) ──── - phpmyadmin: - image: phpmyadmin:latest - ports: - - "8081:80" - environment: - PMA_HOST: mysql - PMA_PORT: "3306" - PMA_USER: metabuilder - PMA_PASSWORD: metabuilder - MYSQL_ROOT_PASSWORD: metabuilder - depends_on: - mysql: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-sf", "http://127.0.0.1/"] - interval: 10s - timeout: 5s - retries: 10 - start_period: 15s - networks: - - smoke - - mongo-express: - image: mongo-express:latest - ports: - - "8082:8081" - environment: - ME_CONFIG_MONGODB_ADMINUSERNAME: metabuilder - ME_CONFIG_MONGODB_ADMINPASSWORD: metabuilder - ME_CONFIG_MONGODB_URL: mongodb://metabuilder:metabuilder@mongodb:27017/?authSource=admin - ME_CONFIG_BASICAUTH: "false" - depends_on: - mongodb: - condition: service_healthy - healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://127.0.0.1:8081/"] - interval: 10s - timeout: 5s - retries: 10 - start_period: 20s - networks: - - smoke - - redisinsight: - image: redis/redisinsight:latest - ports: - - "8083:5540" - depends_on: - redis: - condition: service_healthy - healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://127.0.0.1:5540/api/health"] - interval: 10s - timeout: 5s - retries: 10 - start_period: 10s - networks: - - smoke - -volumes: - dbal-schemas: - driver: local - dbal-templates: - driver: local - -networks: - smoke: - driver: bridge diff --git a/deployment/docker-compose.test.yml b/deployment/docker-compose.test.yml deleted file mode 100644 index 79022b216..000000000 --- a/deployment/docker-compose.test.yml +++ /dev/null @@ -1,53 +0,0 @@ -# docker-compose.test.yml — Lightweight DB containers for DBAL integration tests. -# -# DEPRECATED: testcontainers-sidecar now starts/stops containers automatically. -# Keep as fallback for environments without Docker daemon access (e.g. some CI configs). -# Preferred: build with BUILD_INTEGRATION_TESTS=ON and let testcontainers manage lifecycle. -# -# Uses non-conflicting ports (5433, 3307) so it can run alongside the dev stack. -# tmpfs mounts make containers fast and ephemeral (data discarded on stop). -# -# Usage: -# docker compose -f deployment/docker-compose.test.yml up -d -# export DBAL_TEST_POSTGRES_URL=postgresql://testuser:testpass@localhost:5433/dbal_test -# export DBAL_TEST_MYSQL_URL=mysql://root:testpass@localhost:3307/dbal_test -# cd dbal/production/_build && ctest -R dbal_integration_tests --output-on-failure -# docker compose -f deployment/docker-compose.test.yml down - -services: - - postgres-test: - image: postgres:16-alpine - container_name: dbal-test-postgres - environment: - POSTGRES_PASSWORD: testpass - POSTGRES_USER: testuser - POSTGRES_DB: dbal_test - ports: - - "5433:5432" # 5433 avoids conflict with dev stack on 5432 - tmpfs: - - /var/lib/postgresql/data # in-memory storage, instant teardown - healthcheck: - test: ["CMD-SHELL", "pg_isready -U testuser -d dbal_test"] - interval: 2s - timeout: 5s - retries: 10 - - mysql-test: - image: mysql:8-oracle - container_name: dbal-test-mysql - environment: - MYSQL_ROOT_PASSWORD: testpass - MYSQL_DATABASE: dbal_test - MYSQL_USER: testuser - MYSQL_PASSWORD: testpass - ports: - - "3307:3306" # 3307 avoids conflict with dev stack on 3306 - tmpfs: - - /var/lib/mysql # in-memory storage, instant teardown - healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "-ptestpass"] - interval: 5s - timeout: 10s - retries: 10 - start_period: 20s diff --git a/deployment/verdaccio.yaml b/deployment/verdaccio.yaml index 9822a1463..c44713a24 100644 --- a/deployment/verdaccio.yaml +++ b/deployment/verdaccio.yaml @@ -4,12 +4,12 @@ # # Usage: # Local dev: npx verdaccio --config deployment/verdaccio.yaml & -# Compose: docker compose -f docker-compose.stack.yml up verdaccio +# Compose: docker compose -f compose.yml up verdaccio # CI: uses inline config with /tmp/verdaccio-storage # Then: bash deployment/publish-npm-patches.sh --verdaccio # .npmrc already points @esbuild-kit:registry to localhost:4873 -# Docker container path (volume-mounted in docker-compose.stack.yml). +# Docker container path (volume-mounted in compose.yml). # For local dev, use the CI composite action or npx verdaccio (default config). storage: /verdaccio/storage uplinks: diff --git a/e2e/deployment-smoke.spec.ts b/e2e/deployment-smoke.spec.ts index f27da07b1..983988133 100644 --- a/e2e/deployment-smoke.spec.ts +++ b/e2e/deployment-smoke.spec.ts @@ -8,7 +8,7 @@ import { test, expect } from '@playwright/test' * Playwright dev servers bind to 0.0.0.0 so nginx can proxy via host.docker.internal. * * Local: - * cd deployment && docker compose -f docker-compose.stack.yml up -d + * cd deployment && docker compose -f compose.yml up -d * PLAYWRIGHT_BASE_URL=http://localhost/workflowui/ npx playwright test deployment-smoke */ diff --git a/services/media_daemon/Dockerfile b/services/media_daemon/Dockerfile index ead4a6892..68352e8f1 100644 --- a/services/media_daemon/Dockerfile +++ b/services/media_daemon/Dockerfile @@ -24,7 +24,7 @@ RUN conan install ../build-config/conanfile.txt \ RUN cmake ../build-config \ -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_TOOLCHAIN_FILE=./build/Release/generators/conan_toolchain.cmake + -DCMAKE_TOOLCHAIN_FILE=./conan_toolchain.cmake RUN ninja diff --git a/services/media_daemon/build-config/CMakeLists.txt b/services/media_daemon/build-config/CMakeLists.txt index 8726f2b2c..1892189c4 100644 --- a/services/media_daemon/build-config/CMakeLists.txt +++ b/services/media_daemon/build-config/CMakeLists.txt @@ -9,13 +9,14 @@ find_package(nlohmann_json REQUIRED) find_package(yaml-cpp REQUIRED) find_package(spdlog REQUIRED) find_package(fmt REQUIRED) +find_package(cpr REQUIRED) file(GLOB_RECURSE SOURCES "../src/*.cpp") file(GLOB_RECURSE HEADERS "../include/*.hpp" "../include/*.h") add_executable(media_daemon ${SOURCES} ${HEADERS}) -target_include_directories(media_daemon PRIVATE ../include) +target_include_directories(media_daemon PRIVATE ../include ../src) target_link_libraries(media_daemon PRIVATE Drogon::Drogon @@ -23,6 +24,7 @@ target_link_libraries(media_daemon PRIVATE yaml-cpp::yaml-cpp spdlog::spdlog fmt::fmt + cpr::cpr ) install(TARGETS media_daemon DESTINATION bin) diff --git a/services/media_daemon/build-config/conanfile.txt b/services/media_daemon/build-config/conanfile.txt index 5b7ef625c..cfc7b33d3 100644 --- a/services/media_daemon/build-config/conanfile.txt +++ b/services/media_daemon/build-config/conanfile.txt @@ -4,6 +4,7 @@ nlohmann_json/3.11.3 yaml-cpp/0.8.0 spdlog/1.16.0 fmt/12.0.0 +cpr/1.10.5 [generators] CMakeDeps