diff --git a/README.md b/README.md index 9ea0592..d781161 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Aegis is a comprehensive platform for tracking and managing security coverage ag ## Features -- **MITRE ATT&CK Integration**: Automatic synchronization with the MITRE ATT&CK framework via TAXII +- **MITRE ATT&CK Integration**: Automatic synchronization with the MITRE ATT&CK framework via TAXII (with GitHub fallback), scheduled every 24h - **Coverage Tracking**: Track validation status for each technique (validated, partial, not covered, in progress) - **Test Management**: Document and manage security tests with full audit trail - **Evidence Storage**: Secure evidence file storage with SHA256 integrity verification @@ -121,6 +121,12 @@ Once the backend is running, access the interactive API documentation at: | POST | `/api/v1/tests/{test_id}/evidence` | Authenticated | Upload evidence file (SHA-256 verified) | | GET | `/api/v1/evidence/{id}` | Authenticated | Get metadata + presigned download URL | +### System +| Method | Route | Auth | Description | +|--------|-------|------|-------------| +| POST | `/api/v1/system/sync-mitre` | Admin | Manually trigger MITRE ATT&CK sync | +| GET | `/api/v1/system/scheduler-status` | Admin | Background scheduler health & job list | + ## Project Structure ``` @@ -159,12 +165,16 @@ Aegis/ │ │ ├── auth.py # POST /auth/login, GET /auth/me │ │ ├── techniques.py # CRUD techniques (list, detail, create, update, review) │ │ ├── tests.py # CRUD tests (create, detail, update, validate, reject) -│ │ └── evidence.py # Upload evidence, presigned download +│ │ ├── evidence.py # Upload evidence, presigned download +│ │ └── system.py # MITRE sync trigger, scheduler status │ ├── dependencies/ # FastAPI dependencies (DI) │ │ └── auth.py # get_current_user, require_role, require_any_role +│ ├── jobs/ # Background scheduled jobs +│ │ └── mitre_sync_job.py # APScheduler job: sync MITRE every 24h │ └── services/ # Business logic services │ ├── audit_service.py -│ └── status_service.py # Recalculate technique status from tests +│ ├── status_service.py # Recalculate technique status from tests +│ └── mitre_sync_service.py # MITRE ATT&CK sync via TAXII / GitHub └── frontend/ # React frontend (coming soon) ``` diff --git a/backend/app/jobs/__init__.py b/backend/app/jobs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/jobs/mitre_sync_job.py b/backend/app/jobs/mitre_sync_job.py new file mode 100644 index 0000000..002ad3c --- /dev/null +++ b/backend/app/jobs/mitre_sync_job.py @@ -0,0 +1,53 @@ +"""Scheduled job for periodic MITRE ATT&CK synchronisation. + +Uses APScheduler's ``BackgroundScheduler`` to run :func:`sync_mitre` every +24 hours. The job manages its own database session (created on entry, +closed in ``finally``) so it is fully independent from FastAPI's +request-scoped sessions. +""" + +import logging + +from apscheduler.schedulers.background import BackgroundScheduler + +from app.database import SessionLocal +from app.services.mitre_sync_service import sync_mitre + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Module-level scheduler instance +# --------------------------------------------------------------------------- + +scheduler = BackgroundScheduler() + + +def _run_mitre_sync() -> None: + """Execute a MITRE sync inside its own DB session.""" + logger.info("Scheduled MITRE sync job starting...") + db = SessionLocal() + try: + summary = sync_mitre(db) + logger.info("Scheduled MITRE sync job finished — %s", summary) + except Exception: + logger.exception("Scheduled MITRE sync job failed") + finally: + db.close() + + +def start_scheduler() -> None: + """Register the MITRE sync job and start the background scheduler. + + The job runs every **24 hours**. It does **not** fire immediately on + startup — the first execution happens 24 h after the application boots. + """ + scheduler.add_job( + _run_mitre_sync, + trigger="interval", + hours=24, + id="mitre_sync", + name="MITRE ATT&CK sync (every 24h)", + replace_existing=True, + ) + scheduler.start() + logger.info("MITRE sync scheduler started (interval=24h)") diff --git a/backend/app/main.py b/backend/app/main.py index ff29ef7..d9c553a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,3 +1,4 @@ +import logging from contextlib import asynccontextmanager from fastapi import FastAPI @@ -7,13 +8,24 @@ from app.routers import auth as auth_router from app.routers import techniques as techniques_router from app.routers import tests as tests_router from app.routers import evidence as evidence_router +from app.routers import system as system_router from app.storage import ensure_bucket_exists +from app.jobs.mitre_sync_job import start_scheduler, scheduler + +# ── Logging ─────────────────────────────────────────────────────────────── +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)-8s %(name)s — %(message)s", +) @asynccontextmanager async def lifespan(app: FastAPI): """Startup / shutdown logic.""" ensure_bucket_exists() + start_scheduler() yield + # Graceful shutdown of the background scheduler + scheduler.shutdown(wait=False) app = FastAPI(title="Attack Coverage Platform", lifespan=lifespan) @@ -32,6 +44,7 @@ app.include_router(auth_router.router, prefix="/api/v1") app.include_router(techniques_router.router, prefix="/api/v1") app.include_router(tests_router.router, prefix="/api/v1") app.include_router(evidence_router.router, prefix="/api/v1") +app.include_router(system_router.router, prefix="/api/v1") @app.get("/health") diff --git a/backend/app/routers/system.py b/backend/app/routers/system.py new file mode 100644 index 0000000..d1e2807 --- /dev/null +++ b/backend/app/routers/system.py @@ -0,0 +1,58 @@ +"""System-level endpoints (admin only). + +Provides manual triggers for background operations such as the MITRE +ATT&CK synchronisation, and scheduler health introspection. +""" + +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session + +from app.database import get_db +from app.dependencies.auth import require_role +from app.models.user import User +from app.services.mitre_sync_service import sync_mitre +from app.jobs.mitre_sync_job import scheduler + +router = APIRouter(prefix="/system", tags=["system"]) + + +@router.post("/sync-mitre") +def trigger_mitre_sync( + db: Session = Depends(get_db), + current_user: User = Depends(require_role("admin")), +): + """Manually trigger a MITRE ATT&CK synchronisation. + + **Requires** the ``admin`` role. + + Returns a JSON object with the sync summary including the count of + new and updated techniques. + """ + summary = sync_mitre(db) + return { + "message": "MITRE sync completed", + "new": summary["created"], + "updated": summary["updated"], + } + + +@router.get("/scheduler-status") +def scheduler_status( + current_user: User = Depends(require_role("admin")), +): + """Return the current state of the background scheduler. + + **Requires** the ``admin`` role. + """ + jobs = scheduler.get_jobs() + return { + "running": scheduler.running, + "jobs": [ + { + "id": job.id, + "name": job.name, + "next_run_time": str(job.next_run_time) if job.next_run_time else None, + } + for job in jobs + ], + } diff --git a/backend/app/services/mitre_sync_service.py b/backend/app/services/mitre_sync_service.py new file mode 100644 index 0000000..c09f809 --- /dev/null +++ b/backend/app/services/mitre_sync_service.py @@ -0,0 +1,247 @@ +"""Service for synchronizing MITRE ATT&CK techniques via TAXII 2.0. + +Connects to the official MITRE CTI TAXII server, fetches the Enterprise +ATT&CK collection, and upserts attack-pattern objects into the local +``techniques`` table. Falls back to the MITRE CTI GitHub repository +when the TAXII server is unreachable. +""" + +import logging +from datetime import datetime + +import requests as _requests +from sqlalchemy.orm import Session +from taxii2client.v20 import Server as TaxiiServer + +from app.models.technique import Technique +from app.models.enums import TechniqueStatus +from app.services.audit_service import log_action + +logger = logging.getLogger(__name__) + +TAXII_SERVER_URL = "https://cti-taxii.mitre.org/taxii/" +MITRE_SOURCE_NAME = "mitre-attack" +GITHUB_ENTERPRISE_URL = ( + "https://raw.githubusercontent.com/mitre/cti/master/" + "enterprise-attack/enterprise-attack.json" +) + + +def _extract_mitre_id(external_references: list) -> str | None: + """Return the MITRE ATT&CK ID (e.g. ``T1059.001``) from external_references.""" + if not external_references: + return None + for ref in external_references: + if ref.get("source_name") == MITRE_SOURCE_NAME: + return ref.get("external_id") + return None + + +def _extract_tactics(kill_chain_phases: list) -> str | None: + """Return a comma-separated string of tactic phase names.""" + if not kill_chain_phases: + return None + tactics = [ + phase.get("phase_name") + for phase in kill_chain_phases + if phase.get("kill_chain_name") == "mitre-attack" + ] + return ", ".join(tactics) if tactics else None + + +def _extract_platforms(stix_object: dict) -> list: + """Return the list of platforms from the STIX object.""" + return stix_object.get("x_mitre_platforms", []) + + +def _extract_version(stix_object: dict) -> str | None: + """Return the MITRE ATT&CK version string.""" + return stix_object.get("x_mitre_version") + + +def _extract_last_modified(stix_object: dict) -> datetime | None: + """Return the ``modified`` timestamp as a datetime, or None.""" + modified = stix_object.get("modified") + if modified is None: + return None + if isinstance(modified, datetime): + return modified + try: + return datetime.fromisoformat(modified.replace("Z", "+00:00")) + except (ValueError, AttributeError): + return None + + +def _fetch_attack_patterns_taxii() -> list[dict]: + """Connect to the MITRE TAXII server and return all attack-pattern objects.""" + logger.info("Connecting to MITRE TAXII server at %s", TAXII_SERVER_URL) + server = TaxiiServer(TAXII_SERVER_URL) + + api_root = server.api_roots[0] + collection = api_root.collections[0] # Enterprise ATT&CK + + logger.info( + "Fetching objects from collection '%s' (id=%s)", + collection.title, + collection.id, + ) + + bundle = collection.get_objects() + objects = bundle.get("objects", []) + + attack_patterns = [ + obj for obj in objects if obj.get("type") == "attack-pattern" + ] + logger.info("Retrieved %d attack-pattern objects via TAXII", len(attack_patterns)) + return attack_patterns + + +def _fetch_attack_patterns_github() -> list[dict]: + """Fallback: fetch Enterprise ATT&CK bundle from the MITRE CTI GitHub repo.""" + logger.info("Fetching Enterprise ATT&CK bundle from GitHub (%s)", GITHUB_ENTERPRISE_URL) + resp = _requests.get(GITHUB_ENTERPRISE_URL, timeout=120) + resp.raise_for_status() + bundle = resp.json() + objects = bundle.get("objects", []) + + attack_patterns = [ + obj for obj in objects if obj.get("type") == "attack-pattern" + ] + logger.info("Retrieved %d attack-pattern objects via GitHub", len(attack_patterns)) + return attack_patterns + + +def _fetch_attack_patterns() -> list[dict]: + """Return all attack-pattern objects, trying TAXII first then GitHub.""" + try: + return _fetch_attack_patterns_taxii() + except Exception as exc: + logger.warning( + "TAXII server unavailable (%s), falling back to GitHub mirror", + exc, + ) + return _fetch_attack_patterns_github() + + +def sync_mitre(db: Session) -> dict: + """Synchronize MITRE ATT&CK techniques into the local database. + + Parameters + ---------- + db : Session + Active SQLAlchemy database session. + + Returns + ------- + dict + Summary with keys ``created``, ``updated``, ``unchanged``, ``skipped``. + """ + attack_patterns = _fetch_attack_patterns() + + # Pre-load existing techniques keyed by mitre_id for fast lookup + existing_techniques: dict[str, Technique] = { + t.mitre_id: t for t in db.query(Technique).all() + } + + created = 0 + updated = 0 + unchanged = 0 + skipped = 0 + + for obj in attack_patterns: + # ------------------------------------------------------------------ + # Skip revoked / deprecated objects + # ------------------------------------------------------------------ + if obj.get("revoked", False) or obj.get("x_mitre_deprecated", False): + skipped += 1 + continue + + mitre_id = _extract_mitre_id(obj.get("external_references", [])) + if not mitre_id: + skipped += 1 + continue + + name = obj.get("name", "") + description = obj.get("description", "") + tactic = _extract_tactics(obj.get("kill_chain_phases", [])) + platforms = _extract_platforms(obj) + version = _extract_version(obj) + last_modified = _extract_last_modified(obj) + is_subtechnique = "." in mitre_id + parent_mitre_id = mitre_id.split(".")[0] if is_subtechnique else None + + existing = existing_techniques.get(mitre_id) + + if existing is None: + # ---- Create new technique ---- + technique = Technique( + mitre_id=mitre_id, + name=name, + description=description, + tactic=tactic, + platforms=platforms, + mitre_version=version, + mitre_last_modified=last_modified, + is_subtechnique=is_subtechnique, + parent_mitre_id=parent_mitre_id, + status_global=TechniqueStatus.not_evaluated, + review_required=False, + ) + db.add(technique) + existing_techniques[mitre_id] = technique + created += 1 + else: + # ---- Update if name or description changed ---- + changes = False + + if existing.name != name: + existing.name = name + changes = True + + if (existing.description or "") != (description or ""): + existing.description = description + changes = True + + # Always keep metadata up-to-date (does not trigger review) + existing.tactic = tactic + existing.platforms = platforms + existing.mitre_version = version + existing.mitre_last_modified = last_modified + existing.is_subtechnique = is_subtechnique + existing.parent_mitre_id = parent_mitre_id + + if changes: + existing.review_required = True + updated += 1 + else: + unchanged += 1 + + # Single commit for the whole batch + db.commit() + + summary = { + "created": created, + "updated": updated, + "unchanged": unchanged, + "skipped": skipped, + } + + logger.info( + "MITRE sync complete — created=%d, updated=%d, unchanged=%d, skipped=%d", + created, + updated, + unchanged, + skipped, + ) + + # Audit log (system action → user_id=None) + log_action( + db, + user_id=None, + action="mitre_sync", + entity_type="technique", + entity_id=None, + details=summary, + ) + + return summary