diff --git a/README.md b/README.md index 9dddf0e..4c37101 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Once the backend is running, access the interactive API documentation at: | Method | Route | Auth | Description | |--------|-------|------|-------------| | POST | `/api/v1/system/sync-mitre` | Admin | Manually trigger MITRE ATT&CK sync | +| POST | `/api/v1/system/run-intel-scan` | Admin | Manually trigger threat-intel RSS scan | | GET | `/api/v1/system/scheduler-status` | Admin | Background scheduler health & job list | ### Metrics @@ -177,11 +178,12 @@ Aegis/ │ ├── dependencies/ # FastAPI dependencies (DI) │ │ └── auth.py # get_current_user, require_role, require_any_role │ ├── jobs/ # Background scheduled jobs -│ │ └── mitre_sync_job.py # APScheduler job: sync MITRE every 24h +│ │ └── mitre_sync_job.py # APScheduler: MITRE sync (24h) + Intel scan (7d) │ └── services/ # Business logic services │ ├── audit_service.py │ ├── status_service.py # Recalculate technique status from tests -│ └── mitre_sync_service.py # MITRE ATT&CK sync via TAXII / GitHub +│ ├── mitre_sync_service.py # MITRE ATT&CK sync via TAXII / GitHub +│ └── intel_service.py # Automated intel scan via RSS feeds └── frontend/ # React frontend (coming soon) ``` diff --git a/backend/app/jobs/mitre_sync_job.py b/backend/app/jobs/mitre_sync_job.py index 002ad3c..bacc91e 100644 --- a/backend/app/jobs/mitre_sync_job.py +++ b/backend/app/jobs/mitre_sync_job.py @@ -1,9 +1,13 @@ -"""Scheduled job for periodic MITRE ATT&CK synchronisation. +"""Scheduled background jobs. -Uses APScheduler's ``BackgroundScheduler`` to run :func:`sync_mitre` every -24 hours. The job manages its own database session (created on entry, -closed in ``finally``) so it is fully independent from FastAPI's -request-scoped sessions. +Registers periodic tasks on an APScheduler ``BackgroundScheduler``: + +* **MITRE sync** — every 24 hours (see :func:`sync_mitre`) +* **Intel scan** — every 7 days (see :func:`scan_intel`) + +Each job manages its own database session (created on entry, closed in +``finally``) so it is fully independent from FastAPI's request-scoped +sessions. """ import logging @@ -12,6 +16,7 @@ from apscheduler.schedulers.background import BackgroundScheduler from app.database import SessionLocal from app.services.mitre_sync_service import sync_mitre +from app.services.intel_service import scan_intel logger = logging.getLogger(__name__) @@ -22,6 +27,11 @@ logger = logging.getLogger(__name__) scheduler = BackgroundScheduler() +# --------------------------------------------------------------------------- +# Job functions +# --------------------------------------------------------------------------- + + def _run_mitre_sync() -> None: """Execute a MITRE sync inside its own DB session.""" logger.info("Scheduled MITRE sync job starting...") @@ -35,11 +45,33 @@ def _run_mitre_sync() -> None: db.close() -def start_scheduler() -> None: - """Register the MITRE sync job and start the background scheduler. +def _run_intel_scan() -> None: + """Execute an intel scan inside its own DB session.""" + logger.info("Scheduled intel scan job starting...") + db = SessionLocal() + try: + summary = scan_intel(db) + logger.info("Scheduled intel scan job finished — %s", summary) + except Exception: + logger.exception("Scheduled intel scan job failed") + finally: + db.close() - The job runs every **24 hours**. It does **not** fire immediately on - startup — the first execution happens 24 h after the application boots. + +# --------------------------------------------------------------------------- +# Scheduler bootstrap +# --------------------------------------------------------------------------- + + +def start_scheduler() -> None: + """Register all periodic jobs and start the background scheduler. + + Jobs registered: + + * ``mitre_sync`` — every **24 hours** + * ``intel_scan`` — every **7 days** + + Neither job fires immediately on startup. """ scheduler.add_job( _run_mitre_sync, @@ -49,5 +81,13 @@ def start_scheduler() -> None: name="MITRE ATT&CK sync (every 24h)", replace_existing=True, ) + scheduler.add_job( + _run_intel_scan, + trigger="interval", + weeks=1, + id="intel_scan", + name="Intel scan (every 7d)", + replace_existing=True, + ) scheduler.start() - logger.info("MITRE sync scheduler started (interval=24h)") + logger.info("Background scheduler started — mitre_sync (24h), intel_scan (7d)") diff --git a/backend/app/routers/system.py b/backend/app/routers/system.py index d1e2807..a0fefe0 100644 --- a/backend/app/routers/system.py +++ b/backend/app/routers/system.py @@ -1,7 +1,7 @@ """System-level endpoints (admin only). Provides manual triggers for background operations such as the MITRE -ATT&CK synchronisation, and scheduler health introspection. +ATT&CK synchronisation, intel scanning, and scheduler health introspection. """ from fastapi import APIRouter, Depends @@ -11,6 +11,7 @@ from app.database import get_db from app.dependencies.auth import require_role from app.models.user import User from app.services.mitre_sync_service import sync_mitre +from app.services.intel_service import scan_intel from app.jobs.mitre_sync_job import scheduler router = APIRouter(prefix="/system", tags=["system"]) @@ -36,6 +37,25 @@ def trigger_mitre_sync( } +@router.post("/run-intel-scan") +def trigger_intel_scan( + db: Session = Depends(get_db), + current_user: User = Depends(require_role("admin")), +): + """Manually trigger a threat-intelligence scan. + + **Requires** the ``admin`` role. + + Returns a JSON object with the scan summary including the count of + new intel items found. + """ + summary = scan_intel(db) + return { + "message": "Intel scan completed", + "new_items": summary["new_items"], + } + + @router.get("/scheduler-status") def scheduler_status( current_user: User = Depends(require_role("admin")), diff --git a/backend/app/services/intel_service.py b/backend/app/services/intel_service.py new file mode 100644 index 0000000..2962100 --- /dev/null +++ b/backend/app/services/intel_service.py @@ -0,0 +1,254 @@ +"""Automated threat-intelligence scan service. + +Searches public security RSS feeds for mentions of MITRE ATT&CK technique +IDs and names. New findings are stored as :class:`IntelItem` records and +the related technique is flagged for review. + +This is an **MVP** implementation — it queries a small set of well-known +RSS feeds and parses them with the standard-library :mod:`xml.etree` +parser. No LLMs or paid APIs are used. +""" + +import logging +import re +import xml.etree.ElementTree as ET +from datetime import datetime + +import requests as _requests +from sqlalchemy.orm import Session + +from app.models.intel import IntelItem +from app.models.technique import Technique +from app.services.audit_service import log_action + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Public security RSS feeds +# --------------------------------------------------------------------------- + +RSS_FEEDS: list[dict[str, str]] = [ + { + "name": "CISA Alerts", + "url": "https://www.cisa.gov/cybersecurity-advisories/all.xml", + }, + { + "name": "NIST NVD CVE", + "url": "https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss.xml", + }, + { + "name": "SANS ISC", + "url": "https://isc.sans.edu/rssfeed.xml", + }, + { + "name": "BleepingComputer", + "url": "https://www.bleepingcomputer.com/feed/", + }, + { + "name": "The Hacker News", + "url": "https://feeds.feedburner.com/TheHackersNews", + }, + { + "name": "Krebs on Security", + "url": "https://krebsonsecurity.com/feed/", + }, +] + +# Timeout for each feed request (seconds) +_FEED_TIMEOUT = 15 + +# Maximum number of techniques to scan (to keep MVP fast) +_MAX_TECHNIQUES = 50 + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _fetch_feed(url: str) -> list[dict[str, str]]: + """Download and parse an RSS/Atom feed, returning a list of entries. + + Each entry is a dict with keys ``title``, ``link``, and ``description``. + Returns an empty list on any error so the scan can continue. + """ + try: + resp = _requests.get(url, timeout=_FEED_TIMEOUT, headers={ + "User-Agent": "AegisPlatform/1.0 IntelScan", + }) + resp.raise_for_status() + except Exception as exc: + logger.warning("Failed to fetch feed %s: %s", url, exc) + return [] + + try: + root = ET.fromstring(resp.content) + except ET.ParseError as exc: + logger.warning("Failed to parse feed %s: %s", url, exc) + return [] + + entries: list[dict[str, str]] = [] + + # RSS 2.0 format: ... + for item in root.iter("item"): + title_el = item.find("title") + link_el = item.find("link") + desc_el = item.find("description") + entries.append({ + "title": title_el.text.strip() if title_el is not None and title_el.text else "", + "link": link_el.text.strip() if link_el is not None and link_el.text else "", + "description": desc_el.text.strip() if desc_el is not None and desc_el.text else "", + }) + + # Atom format: ... + ns = {"atom": "http://www.w3.org/2005/Atom"} + for entry in root.iter("{http://www.w3.org/2005/Atom}entry"): + title_el = entry.find("atom:title", ns) + link_el = entry.find("atom:link", ns) + summary_el = entry.find("atom:summary", ns) + link_href = "" + if link_el is not None: + link_href = link_el.get("href", "") + entries.append({ + "title": title_el.text.strip() if title_el is not None and title_el.text else "", + "link": link_href.strip(), + "description": summary_el.text.strip() if summary_el is not None and summary_el.text else "", + }) + + return entries + + +def _build_patterns(technique: Technique) -> list[re.Pattern]: + """Build regex patterns to search feed content for a given technique.""" + patterns: list[re.Pattern] = [] + + mitre_id = re.escape(technique.mitre_id) + patterns.append(re.compile(mitre_id, re.IGNORECASE)) + + # Technique name — match if the full name appears + if technique.name and len(technique.name) > 4: + name_escaped = re.escape(technique.name) + patterns.append(re.compile(name_escaped, re.IGNORECASE)) + + return patterns + + +def _entry_matches(entry: dict[str, str], patterns: list[re.Pattern]) -> bool: + """Return True if any pattern matches the entry's title or description.""" + text = f"{entry.get('title', '')} {entry.get('description', '')}" + return any(p.search(text) for p in patterns) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def scan_intel(db: Session) -> dict: + """Run the intel scan across RSS feeds for known techniques. + + Parameters + ---------- + db : Session + Active SQLAlchemy database session. + + Returns + ------- + dict + Summary with keys ``new_items``, ``duplicates_skipped``, + ``techniques_flagged``, ``feeds_checked``. + """ + logger.info("Intel scan starting...") + + # 1. Load techniques (limit for MVP speed) + techniques = ( + db.query(Technique) + .order_by(Technique.mitre_id) + .limit(_MAX_TECHNIQUES) + .all() + ) + logger.info("Scanning %d techniques against %d feeds", len(techniques), len(RSS_FEEDS)) + + # 2. Pre-load all existing intel URLs for dedup + existing_urls: set[str] = { + row[0] for row in db.query(IntelItem.url).all() + } + + # 3. Fetch all feeds once + all_entries: list[tuple[str, dict[str, str]]] = [] # (feed_name, entry) + feeds_ok = 0 + for feed in RSS_FEEDS: + entries = _fetch_feed(feed["url"]) + if entries: + feeds_ok += 1 + for entry in entries: + all_entries.append((feed["name"], entry)) + + logger.info("Fetched %d entries from %d/%d feeds", len(all_entries), feeds_ok, len(RSS_FEEDS)) + + # 4. Match entries to techniques + new_items = 0 + duplicates_skipped = 0 + techniques_flagged: set[str] = set() + + for technique in techniques: + patterns = _build_patterns(technique) + + for feed_name, entry in all_entries: + if not _entry_matches(entry, patterns): + continue + + url = entry.get("link", "").strip() + if not url: + continue + + # Dedup + if url in existing_urls: + duplicates_skipped += 1 + continue + + # Create IntelItem + intel_item = IntelItem( + technique_id=technique.id, + url=url, + title=entry.get("title", "")[:500], + source=feed_name, + detected_at=datetime.utcnow(), + reviewed=False, + ) + db.add(intel_item) + existing_urls.add(url) + new_items += 1 + + # Flag technique for review + if not technique.review_required: + technique.review_required = True + techniques_flagged.add(technique.mitre_id) + + # 5. Single commit + db.commit() + + summary = { + "new_items": new_items, + "duplicates_skipped": duplicates_skipped, + "techniques_flagged": len(techniques_flagged), + "feeds_checked": feeds_ok, + } + + logger.info( + "Intel scan complete — new=%d, duplicates_skipped=%d, " + "techniques_flagged=%d, feeds_checked=%d", + new_items, duplicates_skipped, len(techniques_flagged), feeds_ok, + ) + + # 6. Audit log + log_action( + db, + user_id=None, + action="intel_scan", + entity_type="intel_item", + entity_id=None, + details=summary, + ) + + return summary