feat: Phase 6 - Automated intel scanning (T-021, T-022)

- Add intel_service.py: RSS feed scanner for threat intelligence
  Searches CISA, NIST NVD, SANS ISC, BleepingComputer, The Hacker News,
  Krebs on Security for mentions of MITRE technique IDs and names
- New intel items stored in intel_items table with URL deduplication
- Techniques with new intel flagged with review_required=True
- Add POST /system/run-intel-scan endpoint (admin only)
- Register weekly intel scan job in APScheduler (every 7 days)
- Audit log records each scan execution with summary stats
- Update README with new endpoint and project structure

README.md

@@ -125,6 +125,7 @@ Once the backend is running, access the interactive API documentation at:
 | Method | Route | Auth | Description |
 |--------|-------|------|-------------|
 | POST | `/api/v1/system/sync-mitre` | Admin | Manually trigger MITRE ATT&CK sync |
+| POST | `/api/v1/system/run-intel-scan` | Admin | Manually trigger threat-intel RSS scan |
 | GET | `/api/v1/system/scheduler-status` | Admin | Background scheduler health & job list |
 
 ### Metrics
@@ -177,11 +178,12 @@ Aegis/
 │       ├── dependencies/       # FastAPI dependencies (DI)
 │       │   └── auth.py         # get_current_user, require_role, require_any_role
 │       ├── jobs/               # Background scheduled jobs
-│       │   └── mitre_sync_job.py  # APScheduler job: sync MITRE every 24h
+│       │   └── mitre_sync_job.py  # APScheduler: MITRE sync (24h) + Intel scan (7d)
 │       └── services/           # Business logic services
 │           ├── audit_service.py
 │           ├── status_service.py  # Recalculate technique status from tests
-│           └── mitre_sync_service.py  # MITRE ATT&CK sync via TAXII / GitHub
+│           ├── mitre_sync_service.py  # MITRE ATT&CK sync via TAXII / GitHub
+│           └── intel_service.py   # Automated intel scan via RSS feeds
 └── frontend/                   # React frontend (coming soon)
 ```
 

backend/app/jobs/mitre_sync_job.py

@@ -1,9 +1,13 @@
-"""Scheduled job for periodic MITRE ATT&CK synchronisation.
+"""Scheduled background jobs.
 
-Uses APScheduler's ``BackgroundScheduler`` to run :func:`sync_mitre` every
+Registers periodic tasks on an APScheduler ``BackgroundScheduler``:
-24 hours.  The job manages its own database session (created on entry,
+
-closed in ``finally``) so it is fully independent from FastAPI's
+* **MITRE sync** — every 24 hours (see :func:`sync_mitre`)
-request-scoped sessions.
+* **Intel scan** — every 7 days (see :func:`scan_intel`)
+
+Each job manages its own database session (created on entry, closed in
+``finally``) so it is fully independent from FastAPI's request-scoped
+sessions.
 """
 
 import logging
@@ -12,6 +16,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
 
 from app.database import SessionLocal
 from app.services.mitre_sync_service import sync_mitre
+from app.services.intel_service import scan_intel
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +27,11 @@ logger = logging.getLogger(__name__)
 scheduler = BackgroundScheduler()
 
 
+# ---------------------------------------------------------------------------
+# Job functions
+# ---------------------------------------------------------------------------
+
+
 def _run_mitre_sync() -> None:
     """Execute a MITRE sync inside its own DB session."""
     logger.info("Scheduled MITRE sync job starting...")
@@ -35,11 +45,33 @@ def _run_mitre_sync() -> None:
         db.close()
 
 
-def start_scheduler() -> None:
+def _run_intel_scan() -> None:
-    """Register the MITRE sync job and start the background scheduler.
+    """Execute an intel scan inside its own DB session."""
+    logger.info("Scheduled intel scan job starting...")
+    db = SessionLocal()
+    try:
+        summary = scan_intel(db)
+        logger.info("Scheduled intel scan job finished — %s", summary)
+    except Exception:
+        logger.exception("Scheduled intel scan job failed")
+    finally:
+        db.close()
 
-    The job runs every **24 hours**.  It does **not** fire immediately on
+
-    startup — the first execution happens 24 h after the application boots.
+# ---------------------------------------------------------------------------
+# Scheduler bootstrap
+# ---------------------------------------------------------------------------
+
+
+def start_scheduler() -> None:
+    """Register all periodic jobs and start the background scheduler.
+
+    Jobs registered:
+
+    * ``mitre_sync`` — every **24 hours**
+    * ``intel_scan`` — every **7 days**
+
+    Neither job fires immediately on startup.
     """
     scheduler.add_job(
         _run_mitre_sync,
@@ -49,5 +81,13 @@ def start_scheduler() -> None:
         name="MITRE ATT&CK sync (every 24h)",
         replace_existing=True,
     )
+    scheduler.add_job(
+        _run_intel_scan,
+        trigger="interval",
+        weeks=1,
+        id="intel_scan",
+        name="Intel scan (every 7d)",
+        replace_existing=True,
+    )
     scheduler.start()
-    logger.info("MITRE sync scheduler started (interval=24h)")
+    logger.info("Background scheduler started — mitre_sync (24h), intel_scan (7d)")

backend/app/routers/system.py

@@ -1,7 +1,7 @@
 """System-level endpoints (admin only).
 
 Provides manual triggers for background operations such as the MITRE
-ATT&CK synchronisation, and scheduler health introspection.
+ATT&CK synchronisation, intel scanning, and scheduler health introspection.
 """
 
 from fastapi import APIRouter, Depends
@@ -11,6 +11,7 @@ from app.database import get_db
 from app.dependencies.auth import require_role
 from app.models.user import User
 from app.services.mitre_sync_service import sync_mitre
+from app.services.intel_service import scan_intel
 from app.jobs.mitre_sync_job import scheduler
 
 router = APIRouter(prefix="/system", tags=["system"])
@@ -36,6 +37,25 @@ def trigger_mitre_sync(
     }
 
 
+@router.post("/run-intel-scan")
+def trigger_intel_scan(
+    db: Session = Depends(get_db),
+    current_user: User = Depends(require_role("admin")),
+):
+    """Manually trigger a threat-intelligence scan.
+
+    **Requires** the ``admin`` role.
+
+    Returns a JSON object with the scan summary including the count of
+    new intel items found.
+    """
+    summary = scan_intel(db)
+    return {
+        "message": "Intel scan completed",
+        "new_items": summary["new_items"],
+    }
+
+
 @router.get("/scheduler-status")
 def scheduler_status(
     current_user: User = Depends(require_role("admin")),

backend/app/services/intel_service.py

@@ -0,0 +1,254 @@
+"""Automated threat-intelligence scan service.
+
+Searches public security RSS feeds for mentions of MITRE ATT&CK technique
+IDs and names.  New findings are stored as :class:`IntelItem` records and
+the related technique is flagged for review.
+
+This is an **MVP** implementation — it queries a small set of well-known
+RSS feeds and parses them with the standard-library :mod:`xml.etree`
+parser.  No LLMs or paid APIs are used.
+"""
+
+import logging
+import re
+import xml.etree.ElementTree as ET
+from datetime import datetime
+
+import requests as _requests
+from sqlalchemy.orm import Session
+
+from app.models.intel import IntelItem
+from app.models.technique import Technique
+from app.services.audit_service import log_action
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Public security RSS feeds
+# ---------------------------------------------------------------------------
+
+RSS_FEEDS: list[dict[str, str]] = [
+    {
+        "name": "CISA Alerts",
+        "url": "https://www.cisa.gov/cybersecurity-advisories/all.xml",
+    },
+    {
+        "name": "NIST NVD CVE",
+        "url": "https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss.xml",
+    },
+    {
+        "name": "SANS ISC",
+        "url": "https://isc.sans.edu/rssfeed.xml",
+    },
+    {
+        "name": "BleepingComputer",
+        "url": "https://www.bleepingcomputer.com/feed/",
+    },
+    {
+        "name": "The Hacker News",
+        "url": "https://feeds.feedburner.com/TheHackersNews",
+    },
+    {
+        "name": "Krebs on Security",
+        "url": "https://krebsonsecurity.com/feed/",
+    },
+]
+
+# Timeout for each feed request (seconds)
+_FEED_TIMEOUT = 15
+
+# Maximum number of techniques to scan (to keep MVP fast)
+_MAX_TECHNIQUES = 50
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _fetch_feed(url: str) -> list[dict[str, str]]:
+    """Download and parse an RSS/Atom feed, returning a list of entries.
+
+    Each entry is a dict with keys ``title``, ``link``, and ``description``.
+    Returns an empty list on any error so the scan can continue.
+    """
+    try:
+        resp = _requests.get(url, timeout=_FEED_TIMEOUT, headers={
+            "User-Agent": "AegisPlatform/1.0 IntelScan",
+        })
+        resp.raise_for_status()
+    except Exception as exc:
+        logger.warning("Failed to fetch feed %s: %s", url, exc)
+        return []
+
+    try:
+        root = ET.fromstring(resp.content)
+    except ET.ParseError as exc:
+        logger.warning("Failed to parse feed %s: %s", url, exc)
+        return []
+
+    entries: list[dict[str, str]] = []
+
+    # RSS 2.0 format: <channel><item>...
+    for item in root.iter("item"):
+        title_el = item.find("title")
+        link_el = item.find("link")
+        desc_el = item.find("description")
+        entries.append({
+            "title": title_el.text.strip() if title_el is not None and title_el.text else "",
+            "link": link_el.text.strip() if link_el is not None and link_el.text else "",
+            "description": desc_el.text.strip() if desc_el is not None and desc_el.text else "",
+        })
+
+    # Atom format: <feed><entry>...
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    for entry in root.iter("{http://www.w3.org/2005/Atom}entry"):
+        title_el = entry.find("atom:title", ns)
+        link_el = entry.find("atom:link", ns)
+        summary_el = entry.find("atom:summary", ns)
+        link_href = ""
+        if link_el is not None:
+            link_href = link_el.get("href", "")
+        entries.append({
+            "title": title_el.text.strip() if title_el is not None and title_el.text else "",
+            "link": link_href.strip(),
+            "description": summary_el.text.strip() if summary_el is not None and summary_el.text else "",
+        })
+
+    return entries
+
+
+def _build_patterns(technique: Technique) -> list[re.Pattern]:
+    """Build regex patterns to search feed content for a given technique."""
+    patterns: list[re.Pattern] = []
+
+    mitre_id = re.escape(technique.mitre_id)
+    patterns.append(re.compile(mitre_id, re.IGNORECASE))
+
+    # Technique name — match if the full name appears
+    if technique.name and len(technique.name) > 4:
+        name_escaped = re.escape(technique.name)
+        patterns.append(re.compile(name_escaped, re.IGNORECASE))
+
+    return patterns
+
+
+def _entry_matches(entry: dict[str, str], patterns: list[re.Pattern]) -> bool:
+    """Return True if any pattern matches the entry's title or description."""
+    text = f"{entry.get('title', '')} {entry.get('description', '')}"
+    return any(p.search(text) for p in patterns)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def scan_intel(db: Session) -> dict:
+    """Run the intel scan across RSS feeds for known techniques.
+
+    Parameters
+    ----------
+    db : Session
+        Active SQLAlchemy database session.
+
+    Returns
+    -------
+    dict
+        Summary with keys ``new_items``, ``duplicates_skipped``,
+        ``techniques_flagged``, ``feeds_checked``.
+    """
+    logger.info("Intel scan starting...")
+
+    # 1. Load techniques (limit for MVP speed)
+    techniques = (
+        db.query(Technique)
+        .order_by(Technique.mitre_id)
+        .limit(_MAX_TECHNIQUES)
+        .all()
+    )
+    logger.info("Scanning %d techniques against %d feeds", len(techniques), len(RSS_FEEDS))
+
+    # 2. Pre-load all existing intel URLs for dedup
+    existing_urls: set[str] = {
+        row[0] for row in db.query(IntelItem.url).all()
+    }
+
+    # 3. Fetch all feeds once
+    all_entries: list[tuple[str, dict[str, str]]] = []  # (feed_name, entry)
+    feeds_ok = 0
+    for feed in RSS_FEEDS:
+        entries = _fetch_feed(feed["url"])
+        if entries:
+            feeds_ok += 1
+        for entry in entries:
+            all_entries.append((feed["name"], entry))
+
+    logger.info("Fetched %d entries from %d/%d feeds", len(all_entries), feeds_ok, len(RSS_FEEDS))
+
+    # 4. Match entries to techniques
+    new_items = 0
+    duplicates_skipped = 0
+    techniques_flagged: set[str] = set()
+
+    for technique in techniques:
+        patterns = _build_patterns(technique)
+
+        for feed_name, entry in all_entries:
+            if not _entry_matches(entry, patterns):
+                continue
+
+            url = entry.get("link", "").strip()
+            if not url:
+                continue
+
+            # Dedup
+            if url in existing_urls:
+                duplicates_skipped += 1
+                continue
+
+            # Create IntelItem
+            intel_item = IntelItem(
+                technique_id=technique.id,
+                url=url,
+                title=entry.get("title", "")[:500],
+                source=feed_name,
+                detected_at=datetime.utcnow(),
+                reviewed=False,
+            )
+            db.add(intel_item)
+            existing_urls.add(url)
+            new_items += 1
+
+            # Flag technique for review
+            if not technique.review_required:
+                technique.review_required = True
+                techniques_flagged.add(technique.mitre_id)
+
+    # 5. Single commit
+    db.commit()
+
+    summary = {
+        "new_items": new_items,
+        "duplicates_skipped": duplicates_skipped,
+        "techniques_flagged": len(techniques_flagged),
+        "feeds_checked": feeds_ok,
+    }
+
+    logger.info(
+        "Intel scan complete — new=%d, duplicates_skipped=%d, "
+        "techniques_flagged=%d, feeds_checked=%d",
+        new_items, duplicates_skipped, len(techniques_flagged), feeds_ok,
+    )
+
+    # 6. Audit log
+    log_action(
+        db,
+        user_id=None,
+        action="intel_scan",
+        entity_type="intel_item",
+        entity_id=None,
+        details=summary,
+    )
+
+    return summary