feat: Phase 6 - Automated intel scanning (T-021, T-022)
- Add intel_service.py: RSS feed scanner for threat intelligence Searches CISA, NIST NVD, SANS ISC, BleepingComputer, The Hacker News, Krebs on Security for mentions of MITRE technique IDs and names - New intel items stored in intel_items table with URL deduplication - Techniques with new intel flagged with review_required=True - Add POST /system/run-intel-scan endpoint (admin only) - Register weekly intel scan job in APScheduler (every 7 days) - Audit log records each scan execution with summary stats - Update README with new endpoint and project structure
This commit is contained in:
@@ -1,9 +1,13 @@
|
||||
"""Scheduled job for periodic MITRE ATT&CK synchronisation.
|
||||
"""Scheduled background jobs.
|
||||
|
||||
Uses APScheduler's ``BackgroundScheduler`` to run :func:`sync_mitre` every
|
||||
24 hours. The job manages its own database session (created on entry,
|
||||
closed in ``finally``) so it is fully independent from FastAPI's
|
||||
request-scoped sessions.
|
||||
Registers periodic tasks on an APScheduler ``BackgroundScheduler``:
|
||||
|
||||
* **MITRE sync** — every 24 hours (see :func:`sync_mitre`)
|
||||
* **Intel scan** — every 7 days (see :func:`scan_intel`)
|
||||
|
||||
Each job manages its own database session (created on entry, closed in
|
||||
``finally``) so it is fully independent from FastAPI's request-scoped
|
||||
sessions.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -12,6 +16,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.services.mitre_sync_service import sync_mitre
|
||||
from app.services.intel_service import scan_intel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -22,6 +27,11 @@ logger = logging.getLogger(__name__)
|
||||
scheduler = BackgroundScheduler()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_mitre_sync() -> None:
|
||||
"""Execute a MITRE sync inside its own DB session."""
|
||||
logger.info("Scheduled MITRE sync job starting...")
|
||||
@@ -35,11 +45,33 @@ def _run_mitre_sync() -> None:
|
||||
db.close()
|
||||
|
||||
|
||||
def start_scheduler() -> None:
|
||||
"""Register the MITRE sync job and start the background scheduler.
|
||||
def _run_intel_scan() -> None:
|
||||
"""Execute an intel scan inside its own DB session."""
|
||||
logger.info("Scheduled intel scan job starting...")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
summary = scan_intel(db)
|
||||
logger.info("Scheduled intel scan job finished — %s", summary)
|
||||
except Exception:
|
||||
logger.exception("Scheduled intel scan job failed")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
The job runs every **24 hours**. It does **not** fire immediately on
|
||||
startup — the first execution happens 24 h after the application boots.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scheduler bootstrap
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def start_scheduler() -> None:
|
||||
"""Register all periodic jobs and start the background scheduler.
|
||||
|
||||
Jobs registered:
|
||||
|
||||
* ``mitre_sync`` — every **24 hours**
|
||||
* ``intel_scan`` — every **7 days**
|
||||
|
||||
Neither job fires immediately on startup.
|
||||
"""
|
||||
scheduler.add_job(
|
||||
_run_mitre_sync,
|
||||
@@ -49,5 +81,13 @@ def start_scheduler() -> None:
|
||||
name="MITRE ATT&CK sync (every 24h)",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
_run_intel_scan,
|
||||
trigger="interval",
|
||||
weeks=1,
|
||||
id="intel_scan",
|
||||
name="Intel scan (every 7d)",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.start()
|
||||
logger.info("MITRE sync scheduler started (interval=24h)")
|
||||
logger.info("Background scheduler started — mitre_sync (24h), intel_scan (7d)")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""System-level endpoints (admin only).
|
||||
|
||||
Provides manual triggers for background operations such as the MITRE
|
||||
ATT&CK synchronisation, and scheduler health introspection.
|
||||
ATT&CK synchronisation, intel scanning, and scheduler health introspection.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
@@ -11,6 +11,7 @@ from app.database import get_db
|
||||
from app.dependencies.auth import require_role
|
||||
from app.models.user import User
|
||||
from app.services.mitre_sync_service import sync_mitre
|
||||
from app.services.intel_service import scan_intel
|
||||
from app.jobs.mitre_sync_job import scheduler
|
||||
|
||||
router = APIRouter(prefix="/system", tags=["system"])
|
||||
@@ -36,6 +37,25 @@ def trigger_mitre_sync(
|
||||
}
|
||||
|
||||
|
||||
@router.post("/run-intel-scan")
|
||||
def trigger_intel_scan(
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(require_role("admin")),
|
||||
):
|
||||
"""Manually trigger a threat-intelligence scan.
|
||||
|
||||
**Requires** the ``admin`` role.
|
||||
|
||||
Returns a JSON object with the scan summary including the count of
|
||||
new intel items found.
|
||||
"""
|
||||
summary = scan_intel(db)
|
||||
return {
|
||||
"message": "Intel scan completed",
|
||||
"new_items": summary["new_items"],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/scheduler-status")
|
||||
def scheduler_status(
|
||||
current_user: User = Depends(require_role("admin")),
|
||||
|
||||
254
backend/app/services/intel_service.py
Normal file
254
backend/app/services/intel_service.py
Normal file
@@ -0,0 +1,254 @@
|
||||
"""Automated threat-intelligence scan service.
|
||||
|
||||
Searches public security RSS feeds for mentions of MITRE ATT&CK technique
|
||||
IDs and names. New findings are stored as :class:`IntelItem` records and
|
||||
the related technique is flagged for review.
|
||||
|
||||
This is an **MVP** implementation — it queries a small set of well-known
|
||||
RSS feeds and parses them with the standard-library :mod:`xml.etree`
|
||||
parser. No LLMs or paid APIs are used.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime
|
||||
|
||||
import requests as _requests
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models.intel import IntelItem
|
||||
from app.models.technique import Technique
|
||||
from app.services.audit_service import log_action
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public security RSS feeds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RSS_FEEDS: list[dict[str, str]] = [
|
||||
{
|
||||
"name": "CISA Alerts",
|
||||
"url": "https://www.cisa.gov/cybersecurity-advisories/all.xml",
|
||||
},
|
||||
{
|
||||
"name": "NIST NVD CVE",
|
||||
"url": "https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss.xml",
|
||||
},
|
||||
{
|
||||
"name": "SANS ISC",
|
||||
"url": "https://isc.sans.edu/rssfeed.xml",
|
||||
},
|
||||
{
|
||||
"name": "BleepingComputer",
|
||||
"url": "https://www.bleepingcomputer.com/feed/",
|
||||
},
|
||||
{
|
||||
"name": "The Hacker News",
|
||||
"url": "https://feeds.feedburner.com/TheHackersNews",
|
||||
},
|
||||
{
|
||||
"name": "Krebs on Security",
|
||||
"url": "https://krebsonsecurity.com/feed/",
|
||||
},
|
||||
]
|
||||
|
||||
# Timeout for each feed request (seconds)
|
||||
_FEED_TIMEOUT = 15
|
||||
|
||||
# Maximum number of techniques to scan (to keep MVP fast)
|
||||
_MAX_TECHNIQUES = 50
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fetch_feed(url: str) -> list[dict[str, str]]:
|
||||
"""Download and parse an RSS/Atom feed, returning a list of entries.
|
||||
|
||||
Each entry is a dict with keys ``title``, ``link``, and ``description``.
|
||||
Returns an empty list on any error so the scan can continue.
|
||||
"""
|
||||
try:
|
||||
resp = _requests.get(url, timeout=_FEED_TIMEOUT, headers={
|
||||
"User-Agent": "AegisPlatform/1.0 IntelScan",
|
||||
})
|
||||
resp.raise_for_status()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to fetch feed %s: %s", url, exc)
|
||||
return []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(resp.content)
|
||||
except ET.ParseError as exc:
|
||||
logger.warning("Failed to parse feed %s: %s", url, exc)
|
||||
return []
|
||||
|
||||
entries: list[dict[str, str]] = []
|
||||
|
||||
# RSS 2.0 format: <channel><item>...
|
||||
for item in root.iter("item"):
|
||||
title_el = item.find("title")
|
||||
link_el = item.find("link")
|
||||
desc_el = item.find("description")
|
||||
entries.append({
|
||||
"title": title_el.text.strip() if title_el is not None and title_el.text else "",
|
||||
"link": link_el.text.strip() if link_el is not None and link_el.text else "",
|
||||
"description": desc_el.text.strip() if desc_el is not None and desc_el.text else "",
|
||||
})
|
||||
|
||||
# Atom format: <feed><entry>...
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
||||
for entry in root.iter("{http://www.w3.org/2005/Atom}entry"):
|
||||
title_el = entry.find("atom:title", ns)
|
||||
link_el = entry.find("atom:link", ns)
|
||||
summary_el = entry.find("atom:summary", ns)
|
||||
link_href = ""
|
||||
if link_el is not None:
|
||||
link_href = link_el.get("href", "")
|
||||
entries.append({
|
||||
"title": title_el.text.strip() if title_el is not None and title_el.text else "",
|
||||
"link": link_href.strip(),
|
||||
"description": summary_el.text.strip() if summary_el is not None and summary_el.text else "",
|
||||
})
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def _build_patterns(technique: Technique) -> list[re.Pattern]:
|
||||
"""Build regex patterns to search feed content for a given technique."""
|
||||
patterns: list[re.Pattern] = []
|
||||
|
||||
mitre_id = re.escape(technique.mitre_id)
|
||||
patterns.append(re.compile(mitre_id, re.IGNORECASE))
|
||||
|
||||
# Technique name — match if the full name appears
|
||||
if technique.name and len(technique.name) > 4:
|
||||
name_escaped = re.escape(technique.name)
|
||||
patterns.append(re.compile(name_escaped, re.IGNORECASE))
|
||||
|
||||
return patterns
|
||||
|
||||
|
||||
def _entry_matches(entry: dict[str, str], patterns: list[re.Pattern]) -> bool:
|
||||
"""Return True if any pattern matches the entry's title or description."""
|
||||
text = f"{entry.get('title', '')} {entry.get('description', '')}"
|
||||
return any(p.search(text) for p in patterns)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def scan_intel(db: Session) -> dict:
|
||||
"""Run the intel scan across RSS feeds for known techniques.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
db : Session
|
||||
Active SQLAlchemy database session.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
Summary with keys ``new_items``, ``duplicates_skipped``,
|
||||
``techniques_flagged``, ``feeds_checked``.
|
||||
"""
|
||||
logger.info("Intel scan starting...")
|
||||
|
||||
# 1. Load techniques (limit for MVP speed)
|
||||
techniques = (
|
||||
db.query(Technique)
|
||||
.order_by(Technique.mitre_id)
|
||||
.limit(_MAX_TECHNIQUES)
|
||||
.all()
|
||||
)
|
||||
logger.info("Scanning %d techniques against %d feeds", len(techniques), len(RSS_FEEDS))
|
||||
|
||||
# 2. Pre-load all existing intel URLs for dedup
|
||||
existing_urls: set[str] = {
|
||||
row[0] for row in db.query(IntelItem.url).all()
|
||||
}
|
||||
|
||||
# 3. Fetch all feeds once
|
||||
all_entries: list[tuple[str, dict[str, str]]] = [] # (feed_name, entry)
|
||||
feeds_ok = 0
|
||||
for feed in RSS_FEEDS:
|
||||
entries = _fetch_feed(feed["url"])
|
||||
if entries:
|
||||
feeds_ok += 1
|
||||
for entry in entries:
|
||||
all_entries.append((feed["name"], entry))
|
||||
|
||||
logger.info("Fetched %d entries from %d/%d feeds", len(all_entries), feeds_ok, len(RSS_FEEDS))
|
||||
|
||||
# 4. Match entries to techniques
|
||||
new_items = 0
|
||||
duplicates_skipped = 0
|
||||
techniques_flagged: set[str] = set()
|
||||
|
||||
for technique in techniques:
|
||||
patterns = _build_patterns(technique)
|
||||
|
||||
for feed_name, entry in all_entries:
|
||||
if not _entry_matches(entry, patterns):
|
||||
continue
|
||||
|
||||
url = entry.get("link", "").strip()
|
||||
if not url:
|
||||
continue
|
||||
|
||||
# Dedup
|
||||
if url in existing_urls:
|
||||
duplicates_skipped += 1
|
||||
continue
|
||||
|
||||
# Create IntelItem
|
||||
intel_item = IntelItem(
|
||||
technique_id=technique.id,
|
||||
url=url,
|
||||
title=entry.get("title", "")[:500],
|
||||
source=feed_name,
|
||||
detected_at=datetime.utcnow(),
|
||||
reviewed=False,
|
||||
)
|
||||
db.add(intel_item)
|
||||
existing_urls.add(url)
|
||||
new_items += 1
|
||||
|
||||
# Flag technique for review
|
||||
if not technique.review_required:
|
||||
technique.review_required = True
|
||||
techniques_flagged.add(technique.mitre_id)
|
||||
|
||||
# 5. Single commit
|
||||
db.commit()
|
||||
|
||||
summary = {
|
||||
"new_items": new_items,
|
||||
"duplicates_skipped": duplicates_skipped,
|
||||
"techniques_flagged": len(techniques_flagged),
|
||||
"feeds_checked": feeds_ok,
|
||||
}
|
||||
|
||||
logger.info(
|
||||
"Intel scan complete — new=%d, duplicates_skipped=%d, "
|
||||
"techniques_flagged=%d, feeds_checked=%d",
|
||||
new_items, duplicates_skipped, len(techniques_flagged), feeds_ok,
|
||||
)
|
||||
|
||||
# 6. Audit log
|
||||
log_action(
|
||||
db,
|
||||
user_id=None,
|
||||
action="intel_scan",
|
||||
entity_type="intel_item",
|
||||
entity_id=None,
|
||||
details=summary,
|
||||
)
|
||||
|
||||
return summary
|
||||
Reference in New Issue
Block a user