kitos
799 lines
28 KiB
Python
799 lines
28 KiB
Python
"""ATT&CK Evaluations importer — fetches real CrowdStrike detection results
|
|
from MITRE Engenuity's public API and seeds the platform with validated tests.
|
|
|
|
Data source
|
|
-----------
|
|
https://evals.mitre.org/api/
|
|
- /participants/ → list of vendors + rounds they completed
|
|
- /results/?participant=crowdstrike&domain=ENTERPRISE
|
|
→ per-substep detection results per adversary
|
|
|
|
Detection level mapping (MITRE → Aegis)
|
|
---------------------------------------
|
|
Technique / Specific Behavior → detected (correctly identified ATT&CK technique)
|
|
Tactic → partially_detected (behavior noted but not categorized)
|
|
General / IOC / MSSP → partially_detected (anomaly detected, not ATT&CK-mapped)
|
|
Telemetry → partially_detected (raw data only — marginal detection)
|
|
None / N/A → not_detected
|
|
|
|
All imported tests are created in ``in_review`` state so Blue Leads must
|
|
confirm each result before it counts as real coverage for the organisation.
|
|
|
|
Important caveats stored in every test's description
|
|
------------------------------------------------------
|
|
"Source: MITRE ATT&CK Evaluation (Round N — Adversary). Results reflect
|
|
CrowdStrike Falcon in a controlled lab environment, NOT this organisation's
|
|
deployment. Validate detection in your own environment before approving."
|
|
"""
|
|
|
|
import logging
|
|
import re
|
|
import uuid
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
import requests
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.models.enums import TestState, TestResult
|
|
from app.models.evaluation_import import EvaluationImport
|
|
from app.models.technique import Technique
|
|
from app.models.test import Test
|
|
from app.models.user import User
|
|
from app.services.audit_service import log_action
|
|
from app.services.status_service import recalculate_technique_status
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_BASE = "https://evals.mitre.org"
|
|
_TIMEOUT = 30 # seconds per HTTP call
|
|
_VENDOR = "crowdstrike"
|
|
_DOMAIN = "ENTERPRISE"
|
|
|
|
# Browser-like headers to bypass Cloudflare bot protection on evals.mitre.org
|
|
_HEADERS = {
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/124.0.0.0 Safari/537.36"
|
|
),
|
|
"Accept": "application/json, text/plain, */*",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
"Referer": "https://evals.mitre.org/",
|
|
"Origin": "https://evals.mitre.org",
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fallback: hardcoded public CrowdStrike ENTERPRISE rounds
|
|
# Used when evals.mitre.org API is unreachable (Cloudflare 502, outage, etc.)
|
|
#
|
|
# Names use the EXACT slugs the live API returns (hyphens, not underscores).
|
|
# Verified from live API response on 2025-06-05.
|
|
# CrowdStrike did NOT participate in Round 6 (OilRig) — not included.
|
|
# ---------------------------------------------------------------------------
|
|
_FALLBACK_ROUNDS: list[dict[str, Any]] = [
|
|
{
|
|
"name": "apt3",
|
|
"display_name": "APT3",
|
|
"eval_round": 1,
|
|
"domain": "ENTERPRISE",
|
|
"status": "PUBLIC",
|
|
},
|
|
{
|
|
"name": "apt29",
|
|
"display_name": "APT29",
|
|
"eval_round": 2,
|
|
"domain": "ENTERPRISE",
|
|
"status": "PUBLIC",
|
|
},
|
|
{
|
|
"name": "carbanak-fin7",
|
|
"display_name": "Carbanak+FIN7",
|
|
"eval_round": 3,
|
|
"domain": "ENTERPRISE",
|
|
"status": "PUBLIC",
|
|
},
|
|
{
|
|
"name": "wizard-spider-sandworm",
|
|
"display_name": "Wizard Spider + Sandworm",
|
|
"eval_round": 4,
|
|
"domain": "ENTERPRISE",
|
|
"status": "PUBLIC",
|
|
},
|
|
{
|
|
"name": "turla",
|
|
"display_name": "Turla",
|
|
"eval_round": 5,
|
|
"domain": "ENTERPRISE",
|
|
"status": "PUBLIC",
|
|
},
|
|
{
|
|
"name": "er7",
|
|
"display_name": "Enterprise 2025",
|
|
"eval_round": 7,
|
|
"domain": "ENTERPRISE",
|
|
"status": "PUBLIC",
|
|
},
|
|
]
|
|
|
|
# Detection type → quality score (higher = better)
|
|
_DETECTION_SCORE: dict[str, int] = {
|
|
"none": 0,
|
|
"n/a": 0,
|
|
"telemetry": 1,
|
|
"mssp": 2,
|
|
"general": 2,
|
|
"ioc": 2,
|
|
"tactic": 3,
|
|
"technique": 4,
|
|
"specific behavior": 4,
|
|
}
|
|
|
|
|
|
def _score(detection_type: str) -> int:
|
|
key = (detection_type or "").lower().strip()
|
|
for pattern, score in _DETECTION_SCORE.items():
|
|
if pattern in key:
|
|
return score
|
|
return 0
|
|
|
|
|
|
def _score_to_result(score: int) -> TestResult:
|
|
if score >= 4:
|
|
return TestResult.detected
|
|
if score >= 1:
|
|
return TestResult.partially_detected
|
|
return TestResult.not_detected
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public API helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def fetch_rounds_with_status() -> dict[str, Any]:
|
|
"""Fetch CrowdStrike ENTERPRISE rounds and report whether the live API was reachable.
|
|
|
|
Returns::
|
|
|
|
{
|
|
"rounds": [{"name": ..., "display_name": ..., "eval_round": ...}, ...],
|
|
"api_reachable": True | False,
|
|
"api_error": None | "<error message>",
|
|
}
|
|
"""
|
|
try:
|
|
session = requests.Session()
|
|
session.headers.update(_HEADERS)
|
|
resp = session.get(f"{_BASE}/api/participants/", timeout=_TIMEOUT)
|
|
resp.raise_for_status()
|
|
participants = resp.json()
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"evals.mitre.org API unreachable (%s) — using hardcoded fallback round list.",
|
|
exc,
|
|
)
|
|
return {
|
|
"rounds": list(_FALLBACK_ROUNDS),
|
|
"api_reachable": False,
|
|
"api_error": str(exc),
|
|
}
|
|
|
|
crowdstrike = next(
|
|
(p for p in participants if p.get("name", "").lower() == _VENDOR),
|
|
None,
|
|
)
|
|
if not crowdstrike:
|
|
logger.warning("Vendor '%s' not found in live data — using fallback.", _VENDOR)
|
|
return {
|
|
"rounds": list(_FALLBACK_ROUNDS),
|
|
"api_reachable": True, # API was reachable, vendor just wasn't listed
|
|
"api_error": f"Vendor '{_VENDOR}' not found in participants list",
|
|
}
|
|
|
|
rounds = [
|
|
adv
|
|
for adv in crowdstrike.get("adversaries_completed", [])
|
|
if adv.get("domain", "").upper() == _DOMAIN
|
|
and adv.get("status", "").upper() == "PUBLIC"
|
|
]
|
|
rounds.sort(key=lambda x: x.get("eval_round", 0))
|
|
return {
|
|
"rounds": rounds if rounds else list(_FALLBACK_ROUNDS),
|
|
"api_reachable": True,
|
|
"api_error": None,
|
|
}
|
|
|
|
|
|
def fetch_available_rounds() -> list[dict[str, Any]]:
|
|
"""Return all evaluation rounds CrowdStrike has completed (ENTERPRISE only).
|
|
|
|
Each dict has: name, display_name, eval_round.
|
|
Sorted by eval_round ascending.
|
|
|
|
Falls back to ``_FALLBACK_ROUNDS`` if the live API is unreachable.
|
|
"""
|
|
return fetch_rounds_with_status()["rounds"]
|
|
|
|
|
|
def get_latest_round() -> dict[str, Any]:
|
|
"""Return the most recent PUBLIC ENTERPRISE round CrowdStrike participated in."""
|
|
rounds = fetch_available_rounds()
|
|
if not rounds:
|
|
raise ValueError("No public Enterprise evaluation rounds found for CrowdStrike")
|
|
return rounds[-1]
|
|
|
|
|
|
def fetch_results_for_adversary(adversary_name: str) -> list[dict[str, Any]]:
|
|
"""Fetch all per-substep detection results for a specific adversary round.
|
|
|
|
Returns a flat list of substep dicts, each containing:
|
|
technique_id, technique_name, tactic_id, best_score, detection_type, note.
|
|
"""
|
|
url = f"{_BASE}/api/results/?participant={_VENDOR}&domain={_DOMAIN}"
|
|
try:
|
|
session = requests.Session()
|
|
session.headers.update(_HEADERS)
|
|
resp = session.get(url, timeout=_TIMEOUT)
|
|
resp.raise_for_status()
|
|
data = resp.json()
|
|
except Exception as exc:
|
|
logger.error("Failed to fetch ATT&CK Evaluations results: %s", exc)
|
|
raise
|
|
|
|
# The results endpoint returns a LIST of vendor objects:
|
|
# [{"name": "crowdstrike", "adversaries": [{"Adversary_Name": "apt3", ...}, ...]}, ...]
|
|
# (not a dict — hence the explicit vendor lookup below)
|
|
if isinstance(data, list):
|
|
vendor_entry = next(
|
|
(v for v in data if isinstance(v, dict) and v.get("name", "").lower() == _VENDOR),
|
|
None,
|
|
)
|
|
if not vendor_entry:
|
|
raise ValueError(
|
|
f"Vendor '{_VENDOR}' not found in results response. "
|
|
f"Available vendors: {[v.get('name') for v in data if isinstance(v, dict)]}"
|
|
)
|
|
adversaries = vendor_entry.get("adversaries", [])
|
|
else:
|
|
# Fallback for legacy dict-shaped response (just in case API changes again)
|
|
adversaries = data.get("adversaries", [])
|
|
|
|
target = next(
|
|
(a for a in adversaries if a.get("Adversary_Name", "").lower() == adversary_name.lower()),
|
|
None,
|
|
)
|
|
if not target:
|
|
raise ValueError(
|
|
f"Adversary '{adversary_name}' not found in results. "
|
|
f"Available: {[a.get('Adversary_Name') for a in adversaries]}"
|
|
)
|
|
|
|
substeps: list[dict[str, Any]] = []
|
|
|
|
scenarios = target.get("Detections_By_Step", {})
|
|
for scenario_name, scenario_data in scenarios.items():
|
|
for step in scenario_data.get("Steps", []):
|
|
step_num = step.get("Step_Num", "")
|
|
step_name = step.get("Step_Name", "")
|
|
# Strip HTML tags from the Step.Description narrative
|
|
step_desc_raw = step.get("Description") or ""
|
|
step_description = re.sub(r"<[^>]+>", " ", step_desc_raw)
|
|
step_description = re.sub(r"\s+", " ", step_description).strip()
|
|
|
|
for substep in step.get("Substeps", []):
|
|
# Prefer sub-technique over technique
|
|
sub = substep.get("Subtechnique") or {}
|
|
tech = substep.get("Technique") or {}
|
|
tactic = substep.get("Tactic") or {}
|
|
|
|
technique_id = (
|
|
sub.get("Subtechnique_Id")
|
|
or tech.get("Technique_Id")
|
|
or ""
|
|
).strip()
|
|
technique_name = (
|
|
sub.get("Subtechnique_Name")
|
|
or tech.get("Technique_Name")
|
|
or "Unknown"
|
|
).strip()
|
|
|
|
if not technique_id:
|
|
continue
|
|
|
|
detections = substep.get("Detections", [])
|
|
best_score = 0
|
|
best_type = "None"
|
|
best_note = ""
|
|
for det in detections:
|
|
dtype = det.get("Detection_Type", "None")
|
|
s = _score(dtype)
|
|
if s > best_score:
|
|
best_score = s
|
|
best_type = dtype
|
|
best_note = det.get("Detection_Note", "")
|
|
|
|
# Collect all unique data sources from screenshots across all detections
|
|
data_sources: list[str] = sorted({
|
|
src
|
|
for det in detections
|
|
for sc in det.get("Screenshots", [])
|
|
for src in sc.get("Data_Sources", [])
|
|
})
|
|
|
|
substeps.append(
|
|
{
|
|
"technique_id": technique_id,
|
|
"technique_name": technique_name,
|
|
"tactic_id": tactic.get("Tactic_Id", ""),
|
|
"tactic_name": tactic.get("Tactic_Name", ""),
|
|
"best_score": best_score,
|
|
"detection_type": best_type,
|
|
"note": best_note,
|
|
# Enrichment fields from the API
|
|
"scenario_name": scenario_name,
|
|
"step_num": step_num,
|
|
"step_name": step_name,
|
|
"step_description": step_description,
|
|
"substep_ref": substep.get("Substep", ""),
|
|
"criteria": (substep.get("Criteria") or "").strip(),
|
|
"data_sources": data_sources,
|
|
}
|
|
)
|
|
|
|
return substeps
|
|
|
|
|
|
def _aggregate_by_technique(substeps: list[dict]) -> dict[str, dict]:
|
|
"""Aggregate substep results per technique.
|
|
|
|
- Deduplicates substeps by (substep_ref, criteria) — prevents duplicates
|
|
that arise when adversaries with multiple scenarios (e.g. Wizard Spider +
|
|
Sandworm) repeat the same substep across a "combined" replay scenario.
|
|
- Groups unique occurrences by scenario_name so the narrative can show
|
|
"Wizard Spider scenario" vs "Sandworm scenario" separately.
|
|
- Tracks best detection score across all unique substeps.
|
|
"""
|
|
by_technique: dict[str, dict] = {}
|
|
|
|
for sub in substeps:
|
|
tid = sub["technique_id"]
|
|
if tid not in by_technique:
|
|
by_technique[tid] = {
|
|
**sub,
|
|
"occurrences": [], # flat list of unique occurrences
|
|
"_seen_keys": set(), # (substep_ref, criteria) dedup set
|
|
}
|
|
|
|
agg = by_technique[tid]
|
|
|
|
# Deduplication key: same substep_ref + same criteria text = duplicate
|
|
dedup_key = (sub["substep_ref"], sub["criteria"])
|
|
if dedup_key in agg["_seen_keys"]:
|
|
continue
|
|
agg["_seen_keys"].add(dedup_key)
|
|
|
|
agg["occurrences"].append({
|
|
"scenario_name": sub["scenario_name"],
|
|
"substep_ref": sub["substep_ref"],
|
|
"step_num": sub["step_num"],
|
|
"step_name": sub["step_name"],
|
|
"step_description": sub["step_description"],
|
|
"criteria": sub["criteria"],
|
|
"data_sources": sub["data_sources"],
|
|
"detection_type": sub["detection_type"],
|
|
"best_score": sub["best_score"],
|
|
"note": sub["note"],
|
|
})
|
|
|
|
# Promote best detection score
|
|
if sub["best_score"] > agg["best_score"]:
|
|
agg["best_score"] = sub["best_score"]
|
|
agg["detection_type"] = sub["detection_type"]
|
|
agg["note"] = sub["note"]
|
|
agg["tactic_id"] = sub["tactic_id"]
|
|
agg["tactic_name"] = sub["tactic_name"]
|
|
|
|
# Clean up internal dedup sets before returning
|
|
for agg in by_technique.values():
|
|
agg.pop("_seen_keys", None)
|
|
|
|
return by_technique
|
|
|
|
|
|
def _group_occurrences_by_scenario(occurrences: list[dict]) -> dict[str, list[dict]]:
|
|
"""Group a technique's occurrences by scenario, preserving insertion order."""
|
|
grouped: dict[str, list[dict]] = {}
|
|
for occ in occurrences:
|
|
sc = occ.get("scenario_name", "Scenario_1")
|
|
grouped.setdefault(sc, []).append(occ)
|
|
return grouped
|
|
|
|
|
|
def _build_procedure_text(agg: dict, adversary_display: str, eval_round: int) -> str:
|
|
"""Build a rich attack-path narrative for the Test.procedure_text field.
|
|
|
|
Groups substeps by scenario so adversaries with multiple threat groups
|
|
(e.g. Wizard Spider + Sandworm with 3 scenarios) are clearly separated.
|
|
Includes Step.Description narrative for context.
|
|
"""
|
|
occurrences = agg.get("occurrences", [])
|
|
if not occurrences:
|
|
return (
|
|
f"MITRE ATT&CK Evaluation simulation using {adversary_display} TTPs. "
|
|
f"See evaluation report at https://evals.mitre.org for full details."
|
|
)
|
|
|
|
lines: list[str] = [f"ATT&CK Evaluation R{eval_round} — {adversary_display}", ""]
|
|
|
|
grouped = _group_occurrences_by_scenario(occurrences)
|
|
scenario_count = len(grouped)
|
|
|
|
for sc_name, sc_occs in grouped.items():
|
|
# Scenario header — only shown when there are multiple scenarios
|
|
if scenario_count > 1:
|
|
idx = sc_name.replace("Scenario_", "Scenario ")
|
|
lines.append(f"=== {idx} ===")
|
|
|
|
# Within each scenario, group by step to emit description once per step
|
|
seen_step_descs: set = set()
|
|
for occ in sc_occs:
|
|
step_num = occ.get("step_num", "")
|
|
step_name = occ.get("step_name", "")
|
|
step_desc = occ.get("step_description", "")
|
|
# Use (step_num or step_name) as dedup key for descriptions
|
|
step_key = str(step_num) if step_num else step_name
|
|
|
|
if step_key and step_key not in seen_step_descs:
|
|
seen_step_descs.add(step_key)
|
|
header = f"Step {step_num} — {step_name}:" if step_num else f"— {step_name}:"
|
|
lines.append(header)
|
|
if step_desc:
|
|
truncated = step_desc[:450] + ("…" if len(step_desc) > 450 else "")
|
|
lines.append(truncated)
|
|
|
|
ref = occ.get("substep_ref", "")
|
|
criteria = occ.get("criteria", "")
|
|
det = occ.get("detection_type", "")
|
|
if criteria:
|
|
tag = f" [{ref}]" if ref else " •"
|
|
det_tag = f" [{det}]" if det and det.lower() not in ("none", "") else ""
|
|
lines.append(f"{tag}{det_tag} {criteria}")
|
|
|
|
lines.append("")
|
|
|
|
return "\n".join(lines).rstrip()
|
|
|
|
|
|
def _build_description(agg: dict, adversary_display: str, eval_round: int) -> str:
|
|
"""Build Test.description with source metadata, detection guidance and warning.
|
|
|
|
The 'criteria' field from the MITRE API describes what each substep does AND
|
|
what should be detected, so it doubles as blue-team detection guidance.
|
|
"""
|
|
occurrences = agg.get("occurrences", [])
|
|
|
|
# Collect all unique data sources across every unique occurrence
|
|
all_data_sources: list[str] = sorted({
|
|
src
|
|
for occ in occurrences
|
|
for src in occ.get("data_sources", [])
|
|
})
|
|
|
|
header = (
|
|
f"Source: MITRE ATT&CK Evaluation — Round {eval_round} ({adversary_display}).\n"
|
|
f"Vendor: CrowdStrike Falcon.\n"
|
|
f"Detection type achieved: {agg['detection_type']}."
|
|
)
|
|
|
|
ds_section = ""
|
|
if all_data_sources:
|
|
ds_section = "\n\nData sources observed:\n" + "\n".join(
|
|
f" • {ds}" for ds in all_data_sources
|
|
)
|
|
|
|
# Detection guidance — what criteria were observed (blue team can use these as IOCs)
|
|
det_lines: list[str] = []
|
|
grouped = _group_occurrences_by_scenario(occurrences)
|
|
for sc_name, sc_occs in grouped.items():
|
|
scenario_label = f"[{sc_name}] " if len(grouped) > 1 else ""
|
|
for occ in sc_occs:
|
|
ref = occ.get("substep_ref", "")
|
|
step_name = occ.get("step_name", "")
|
|
criteria = occ.get("criteria", "")
|
|
det_type = occ.get("detection_type", "")
|
|
if criteria:
|
|
label = f"[{ref}]" if ref else "•"
|
|
step_label = f" ({step_name})" if step_name else ""
|
|
det_label = f" — {det_type}" if det_type and det_type.lower() not in ("none", "") else ""
|
|
det_lines.append(f" {scenario_label}{label}{step_label}{det_label}: {criteria}")
|
|
|
|
det_section = ""
|
|
if det_lines:
|
|
det_section = "\n\nDetection criteria (what to look for):\n" + "\n".join(det_lines)
|
|
|
|
warning = (
|
|
f"\n\n⚠️ IMPORTANT: These results reflect CrowdStrike Falcon performance in a "
|
|
f"controlled MITRE lab environment against a simulated {adversary_display} "
|
|
f"adversary. They do NOT represent your organisation's actual detection "
|
|
f"capability. Validate in your own environment before approving."
|
|
)
|
|
|
|
note_section = f"\n\nMITRE note: {agg['note']}" if agg.get("note") else ""
|
|
|
|
return header + ds_section + det_section + warning + note_section
|
|
|
|
|
|
def _build_red_summary(agg: dict, adversary_display: str, eval_round: int) -> str:
|
|
"""Build the Red Team summary for the Test.red_summary field."""
|
|
occurrences = agg.get("occurrences", [])
|
|
|
|
lines = [
|
|
f"MITRE ATT&CK Evaluation — Round {eval_round} ({adversary_display})",
|
|
f"Vendor: CrowdStrike Falcon",
|
|
f"Best detection level: {agg['detection_type']}",
|
|
f"Tactic: {agg['tactic_name']} ({agg['tactic_id']})",
|
|
f"Unique substeps: {len(occurrences)}",
|
|
]
|
|
|
|
if occurrences:
|
|
lines.append("")
|
|
grouped = _group_occurrences_by_scenario(occurrences)
|
|
for sc_name, sc_occs in grouped.items():
|
|
if len(grouped) > 1:
|
|
lines.append(f"{sc_name}:")
|
|
for occ in sc_occs:
|
|
ref = occ.get("substep_ref", "")
|
|
criteria = occ.get("criteria", "")
|
|
step_name = occ.get("step_name", "")
|
|
det = occ.get("detection_type", "")
|
|
if criteria:
|
|
tag = f" [{ref}]" if ref else " •"
|
|
step_tag = f" {step_name} —" if step_name else ""
|
|
det_tag = f" [{det}]" if det and det.lower() not in ("none", "") else ""
|
|
lines.append(f"{tag}{step_tag}{det_tag} {criteria}")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main import function
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def import_evaluation_round(
|
|
db: Session,
|
|
adversary_name: str,
|
|
adversary_display: str,
|
|
eval_round: int,
|
|
current_user: User,
|
|
) -> dict[str, Any]:
|
|
"""Import a single ATT&CK Evaluation round for CrowdStrike into the platform.
|
|
|
|
Creates one Test per unique technique with the best detection result
|
|
observed across all substeps for that technique. All tests land in
|
|
``in_review`` state — Blue Leads must confirm before they count as coverage.
|
|
|
|
Returns a summary dict: created, skipped, techniques_covered.
|
|
Raises if the round was already imported (idempotency guard).
|
|
"""
|
|
# Idempotency — refuse duplicate imports
|
|
existing = (
|
|
db.query(EvaluationImport)
|
|
.filter(
|
|
EvaluationImport.adversary_name == adversary_name.lower(),
|
|
EvaluationImport.status == "completed",
|
|
)
|
|
.first()
|
|
)
|
|
if existing:
|
|
raise ValueError(
|
|
f"Round '{adversary_display}' (round {eval_round}) was already imported "
|
|
f"on {existing.imported_at.date()}. Re-import is not allowed."
|
|
)
|
|
|
|
# Fetch and aggregate substep results
|
|
substeps = fetch_results_for_adversary(adversary_name)
|
|
by_technique = _aggregate_by_technique(substeps)
|
|
|
|
created = 0
|
|
skipped = 0
|
|
affected_technique_ids: set = set()
|
|
|
|
for mitre_id, agg in by_technique.items():
|
|
# Look up the technique in our DB
|
|
technique = (
|
|
db.query(Technique)
|
|
.filter(Technique.mitre_id == mitre_id.upper())
|
|
.first()
|
|
)
|
|
if technique is None:
|
|
skipped += 1
|
|
continue
|
|
|
|
detection_result = _score_to_result(agg["best_score"])
|
|
|
|
description = _build_description(agg, adversary_display, eval_round)
|
|
red_summary = _build_red_summary(agg, adversary_display, eval_round)
|
|
procedure_text = _build_procedure_text(agg, adversary_display, eval_round)
|
|
|
|
test = Test(
|
|
technique_id=technique.id,
|
|
name=f"[EVAL R{eval_round}] {adversary_display} — {technique.name}",
|
|
description=description,
|
|
platform=None,
|
|
procedure_text=procedure_text,
|
|
created_by=current_user.id,
|
|
state=TestState.in_review,
|
|
attack_success=True,
|
|
red_summary=red_summary,
|
|
red_validation_status="approved",
|
|
red_validated_by=current_user.id,
|
|
red_validated_at=datetime.utcnow(),
|
|
detection_result=detection_result,
|
|
blue_validation_status=None,
|
|
execution_date=datetime.utcnow(),
|
|
created_at=datetime.utcnow(),
|
|
)
|
|
db.add(test)
|
|
db.flush()
|
|
|
|
log_action(
|
|
db,
|
|
user_id=current_user.id,
|
|
action="eval_import_test",
|
|
entity_type="test",
|
|
entity_id=test.id,
|
|
details={
|
|
"adversary": adversary_name,
|
|
"eval_round": eval_round,
|
|
"mitre_id": mitre_id,
|
|
"detection_type": agg["detection_type"],
|
|
},
|
|
)
|
|
|
|
affected_technique_ids.add(technique.id)
|
|
created += 1
|
|
|
|
# Recalculate coverage for all touched techniques
|
|
for tech_id in affected_technique_ids:
|
|
tech = db.query(Technique).filter(Technique.id == tech_id).first()
|
|
if tech:
|
|
recalculate_technique_status(db, tech)
|
|
|
|
# Record the import
|
|
record = EvaluationImport(
|
|
id=uuid.uuid4(),
|
|
adversary_name=adversary_name.lower(),
|
|
adversary_display=adversary_display,
|
|
eval_round=eval_round,
|
|
imported_at=datetime.utcnow(),
|
|
imported_by=current_user.id,
|
|
tests_created=created,
|
|
techniques_covered=len(affected_technique_ids),
|
|
status="completed",
|
|
notes=f"Skipped {skipped} techniques not found in local DB.",
|
|
)
|
|
db.add(record)
|
|
db.commit()
|
|
|
|
logger.info(
|
|
"ATT&CK Evaluation import complete — round %d (%s): %d tests created, %d skipped",
|
|
eval_round, adversary_display, created, skipped,
|
|
)
|
|
return {
|
|
"created": created,
|
|
"skipped": skipped,
|
|
"techniques_covered": len(affected_technique_ids),
|
|
"adversary": adversary_display,
|
|
"eval_round": eval_round,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# New-round check (used by the weekly scheduler)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def check_for_new_round(db: Session) -> dict[str, Any]:
|
|
"""Check if a new evaluation round is available that hasn't been imported yet.
|
|
|
|
Returns:
|
|
{"new_round_available": bool, "latest_round": dict | None, "already_imported": bool}
|
|
"""
|
|
try:
|
|
latest = get_latest_round()
|
|
except Exception as exc:
|
|
logger.warning("Could not check for new ATT&CK Evaluation round: %s", exc)
|
|
return {"new_round_available": False, "latest_round": None, "error": str(exc)}
|
|
|
|
already = (
|
|
db.query(EvaluationImport)
|
|
.filter(
|
|
EvaluationImport.adversary_name == latest["name"].lower(),
|
|
EvaluationImport.status == "completed",
|
|
)
|
|
.first()
|
|
)
|
|
|
|
return {
|
|
"new_round_available": already is None,
|
|
"already_imported": already is not None,
|
|
"latest_round": {
|
|
"name": latest["name"],
|
|
"display_name": latest.get("display_name", latest["name"]),
|
|
"eval_round": latest["eval_round"],
|
|
},
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Re-enrich existing tests with richer API data
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def re_enrich_evaluation_round(
|
|
db: Session,
|
|
adversary_name: str,
|
|
adversary_display: str,
|
|
eval_round: int,
|
|
current_user: User,
|
|
) -> dict[str, Any]:
|
|
"""Update procedure_text / description / red_summary on already-imported tests
|
|
for a given round using the enriched API data (attack path, criteria, data sources).
|
|
|
|
This is non-destructive — it only updates the three narrative fields and does
|
|
not change detection results, state, or validation status.
|
|
"""
|
|
# Fetch & aggregate (same flow as import)
|
|
substeps = fetch_results_for_adversary(adversary_name)
|
|
by_technique = _aggregate_by_technique(substeps)
|
|
|
|
updated = 0
|
|
skipped = 0
|
|
|
|
for mitre_id, agg in by_technique.items():
|
|
technique = (
|
|
db.query(Technique)
|
|
.filter(Technique.mitre_id == mitre_id.upper())
|
|
.first()
|
|
)
|
|
if technique is None:
|
|
skipped += 1
|
|
continue
|
|
|
|
# Find the existing test for this round + technique
|
|
existing_test = (
|
|
db.query(Test)
|
|
.filter(
|
|
Test.technique_id == technique.id,
|
|
Test.name.like(f"[EVAL R{eval_round}]%"),
|
|
)
|
|
.first()
|
|
)
|
|
if not existing_test:
|
|
skipped += 1
|
|
continue
|
|
|
|
existing_test.description = _build_description(agg, adversary_display, eval_round)
|
|
existing_test.red_summary = _build_red_summary(agg, adversary_display, eval_round)
|
|
existing_test.procedure_text = _build_procedure_text(agg, adversary_display, eval_round)
|
|
updated += 1
|
|
|
|
db.commit()
|
|
|
|
logger.info(
|
|
"Re-enrichment complete — round %d (%s): %d tests updated, %d skipped",
|
|
eval_round, adversary_display, updated, skipped,
|
|
)
|
|
return {
|
|
"updated": updated,
|
|
"skipped": skipped,
|
|
"adversary": adversary_display,
|
|
"eval_round": eval_round,
|
|
"message": (
|
|
f"Re-enriched {updated} tests for {adversary_display} (Round {eval_round}) "
|
|
f"with attack path, criteria and data sources from MITRE API."
|
|
),
|
|
}
|