"""ATT&CK Evaluations importer — fetches real CrowdStrike detection results from MITRE Engenuity's public API and seeds the platform with validated tests. Data source ----------- https://evals.mitre.org/api/ - /participants/ → list of vendors + rounds they completed - /results/?participant=crowdstrike&domain=ENTERPRISE → per-substep detection results per adversary Detection level mapping (MITRE → Aegis) --------------------------------------- Technique / Specific Behavior → detected (correctly identified ATT&CK technique) Tactic → partially_detected (behavior noted but not categorized) General / IOC / MSSP → partially_detected (anomaly detected, not ATT&CK-mapped) Telemetry → partially_detected (raw data only — marginal detection) None / N/A → not_detected All imported tests are created in ``in_review`` state so Blue Leads must confirm each result before it counts as real coverage for the organisation. Important caveats stored in every test's description ------------------------------------------------------ "Source: MITRE ATT&CK Evaluation (Round N — Adversary). Results reflect CrowdStrike Falcon in a controlled lab environment, NOT this organisation's deployment. Validate detection in your own environment before approving." """ import logging import re import uuid from datetime import datetime from typing import Any import requests from sqlalchemy.orm import Session from app.models.enums import TestState, TestResult from app.models.evaluation_import import EvaluationImport from app.models.technique import Technique from app.models.test import Test from app.models.user import User from app.services.audit_service import log_action from app.services.status_service import recalculate_technique_status logger = logging.getLogger(__name__) _BASE = "https://evals.mitre.org" _TIMEOUT = 30 # seconds per HTTP call _VENDOR = "crowdstrike" _DOMAIN = "ENTERPRISE" # Browser-like headers to bypass Cloudflare bot protection on evals.mitre.org _HEADERS = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/124.0.0.0 Safari/537.36" ), "Accept": "application/json, text/plain, */*", "Accept-Language": "en-US,en;q=0.9", "Referer": "https://evals.mitre.org/", "Origin": "https://evals.mitre.org", } # --------------------------------------------------------------------------- # Fallback: hardcoded public CrowdStrike ENTERPRISE rounds # Used when evals.mitre.org API is unreachable (Cloudflare 502, outage, etc.) # # Names use the EXACT slugs the live API returns (hyphens, not underscores). # Verified from live API response on 2025-06-05. # CrowdStrike did NOT participate in Round 6 (OilRig) — not included. # --------------------------------------------------------------------------- _FALLBACK_ROUNDS: list[dict[str, Any]] = [ { "name": "apt3", "display_name": "APT3", "eval_round": 1, "domain": "ENTERPRISE", "status": "PUBLIC", }, { "name": "apt29", "display_name": "APT29", "eval_round": 2, "domain": "ENTERPRISE", "status": "PUBLIC", }, { "name": "carbanak-fin7", "display_name": "Carbanak+FIN7", "eval_round": 3, "domain": "ENTERPRISE", "status": "PUBLIC", }, { "name": "wizard-spider-sandworm", "display_name": "Wizard Spider + Sandworm", "eval_round": 4, "domain": "ENTERPRISE", "status": "PUBLIC", }, { "name": "turla", "display_name": "Turla", "eval_round": 5, "domain": "ENTERPRISE", "status": "PUBLIC", }, { "name": "er7", "display_name": "Enterprise 2025", "eval_round": 7, "domain": "ENTERPRISE", "status": "PUBLIC", }, ] # Detection type → quality score (higher = better) _DETECTION_SCORE: dict[str, int] = { "none": 0, "n/a": 0, "telemetry": 1, "mssp": 2, "general": 2, "ioc": 2, "tactic": 3, "technique": 4, "specific behavior": 4, } def _score(detection_type: str) -> int: key = (detection_type or "").lower().strip() for pattern, score in _DETECTION_SCORE.items(): if pattern in key: return score return 0 def _score_to_result(score: int) -> TestResult: if score >= 4: return TestResult.detected if score >= 1: return TestResult.partially_detected return TestResult.not_detected # --------------------------------------------------------------------------- # Public API helpers # --------------------------------------------------------------------------- def fetch_rounds_with_status() -> dict[str, Any]: """Fetch CrowdStrike ENTERPRISE rounds and report whether the live API was reachable. Returns:: { "rounds": [{"name": ..., "display_name": ..., "eval_round": ...}, ...], "api_reachable": True | False, "api_error": None | "", } """ try: session = requests.Session() session.headers.update(_HEADERS) resp = session.get(f"{_BASE}/api/participants/", timeout=_TIMEOUT) resp.raise_for_status() participants = resp.json() except Exception as exc: logger.warning( "evals.mitre.org API unreachable (%s) — using hardcoded fallback round list.", exc, ) return { "rounds": list(_FALLBACK_ROUNDS), "api_reachable": False, "api_error": str(exc), } crowdstrike = next( (p for p in participants if p.get("name", "").lower() == _VENDOR), None, ) if not crowdstrike: logger.warning("Vendor '%s' not found in live data — using fallback.", _VENDOR) return { "rounds": list(_FALLBACK_ROUNDS), "api_reachable": True, # API was reachable, vendor just wasn't listed "api_error": f"Vendor '{_VENDOR}' not found in participants list", } rounds = [ adv for adv in crowdstrike.get("adversaries_completed", []) if adv.get("domain", "").upper() == _DOMAIN and adv.get("status", "").upper() == "PUBLIC" ] rounds.sort(key=lambda x: x.get("eval_round", 0)) return { "rounds": rounds if rounds else list(_FALLBACK_ROUNDS), "api_reachable": True, "api_error": None, } def fetch_available_rounds() -> list[dict[str, Any]]: """Return all evaluation rounds CrowdStrike has completed (ENTERPRISE only). Each dict has: name, display_name, eval_round. Sorted by eval_round ascending. Falls back to ``_FALLBACK_ROUNDS`` if the live API is unreachable. """ return fetch_rounds_with_status()["rounds"] def get_latest_round() -> dict[str, Any]: """Return the most recent PUBLIC ENTERPRISE round CrowdStrike participated in.""" rounds = fetch_available_rounds() if not rounds: raise ValueError("No public Enterprise evaluation rounds found for CrowdStrike") return rounds[-1] def fetch_results_for_adversary(adversary_name: str) -> list[dict[str, Any]]: """Fetch all per-substep detection results for a specific adversary round. Returns a flat list of substep dicts, each containing: technique_id, technique_name, tactic_id, best_score, detection_type, note. """ url = f"{_BASE}/api/results/?participant={_VENDOR}&domain={_DOMAIN}" try: session = requests.Session() session.headers.update(_HEADERS) resp = session.get(url, timeout=_TIMEOUT) resp.raise_for_status() data = resp.json() except Exception as exc: logger.error("Failed to fetch ATT&CK Evaluations results: %s", exc) raise # The results endpoint returns a LIST of vendor objects: # [{"name": "crowdstrike", "adversaries": [{"Adversary_Name": "apt3", ...}, ...]}, ...] # (not a dict — hence the explicit vendor lookup below) if isinstance(data, list): vendor_entry = next( (v for v in data if isinstance(v, dict) and v.get("name", "").lower() == _VENDOR), None, ) if not vendor_entry: raise ValueError( f"Vendor '{_VENDOR}' not found in results response. " f"Available vendors: {[v.get('name') for v in data if isinstance(v, dict)]}" ) adversaries = vendor_entry.get("adversaries", []) else: # Fallback for legacy dict-shaped response (just in case API changes again) adversaries = data.get("adversaries", []) target = next( (a for a in adversaries if a.get("Adversary_Name", "").lower() == adversary_name.lower()), None, ) if not target: raise ValueError( f"Adversary '{adversary_name}' not found in results. " f"Available: {[a.get('Adversary_Name') for a in adversaries]}" ) substeps: list[dict[str, Any]] = [] scenarios = target.get("Detections_By_Step", {}) for scenario_name, scenario_data in scenarios.items(): for step in scenario_data.get("Steps", []): step_num = step.get("Step_Num", "") step_name = step.get("Step_Name", "") # Strip HTML tags from the Step.Description narrative step_desc_raw = step.get("Description") or "" step_description = re.sub(r"<[^>]+>", " ", step_desc_raw) step_description = re.sub(r"\s+", " ", step_description).strip() for substep in step.get("Substeps", []): # Prefer sub-technique over technique sub = substep.get("Subtechnique") or {} tech = substep.get("Technique") or {} tactic = substep.get("Tactic") or {} technique_id = ( sub.get("Subtechnique_Id") or tech.get("Technique_Id") or "" ).strip() technique_name = ( sub.get("Subtechnique_Name") or tech.get("Technique_Name") or "Unknown" ).strip() if not technique_id: continue detections = substep.get("Detections", []) best_score = 0 best_type = "None" best_note = "" for det in detections: dtype = det.get("Detection_Type", "None") s = _score(dtype) if s > best_score: best_score = s best_type = dtype best_note = det.get("Detection_Note", "") # Collect all unique data sources from screenshots across all detections data_sources: list[str] = sorted({ src for det in detections for sc in det.get("Screenshots", []) for src in sc.get("Data_Sources", []) }) substeps.append( { "technique_id": technique_id, "technique_name": technique_name, "tactic_id": tactic.get("Tactic_Id", ""), "tactic_name": tactic.get("Tactic_Name", ""), "best_score": best_score, "detection_type": best_type, "note": best_note, # Enrichment fields from the API "scenario_name": scenario_name, "step_num": step_num, "step_name": step_name, "step_description": step_description, "substep_ref": substep.get("Substep", ""), "criteria": (substep.get("Criteria") or "").strip(), "data_sources": data_sources, } ) return substeps def _aggregate_by_technique(substeps: list[dict]) -> dict[str, dict]: """Aggregate substep results per technique. - Deduplicates substeps by (substep_ref, criteria) — prevents duplicates that arise when adversaries with multiple scenarios (e.g. Wizard Spider + Sandworm) repeat the same substep across a "combined" replay scenario. - Groups unique occurrences by scenario_name so the narrative can show "Wizard Spider scenario" vs "Sandworm scenario" separately. - Tracks best detection score across all unique substeps. """ by_technique: dict[str, dict] = {} for sub in substeps: tid = sub["technique_id"] if tid not in by_technique: by_technique[tid] = { **sub, "occurrences": [], # flat list of unique occurrences "_seen_keys": set(), # (substep_ref, criteria) dedup set } agg = by_technique[tid] # Deduplication key: same substep_ref + same criteria text = duplicate dedup_key = (sub["substep_ref"], sub["criteria"]) if dedup_key in agg["_seen_keys"]: continue agg["_seen_keys"].add(dedup_key) agg["occurrences"].append({ "scenario_name": sub["scenario_name"], "substep_ref": sub["substep_ref"], "step_num": sub["step_num"], "step_name": sub["step_name"], "step_description": sub["step_description"], "criteria": sub["criteria"], "data_sources": sub["data_sources"], "detection_type": sub["detection_type"], "best_score": sub["best_score"], "note": sub["note"], }) # Promote best detection score if sub["best_score"] > agg["best_score"]: agg["best_score"] = sub["best_score"] agg["detection_type"] = sub["detection_type"] agg["note"] = sub["note"] agg["tactic_id"] = sub["tactic_id"] agg["tactic_name"] = sub["tactic_name"] # Clean up internal dedup sets before returning for agg in by_technique.values(): agg.pop("_seen_keys", None) return by_technique def _group_occurrences_by_scenario(occurrences: list[dict]) -> dict[str, list[dict]]: """Group a technique's occurrences by scenario, preserving insertion order.""" grouped: dict[str, list[dict]] = {} for occ in occurrences: sc = occ.get("scenario_name", "Scenario_1") grouped.setdefault(sc, []).append(occ) return grouped def _build_procedure_text(agg: dict, adversary_display: str, eval_round: int) -> str: """Build a rich attack-path narrative for the Test.procedure_text field. Groups substeps by scenario so adversaries with multiple threat groups (e.g. Wizard Spider + Sandworm with 3 scenarios) are clearly separated. Includes Step.Description narrative for context. """ occurrences = agg.get("occurrences", []) if not occurrences: return ( f"MITRE ATT&CK Evaluation simulation using {adversary_display} TTPs. " f"See evaluation report at https://evals.mitre.org for full details." ) lines: list[str] = [f"ATT&CK Evaluation R{eval_round} — {adversary_display}", ""] grouped = _group_occurrences_by_scenario(occurrences) scenario_count = len(grouped) for sc_name, sc_occs in grouped.items(): # Scenario header — only shown when there are multiple scenarios if scenario_count > 1: idx = sc_name.replace("Scenario_", "Scenario ") lines.append(f"=== {idx} ===") # Within each scenario, group by step to emit description once per step seen_step_descs: set = set() for occ in sc_occs: step_num = occ.get("step_num", "") step_name = occ.get("step_name", "") step_desc = occ.get("step_description", "") # Use (step_num or step_name) as dedup key for descriptions step_key = str(step_num) if step_num else step_name if step_key and step_key not in seen_step_descs: seen_step_descs.add(step_key) header = f"Step {step_num} — {step_name}:" if step_num else f"— {step_name}:" lines.append(header) if step_desc: truncated = step_desc[:450] + ("…" if len(step_desc) > 450 else "") lines.append(truncated) ref = occ.get("substep_ref", "") criteria = occ.get("criteria", "") det = occ.get("detection_type", "") if criteria: tag = f" [{ref}]" if ref else " •" det_tag = f" [{det}]" if det and det.lower() not in ("none", "") else "" lines.append(f"{tag}{det_tag} {criteria}") lines.append("") return "\n".join(lines).rstrip() def _build_description(agg: dict, adversary_display: str, eval_round: int) -> str: """Build Test.description with source metadata, detection guidance and warning. The 'criteria' field from the MITRE API describes what each substep does AND what should be detected, so it doubles as blue-team detection guidance. """ occurrences = agg.get("occurrences", []) # Collect all unique data sources across every unique occurrence all_data_sources: list[str] = sorted({ src for occ in occurrences for src in occ.get("data_sources", []) }) header = ( f"Source: MITRE ATT&CK Evaluation — Round {eval_round} ({adversary_display}).\n" f"Vendor: CrowdStrike Falcon.\n" f"Detection type achieved: {agg['detection_type']}." ) ds_section = "" if all_data_sources: ds_section = "\n\nData sources observed:\n" + "\n".join( f" • {ds}" for ds in all_data_sources ) # Detection guidance — what criteria were observed (blue team can use these as IOCs) det_lines: list[str] = [] grouped = _group_occurrences_by_scenario(occurrences) for sc_name, sc_occs in grouped.items(): scenario_label = f"[{sc_name}] " if len(grouped) > 1 else "" for occ in sc_occs: ref = occ.get("substep_ref", "") step_name = occ.get("step_name", "") criteria = occ.get("criteria", "") det_type = occ.get("detection_type", "") if criteria: label = f"[{ref}]" if ref else "•" step_label = f" ({step_name})" if step_name else "" det_label = f" — {det_type}" if det_type and det_type.lower() not in ("none", "") else "" det_lines.append(f" {scenario_label}{label}{step_label}{det_label}: {criteria}") det_section = "" if det_lines: det_section = "\n\nDetection criteria (what to look for):\n" + "\n".join(det_lines) warning = ( f"\n\n⚠️ IMPORTANT: These results reflect CrowdStrike Falcon performance in a " f"controlled MITRE lab environment against a simulated {adversary_display} " f"adversary. They do NOT represent your organisation's actual detection " f"capability. Validate in your own environment before approving." ) note_section = f"\n\nMITRE note: {agg['note']}" if agg.get("note") else "" return header + ds_section + det_section + warning + note_section def _build_red_summary(agg: dict, adversary_display: str, eval_round: int) -> str: """Build the Red Team summary for the Test.red_summary field.""" occurrences = agg.get("occurrences", []) lines = [ f"MITRE ATT&CK Evaluation — Round {eval_round} ({adversary_display})", "Vendor: CrowdStrike Falcon", f"Best detection level: {agg['detection_type']}", f"Tactic: {agg['tactic_name']} ({agg['tactic_id']})", f"Unique substeps: {len(occurrences)}", ] if occurrences: lines.append("") grouped = _group_occurrences_by_scenario(occurrences) for sc_name, sc_occs in grouped.items(): if len(grouped) > 1: lines.append(f"{sc_name}:") for occ in sc_occs: ref = occ.get("substep_ref", "") criteria = occ.get("criteria", "") step_name = occ.get("step_name", "") det = occ.get("detection_type", "") if criteria: tag = f" [{ref}]" if ref else " •" step_tag = f" {step_name} —" if step_name else "" det_tag = f" [{det}]" if det and det.lower() not in ("none", "") else "" lines.append(f"{tag}{step_tag}{det_tag} {criteria}") return "\n".join(lines) # --------------------------------------------------------------------------- # Main import function # --------------------------------------------------------------------------- def import_evaluation_round( db: Session, adversary_name: str, adversary_display: str, eval_round: int, current_user: User, ) -> dict[str, Any]: """Import a single ATT&CK Evaluation round for CrowdStrike into the platform. Creates one Test per unique technique with the best detection result observed across all substeps for that technique. All tests land in ``in_review`` state — Blue Leads must confirm before they count as coverage. Returns a summary dict: created, skipped, techniques_covered. Raises if the round was already imported (idempotency guard). """ # Idempotency — refuse duplicate imports existing = ( db.query(EvaluationImport) .filter( EvaluationImport.adversary_name == adversary_name.lower(), EvaluationImport.status == "completed", ) .first() ) if existing: raise ValueError( f"Round '{adversary_display}' (round {eval_round}) was already imported " f"on {existing.imported_at.date()}. Re-import is not allowed." ) # Fetch and aggregate substep results substeps = fetch_results_for_adversary(adversary_name) by_technique = _aggregate_by_technique(substeps) created = 0 skipped = 0 affected_technique_ids: set = set() for mitre_id, agg in by_technique.items(): # Look up the technique in our DB technique = ( db.query(Technique) .filter(Technique.mitre_id == mitre_id.upper()) .first() ) if technique is None: skipped += 1 continue detection_result = _score_to_result(agg["best_score"]) description = _build_description(agg, adversary_display, eval_round) red_summary = _build_red_summary(agg, adversary_display, eval_round) procedure_text = _build_procedure_text(agg, adversary_display, eval_round) test = Test( technique_id=technique.id, name=f"[EVAL R{eval_round}] {adversary_display} — {technique.name}", description=description, platform=None, procedure_text=procedure_text, created_by=current_user.id, state=TestState.in_review, attack_success=True, red_summary=red_summary, red_validation_status="approved", red_validated_by=current_user.id, red_validated_at=datetime.utcnow(), detection_result=detection_result, blue_validation_status=None, execution_date=datetime.utcnow(), created_at=datetime.utcnow(), ) db.add(test) db.flush() log_action( db, user_id=current_user.id, action="eval_import_test", entity_type="test", entity_id=test.id, details={ "adversary": adversary_name, "eval_round": eval_round, "mitre_id": mitre_id, "detection_type": agg["detection_type"], }, ) affected_technique_ids.add(technique.id) created += 1 # Recalculate coverage for all touched techniques for tech_id in affected_technique_ids: tech = db.query(Technique).filter(Technique.id == tech_id).first() if tech: recalculate_technique_status(db, tech) # Record the import record = EvaluationImport( id=uuid.uuid4(), adversary_name=adversary_name.lower(), adversary_display=adversary_display, eval_round=eval_round, imported_at=datetime.utcnow(), imported_by=current_user.id, tests_created=created, techniques_covered=len(affected_technique_ids), status="completed", notes=f"Skipped {skipped} techniques not found in local DB.", ) db.add(record) db.commit() logger.info( "ATT&CK Evaluation import complete — round %d (%s): %d tests created, %d skipped", eval_round, adversary_display, created, skipped, ) return { "created": created, "skipped": skipped, "techniques_covered": len(affected_technique_ids), "adversary": adversary_display, "eval_round": eval_round, } # --------------------------------------------------------------------------- # New-round check (used by the weekly scheduler) # --------------------------------------------------------------------------- def check_for_new_round(db: Session) -> dict[str, Any]: """Check if a new evaluation round is available that hasn't been imported yet. Returns: {"new_round_available": bool, "latest_round": dict | None, "already_imported": bool} """ try: latest = get_latest_round() except Exception as exc: logger.warning("Could not check for new ATT&CK Evaluation round: %s", exc) return {"new_round_available": False, "latest_round": None, "error": str(exc)} already = ( db.query(EvaluationImport) .filter( EvaluationImport.adversary_name == latest["name"].lower(), EvaluationImport.status == "completed", ) .first() ) return { "new_round_available": already is None, "already_imported": already is not None, "latest_round": { "name": latest["name"], "display_name": latest.get("display_name", latest["name"]), "eval_round": latest["eval_round"], }, } # --------------------------------------------------------------------------- # Re-enrich existing tests with richer API data # --------------------------------------------------------------------------- def re_enrich_evaluation_round( db: Session, adversary_name: str, adversary_display: str, eval_round: int, current_user: User, ) -> dict[str, Any]: """Update procedure_text / description / red_summary on already-imported tests for a given round using the enriched API data (attack path, criteria, data sources). This is non-destructive — it only updates the three narrative fields and does not change detection results, state, or validation status. """ # Fetch & aggregate (same flow as import) substeps = fetch_results_for_adversary(adversary_name) by_technique = _aggregate_by_technique(substeps) updated = 0 skipped = 0 for mitre_id, agg in by_technique.items(): technique = ( db.query(Technique) .filter(Technique.mitre_id == mitre_id.upper()) .first() ) if technique is None: skipped += 1 continue # Find the existing test for this round + technique existing_test = ( db.query(Test) .filter( Test.technique_id == technique.id, Test.name.like(f"[EVAL R{eval_round}]%"), ) .first() ) if not existing_test: skipped += 1 continue existing_test.description = _build_description(agg, adversary_display, eval_round) existing_test.red_summary = _build_red_summary(agg, adversary_display, eval_round) existing_test.procedure_text = _build_procedure_text(agg, adversary_display, eval_round) updated += 1 db.commit() logger.info( "Re-enrichment complete — round %d (%s): %d tests updated, %d skipped", eval_round, adversary_display, updated, skipped, ) return { "updated": updated, "skipped": skipped, "adversary": adversary_display, "eval_round": eval_round, "message": ( f"Re-enriched {updated} tests for {adversary_display} (Round {eval_round}) " f"with attack path, criteria and data sources from MITRE API." ), }