feat(evaluations): ATT&CK Evaluations importer for CrowdStrike Falcon [FASE-6.1]
Some checks failed
Aegis CI / lint-and-test (push) Has been cancelled

- Migration b048: evaluation_imports table (adversary, round, status, tests_created)
- EvaluationImport SQLAlchemy model
- attck_evaluations_service: fetch rounds from evals.mitre.org API, import per-technique
  detection results (Technique/Tactic/Telemetry -> detected/partially/not_detected)
- All imported tests land in in_review state with lab-environment disclaimer
- Idempotency guard prevents duplicate round imports
- 4 new endpoints: list rounds, import specific, import latest, check-new
- Weekly APScheduler cron (Mon 06:00) auto-checks and imports new rounds
- SystemPage UI: rounds table, import buttons, check-new, result feedback
- Disclaimer callout reminding admins these are lab results not org coverage

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
kitos
2026-06-05 15:57:03 +02:00
parent cfc48ccd2b
commit e3e79be35a
7 changed files with 1067 additions and 1 deletions

View File

@@ -0,0 +1,39 @@
"""Add evaluation_imports table.
Revision ID: b048
Revises: b047
Create Date: 2026-06-05
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import UUID
revision = "b048"
down_revision = "b047"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"evaluation_imports",
sa.Column("id", UUID(as_uuid=True), primary_key=True),
sa.Column("adversary_name", sa.String, nullable=False),
sa.Column("adversary_display", sa.String, nullable=False),
sa.Column("eval_round", sa.Integer, nullable=False),
sa.Column("imported_at", sa.DateTime, nullable=False),
sa.Column("imported_by", UUID(as_uuid=True), sa.ForeignKey("users.id"), nullable=True),
sa.Column("tests_created", sa.Integer, default=0),
sa.Column("techniques_covered", sa.Integer, default=0),
sa.Column("status", sa.String, default="completed"),
sa.Column("notes", sa.Text, nullable=True),
)
op.create_index("ix_evaluation_imports_adversary", "evaluation_imports", ["adversary_name"])
op.create_index("ix_evaluation_imports_round", "evaluation_imports", ["eval_round"])
def downgrade() -> None:
op.drop_index("ix_evaluation_imports_round", table_name="evaluation_imports")
op.drop_index("ix_evaluation_imports_adversary", table_name="evaluation_imports")
op.drop_table("evaluation_imports")

View File

@@ -204,6 +204,83 @@ def _run_intel_scan() -> None:
db.close() db.close()
def _run_evaluation_round_check() -> None:
"""Weekly job: check if a new ATT&CK Evaluation round is available.
If a new round is found it is imported automatically and an admin
notification is created so the team knows new baseline data is available.
"""
logger.info("ATT&CK Evaluations new-round check starting...")
db = SessionLocal()
try:
from app.services.attck_evaluations_service import check_for_new_round, import_evaluation_round
from app.models.user import User as UserModel
result = check_for_new_round(db)
if result.get("error"):
logger.warning("ATT&CK Evaluations check failed: %s", result["error"])
return
if not result.get("new_round_available"):
logger.info(
"ATT&CK Evaluations check — latest round '%s' already imported",
result.get("latest_round", {}).get("display_name", "?"),
)
return
latest = result["latest_round"]
logger.info(
"New ATT&CK Evaluation round detected: %s (round %d) — starting auto-import",
latest["display_name"], latest["eval_round"],
)
# Use the first admin user as the importer (system action)
admin = db.query(UserModel).filter(UserModel.role == "admin").first()
if not admin:
logger.warning("ATT&CK Evaluations auto-import: no admin user found — skipping")
return
summary = import_evaluation_round(
db,
latest["name"],
latest["display_name"],
latest["eval_round"],
admin,
)
logger.info(
"ATT&CK Evaluations auto-import complete — round %d (%s): %d tests created",
latest["eval_round"], latest["display_name"], summary["created"],
)
# Notify all admins
try:
from app.services.notification_service import create_notification
admins = db.query(UserModel).filter(UserModel.role == "admin").all()
for adm in admins:
create_notification(
db,
user_id=adm.id,
title="New ATT&CK Evaluation round imported",
message=(
f"Round {latest['eval_round']}{latest['display_name']}"
f"has been automatically imported. "
f"{summary['created']} tests created in In Review state. "
f"Blue Leads must validate each result before it counts as coverage."
),
notification_type="eval_import",
entity_type="evaluation",
entity_id=None,
)
db.commit()
except Exception:
logger.warning("Failed to send eval import notifications", exc_info=True)
except Exception:
logger.exception("ATT&CK Evaluations round check job failed")
finally:
db.close()
def _run_osint_enrichment() -> None: def _run_osint_enrichment() -> None:
"""Execute weekly OSINT enrichment inside its own DB session.""" """Execute weekly OSINT enrichment inside its own DB session."""
logger.info("Scheduled OSINT enrichment job starting...") logger.info("Scheduled OSINT enrichment job starting...")
@@ -463,6 +540,16 @@ def start_scheduler() -> None:
name="Operational alert evaluation (hourly)", name="Operational alert evaluation (hourly)",
replace_existing=True, replace_existing=True,
) )
scheduler.add_job(
_run_evaluation_round_check,
trigger="cron",
day_of_week="mon",
hour=6,
minute=0,
id="attck_evaluation_check",
name="ATT&CK Evaluations new-round check (Mondays 06:00)",
replace_existing=True,
)
scheduler.start() scheduler.start()
logger.info( logger.info(
"Background scheduler started — mitre_sync (24h), intel_scan (7d), " "Background scheduler started — mitre_sync (24h), intel_scan (7d), "
@@ -470,5 +557,5 @@ def start_scheduler() -> None:
"recurring_campaigns (daily), jira_sync (1h), " "recurring_campaigns (daily), jira_sync (1h), "
"osint_enrichment (weekly), stale_detection (daily), " "osint_enrichment (weekly), stale_detection (daily), "
"retention_policies (daily), data_sources_sync (6h), " "retention_policies (daily), data_sources_sync (6h), "
"alert_evaluation (1h)" "alert_evaluation (1h), attck_evaluation_check (Mondays 06:00)"
) )

View File

@@ -0,0 +1,34 @@
"""SQLAlchemy model for tracking imported ATT&CK Evaluation rounds."""
import uuid
from datetime import datetime
from sqlalchemy import Column, String, Integer, DateTime, Text, ForeignKey, Index
from sqlalchemy.dialects.postgresql import UUID
from app.database import Base
class EvaluationImport(Base):
"""Tracks which ATT&CK Evaluation rounds have been imported into the platform.
Each row represents one vendor+adversary combination that has been processed
and turned into Test records. Used to avoid duplicate imports and to show
the admin panel which rounds are available vs imported.
"""
__tablename__ = "evaluation_imports"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
adversary_name = Column(String, nullable=False) # "apt29", "turla"
adversary_display = Column(String, nullable=False) # "APT29", "Turla"
eval_round = Column(Integer, nullable=False) # 1, 2, 3 …
imported_at = Column(DateTime, nullable=False, default=datetime.utcnow)
imported_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
tests_created = Column(Integer, default=0)
techniques_covered = Column(Integer, default=0)
status = Column(String, default="completed") # "completed" | "failed"
notes = Column(Text, nullable=True)
__table_args__ = (
Index("ix_evaluation_imports_adversary", "adversary_name"),
Index("ix_evaluation_imports_round", "eval_round"),
)

View File

@@ -501,6 +501,133 @@ def update_email_config(
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# ATT&CK Evaluations endpoints (admin only)
# ---------------------------------------------------------------------------
@router.get("/attck-evaluations/rounds")
def list_evaluation_rounds(
db: Session = Depends(get_db),
current_user: User = Depends(require_role("admin")),
):
"""Return all public CrowdStrike ENTERPRISE evaluation rounds with import status.
Each entry includes whether it has already been imported into this platform.
"""
from app.services.attck_evaluations_service import fetch_available_rounds
from app.models.evaluation_import import EvaluationImport
try:
rounds = fetch_available_rounds()
except Exception as exc:
raise HTTPException(status_code=502, detail=f"Could not reach MITRE Evaluations API: {exc}")
imported = {
row.adversary_name.lower(): row
for row in db.query(EvaluationImport).filter(EvaluationImport.status == "completed").all()
}
return [
{
"name": r["name"],
"display_name": r.get("display_name", r["name"]),
"eval_round": r["eval_round"],
"imported": r["name"].lower() in imported,
"imported_at": imported[r["name"].lower()].imported_at.isoformat()
if r["name"].lower() in imported else None,
"tests_created": imported[r["name"].lower()].tests_created
if r["name"].lower() in imported else None,
"techniques_covered": imported[r["name"].lower()].techniques_covered
if r["name"].lower() in imported else None,
}
for r in rounds
]
@router.post("/attck-evaluations/import")
def import_evaluation_round(
payload: dict,
db: Session = Depends(get_db),
current_user: User = Depends(require_role("admin")),
):
"""Import a specific ATT&CK Evaluation round for CrowdStrike.
Body: { "adversary_name": "apt29", "adversary_display": "APT29", "eval_round": 2 }
Creates tests in ``in_review`` state — Blue Leads must validate each
result before it counts as real coverage.
"""
from app.services.attck_evaluations_service import import_evaluation_round as _import
adversary_name = payload.get("adversary_name", "")
adversary_display = payload.get("adversary_display", adversary_name)
eval_round = payload.get("eval_round", 0)
if not adversary_name or not eval_round:
raise HTTPException(status_code=400, detail="adversary_name and eval_round are required")
try:
summary = _import(db, adversary_name, adversary_display, eval_round, current_user)
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc))
except Exception as exc:
logger.error("ATT&CK Evaluation import failed: %s", exc, exc_info=True)
raise HTTPException(status_code=502, detail=f"Import failed: {exc}")
return {
"message": f"Import complete — {summary['created']} tests created",
**summary,
}
@router.post("/attck-evaluations/import-latest")
def import_latest_evaluation(
db: Session = Depends(get_db),
current_user: User = Depends(require_role("admin")),
):
"""Import the latest available CrowdStrike evaluation round automatically.
Returns 409 if the latest round was already imported.
"""
from app.services.attck_evaluations_service import get_latest_round, import_evaluation_round as _import
try:
latest = get_latest_round()
except Exception as exc:
raise HTTPException(status_code=502, detail=f"Could not reach MITRE Evaluations API: {exc}")
try:
summary = _import(
db,
latest["name"],
latest.get("display_name", latest["name"]),
latest["eval_round"],
current_user,
)
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc))
except Exception as exc:
logger.error("ATT&CK Evaluation import failed: %s", exc, exc_info=True)
raise HTTPException(status_code=502, detail=f"Import failed: {exc}")
return {
"message": f"Import complete — {summary['created']} tests created",
**summary,
}
@router.get("/attck-evaluations/check-new")
def check_new_evaluation_round(
db: Session = Depends(get_db),
current_user: User = Depends(require_role("admin")),
):
"""Check if a new ATT&CK Evaluation round is available that hasn't been imported yet."""
from app.services.attck_evaluations_service import check_for_new_round
return check_for_new_round(db)
@router.post("/email-test") @router.post("/email-test")
def send_test_email( def send_test_email(
payload: EmailTestRequest, payload: EmailTestRequest,

View File

@@ -0,0 +1,401 @@
"""ATT&CK Evaluations importer — fetches real CrowdStrike detection results
from MITRE Engenuity's public API and seeds the platform with validated tests.
Data source
-----------
https://evals.mitre.org/api/
- /participants/ → list of vendors + rounds they completed
- /results/?participant=crowdstrike&domain=ENTERPRISE
→ per-substep detection results per adversary
Detection level mapping (MITRE → Aegis)
---------------------------------------
Technique / Specific Behavior → detected (correctly identified ATT&CK technique)
Tactic → partially_detected (behavior noted but not categorized)
General / IOC / MSSP → partially_detected (anomaly detected, not ATT&CK-mapped)
Telemetry → partially_detected (raw data only — marginal detection)
None / N/A → not_detected
All imported tests are created in ``in_review`` state so Blue Leads must
confirm each result before it counts as real coverage for the organisation.
Important caveats stored in every test's description
------------------------------------------------------
"Source: MITRE ATT&CK Evaluation (Round N — Adversary). Results reflect
CrowdStrike Falcon in a controlled lab environment, NOT this organisation's
deployment. Validate detection in your own environment before approving."
"""
import logging
import uuid
from datetime import datetime
from typing import Any
import requests
from sqlalchemy.orm import Session
from app.models.enums import TestState, TestResult
from app.models.evaluation_import import EvaluationImport
from app.models.technique import Technique
from app.models.test import Test
from app.models.user import User
from app.services.audit_service import log_action
from app.services.status_service import recalculate_technique_status
logger = logging.getLogger(__name__)
_BASE = "https://evals.mitre.org"
_TIMEOUT = 30 # seconds per HTTP call
_VENDOR = "crowdstrike"
_DOMAIN = "ENTERPRISE"
# Detection type → quality score (higher = better)
_DETECTION_SCORE: dict[str, int] = {
"none": 0,
"n/a": 0,
"telemetry": 1,
"mssp": 2,
"general": 2,
"ioc": 2,
"tactic": 3,
"technique": 4,
"specific behavior": 4,
}
def _score(detection_type: str) -> int:
key = (detection_type or "").lower().strip()
for pattern, score in _DETECTION_SCORE.items():
if pattern in key:
return score
return 0
def _score_to_result(score: int) -> TestResult:
if score >= 4:
return TestResult.detected
if score >= 1:
return TestResult.partially_detected
return TestResult.not_detected
# ---------------------------------------------------------------------------
# Public API helpers
# ---------------------------------------------------------------------------
def fetch_available_rounds() -> list[dict[str, Any]]:
"""Return all evaluation rounds CrowdStrike has completed (ENTERPRISE only).
Each dict has: name, display_name, eval_round.
Sorted by eval_round ascending.
"""
try:
resp = requests.get(f"{_BASE}/api/participants/", timeout=_TIMEOUT)
resp.raise_for_status()
participants = resp.json()
except Exception as exc:
logger.error("Failed to fetch ATT&CK Evaluations participants: %s", exc)
raise
crowdstrike = next(
(p for p in participants if p.get("name", "").lower() == _VENDOR),
None,
)
if not crowdstrike:
raise ValueError(f"Vendor '{_VENDOR}' not found in evaluations participants list")
rounds = [
adv
for adv in crowdstrike.get("adversaries_completed", [])
if adv.get("domain", "").upper() == _DOMAIN
and adv.get("status", "").upper() == "PUBLIC"
]
rounds.sort(key=lambda x: x.get("eval_round", 0))
return rounds
def get_latest_round() -> dict[str, Any]:
"""Return the most recent PUBLIC ENTERPRISE round CrowdStrike participated in."""
rounds = fetch_available_rounds()
if not rounds:
raise ValueError("No public Enterprise evaluation rounds found for CrowdStrike")
return rounds[-1]
def fetch_results_for_adversary(adversary_name: str) -> list[dict[str, Any]]:
"""Fetch all per-substep detection results for a specific adversary round.
Returns a flat list of substep dicts, each containing:
technique_id, technique_name, tactic_id, best_score, detection_type, note.
"""
url = f"{_BASE}/api/results/?participant={_VENDOR}&domain={_DOMAIN}"
try:
resp = requests.get(url, timeout=_TIMEOUT)
resp.raise_for_status()
data = resp.json()
except Exception as exc:
logger.error("Failed to fetch ATT&CK Evaluations results: %s", exc)
raise
# Find the adversary in the response
adversaries = data.get("adversaries", [])
target = next(
(a for a in adversaries if a.get("Adversary_Name", "").lower() == adversary_name.lower()),
None,
)
if not target:
raise ValueError(
f"Adversary '{adversary_name}' not found in results. "
f"Available: {[a.get('Adversary_Name') for a in adversaries]}"
)
substeps: list[dict[str, Any]] = []
scenarios = target.get("Detections_By_Step", {})
for _scenario_name, scenario_data in scenarios.items():
for step in scenario_data.get("Steps", []):
for substep in step.get("Substeps", []):
# Prefer sub-technique over technique
sub = substep.get("Subtechnique") or {}
tech = substep.get("Technique") or {}
tactic = substep.get("Tactic") or {}
technique_id = (
sub.get("Subtechnique_Id")
or tech.get("Technique_Id")
or ""
).strip()
technique_name = (
sub.get("Subtechnique_Name")
or tech.get("Technique_Name")
or "Unknown"
).strip()
if not technique_id:
continue
detections = substep.get("Detections", [])
best_score = 0
best_type = "None"
best_note = ""
for det in detections:
dtype = det.get("Detection_Type", "None")
s = _score(dtype)
if s > best_score:
best_score = s
best_type = dtype
best_note = det.get("Detection_Note", "")
substeps.append(
{
"technique_id": technique_id,
"technique_name": technique_name,
"tactic_id": tactic.get("Tactic_Id", ""),
"tactic_name": tactic.get("Tactic_Name", ""),
"best_score": best_score,
"detection_type": best_type,
"note": best_note,
}
)
return substeps
def _aggregate_by_technique(substeps: list[dict]) -> dict[str, dict]:
"""Aggregate substep results per technique — keep best detection score."""
by_technique: dict[str, dict] = {}
for sub in substeps:
tid = sub["technique_id"]
if tid not in by_technique or sub["best_score"] > by_technique[tid]["best_score"]:
by_technique[tid] = sub
return by_technique
# ---------------------------------------------------------------------------
# Main import function
# ---------------------------------------------------------------------------
def import_evaluation_round(
db: Session,
adversary_name: str,
adversary_display: str,
eval_round: int,
current_user: User,
) -> dict[str, Any]:
"""Import a single ATT&CK Evaluation round for CrowdStrike into the platform.
Creates one Test per unique technique with the best detection result
observed across all substeps for that technique. All tests land in
``in_review`` state — Blue Leads must confirm before they count as coverage.
Returns a summary dict: created, skipped, techniques_covered.
Raises if the round was already imported (idempotency guard).
"""
# Idempotency — refuse duplicate imports
existing = (
db.query(EvaluationImport)
.filter(
EvaluationImport.adversary_name == adversary_name.lower(),
EvaluationImport.status == "completed",
)
.first()
)
if existing:
raise ValueError(
f"Round '{adversary_display}' (round {eval_round}) was already imported "
f"on {existing.imported_at.date()}. Re-import is not allowed."
)
# Fetch and aggregate substep results
substeps = fetch_results_for_adversary(adversary_name)
by_technique = _aggregate_by_technique(substeps)
created = 0
skipped = 0
affected_technique_ids: set = set()
for mitre_id, agg in by_technique.items():
# Look up the technique in our DB
technique = (
db.query(Technique)
.filter(Technique.mitre_id == mitre_id.upper())
.first()
)
if technique is None:
skipped += 1
continue
detection_result = _score_to_result(agg["best_score"])
description = (
f"Source: MITRE ATT&CK Evaluation — Round {eval_round} ({adversary_display}).\n"
f"Vendor: CrowdStrike Falcon.\n"
f"Detection type achieved: {agg['detection_type']}.\n\n"
f"⚠️ IMPORTANT: These results reflect CrowdStrike Falcon performance in a "
f"controlled MITRE lab environment against a simulated {adversary_display} "
f"adversary. They do NOT represent your organisation's actual detection "
f"capability. Validate in your own environment before approving."
)
if agg["note"]:
description += f"\n\nMITRE note: {agg['note']}"
red_summary = (
f"MITRE ATT&CK Evaluation — Round {eval_round} ({adversary_display})\n"
f"Vendor: CrowdStrike Falcon\n"
f"Best detection level: {agg['detection_type']}\n"
f"Tactic: {agg['tactic_name']} ({agg['tactic_id']})"
)
test = Test(
technique_id=technique.id,
name=f"[EVAL R{eval_round}] {adversary_display}{technique.name}",
description=description,
platform=None,
procedure_text=(
f"MITRE ATT&CK Evaluation simulation using {adversary_display} TTPs. "
f"See evaluation report at https://evals.mitre.org for full details."
),
created_by=current_user.id,
state=TestState.in_review,
attack_success=True,
red_summary=red_summary,
red_validation_status="approved",
red_validated_by=current_user.id,
red_validated_at=datetime.utcnow(),
detection_result=detection_result,
blue_validation_status=None,
execution_date=datetime.utcnow(),
created_at=datetime.utcnow(),
)
db.add(test)
db.flush()
log_action(
db,
user_id=current_user.id,
action="eval_import_test",
entity_type="test",
entity_id=test.id,
details={
"adversary": adversary_name,
"eval_round": eval_round,
"mitre_id": mitre_id,
"detection_type": agg["detection_type"],
},
)
affected_technique_ids.add(technique.id)
created += 1
# Recalculate coverage for all touched techniques
for tech_id in affected_technique_ids:
tech = db.query(Technique).filter(Technique.id == tech_id).first()
if tech:
recalculate_technique_status(db, tech)
# Record the import
record = EvaluationImport(
id=uuid.uuid4(),
adversary_name=adversary_name.lower(),
adversary_display=adversary_display,
eval_round=eval_round,
imported_at=datetime.utcnow(),
imported_by=current_user.id,
tests_created=created,
techniques_covered=len(affected_technique_ids),
status="completed",
notes=f"Skipped {skipped} techniques not found in local DB.",
)
db.add(record)
db.commit()
logger.info(
"ATT&CK Evaluation import complete — round %d (%s): %d tests created, %d skipped",
eval_round, adversary_display, created, skipped,
)
return {
"created": created,
"skipped": skipped,
"techniques_covered": len(affected_technique_ids),
"adversary": adversary_display,
"eval_round": eval_round,
}
# ---------------------------------------------------------------------------
# New-round check (used by the weekly scheduler)
# ---------------------------------------------------------------------------
def check_for_new_round(db: Session) -> dict[str, Any]:
"""Check if a new evaluation round is available that hasn't been imported yet.
Returns:
{"new_round_available": bool, "latest_round": dict | None, "already_imported": bool}
"""
try:
latest = get_latest_round()
except Exception as exc:
logger.warning("Could not check for new ATT&CK Evaluation round: %s", exc)
return {"new_round_available": False, "latest_round": None, "error": str(exc)}
already = (
db.query(EvaluationImport)
.filter(
EvaluationImport.adversary_name == latest["name"].lower(),
EvaluationImport.status == "completed",
)
.first()
)
return {
"new_round_available": already is None,
"already_imported": already is not None,
"latest_round": {
"name": latest["name"],
"display_name": latest.get("display_name", latest["name"]),
"eval_round": latest["eval_round"],
},
}

View File

@@ -40,3 +40,59 @@ export async function getSchedulerStatus(): Promise<SchedulerStatusResponse> {
const { data } = await client.get<SchedulerStatusResponse>("/system/scheduler-status"); const { data } = await client.get<SchedulerStatusResponse>("/system/scheduler-status");
return data; return data;
} }
// ── ATT&CK Evaluations ─────────────────────────────────────────────
export interface EvaluationRound {
name: string;
display_name: string;
eval_round: number;
imported: boolean;
imported_at: string | null;
tests_created: number | null;
techniques_covered: number | null;
}
export interface EvaluationImportResult {
message: string;
created: number;
skipped: number;
techniques_covered: number;
adversary: string;
eval_round: number;
}
export interface NewRoundCheckResult {
new_round_available: boolean;
already_imported: boolean;
latest_round: { name: string; display_name: string; eval_round: number } | null;
error?: string;
}
/** List all public CrowdStrike evaluation rounds with import status. */
export async function listEvaluationRounds(): Promise<EvaluationRound[]> {
const { data } = await client.get<EvaluationRound[]>("/system/attck-evaluations/rounds");
return data;
}
/** Import a specific evaluation round. */
export async function importEvaluationRound(payload: {
adversary_name: string;
adversary_display: string;
eval_round: number;
}): Promise<EvaluationImportResult> {
const { data } = await client.post<EvaluationImportResult>("/system/attck-evaluations/import", payload);
return data;
}
/** Import the latest available round automatically. */
export async function importLatestEvaluation(): Promise<EvaluationImportResult> {
const { data } = await client.post<EvaluationImportResult>("/system/attck-evaluations/import-latest");
return data;
}
/** Check if a new round is available. */
export async function checkNewEvaluationRound(): Promise<NewRoundCheckResult> {
const { data } = await client.get<NewRoundCheckResult>("/system/attck-evaluations/check-new");
return data;
}

View File

@@ -21,14 +21,26 @@ import {
Download, Download,
Upload, Upload,
PackageOpen, PackageOpen,
Swords,
Sparkles,
AlertTriangle,
ExternalLink,
CalendarCheck,
} from "lucide-react"; } from "lucide-react";
import client from "../api/client"; import client from "../api/client";
import { import {
triggerMitreSync, triggerMitreSync,
triggerIntelScan, triggerIntelScan,
getSchedulerStatus, getSchedulerStatus,
listEvaluationRounds,
importEvaluationRound,
importLatestEvaluation,
checkNewEvaluationRound,
type SyncMitreResponse, type SyncMitreResponse,
type IntelScanResponse, type IntelScanResponse,
type EvaluationRound,
type EvaluationImportResult,
type NewRoundCheckResult,
} from "../api/system"; } from "../api/system";
import { import {
getTemplateStats, getTemplateStats,
@@ -51,6 +63,11 @@ export default function SystemPage() {
const [bulkConfirm, setBulkConfirm] = useState<"activate" | "deactivate" | null>(null); const [bulkConfirm, setBulkConfirm] = useState<"activate" | "deactivate" | null>(null);
const [selectedTemplateId, setSelectedTemplateId] = useState<string | null>(null); const [selectedTemplateId, setSelectedTemplateId] = useState<string | null>(null);
// ── ATT&CK Evaluations state ─────────────────────────────────────
const [evalImportResult, setEvalImportResult] = useState<EvaluationImportResult | null>(null);
const [evalCheckResult, setEvalCheckResult] = useState<NewRoundCheckResult | null>(null);
const [evalImportingRound, setEvalImportingRound] = useState<string | null>(null);
// ── Existing queries ───────────────────────────────────────────── // ── Existing queries ─────────────────────────────────────────────
const { const {
data: schedulerStatus, data: schedulerStatus,
@@ -145,6 +162,53 @@ export default function SystemPage() {
}, },
}); });
// ── ATT&CK Evaluations queries & mutations ───────────────────────
const {
data: evalRounds,
isLoading: evalRoundsLoading,
refetch: refetchEvalRounds,
} = useQuery<EvaluationRound[]>({
queryKey: ["eval-rounds"],
queryFn: listEvaluationRounds,
});
const checkNewRoundMutation = useMutation({
mutationFn: checkNewEvaluationRound,
onSuccess: (data) => {
setEvalCheckResult(data);
refetchEvalRounds();
},
});
const importLatestMutation = useMutation({
mutationFn: importLatestEvaluation,
onSuccess: (data) => {
setEvalImportResult(data);
setEvalImportingRound(null);
refetchEvalRounds();
queryClient.invalidateQueries({ queryKey: ["techniques"] });
queryClient.invalidateQueries({ queryKey: ["metrics"] });
},
onError: () => {
setEvalImportingRound(null);
},
});
const importRoundMutation = useMutation({
mutationFn: (payload: { adversary_name: string; adversary_display: string; eval_round: number }) =>
importEvaluationRound(payload),
onSuccess: (data) => {
setEvalImportResult(data);
setEvalImportingRound(null);
refetchEvalRounds();
queryClient.invalidateQueries({ queryKey: ["techniques"] });
queryClient.invalidateQueries({ queryKey: ["metrics"] });
},
onError: () => {
setEvalImportingRound(null);
},
});
const formatNextRun = (dateStr: string | null) => { const formatNextRun = (dateStr: string | null) => {
if (!dateStr) return "Not scheduled"; if (!dateStr) return "Not scheduled";
const date = new Date(dateStr); const date = new Date(dateStr);
@@ -294,6 +358,264 @@ export default function SystemPage() {
</div> </div>
</div> </div>
{/* ────────────────────────────────────────────────────────────────
ATT&CK EVALUATIONS — CrowdStrike
──────────────────────────────────────────────────────────────── */}
<div className="rounded-xl border border-orange-500/30 bg-gray-900 p-6">
{/* Header */}
<div className="flex flex-wrap items-start justify-between gap-4 mb-5">
<div className="flex items-start gap-4">
<div className="rounded-lg bg-orange-500/10 p-3 mt-0.5">
<Swords className="h-6 w-6 text-orange-400" />
</div>
<div>
<h2 className="text-lg font-semibold text-white">
ATT&CK Evaluations CrowdStrike Falcon
</h2>
<p className="mt-1 text-sm text-gray-400 max-w-2xl">
Seed the platform with real detection data from MITRE Engenuity's public ATT&CK
Evaluations. Results are imported as <span className="text-yellow-400 font-medium">In Review</span> tests —
Blue Leads must validate each one before it counts as coverage.
</p>
</div>
</div>
<div className="flex items-center gap-2 flex-shrink-0">
<button
onClick={() => checkNewRoundMutation.mutate()}
disabled={checkNewRoundMutation.isPending || evalRoundsLoading}
className="flex items-center gap-2 rounded-lg border border-gray-700 bg-gray-800 px-3 py-2 text-sm font-medium text-gray-300 hover:bg-gray-700 disabled:opacity-50 transition-colors"
>
{checkNewRoundMutation.isPending ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<RefreshCw className="h-4 w-4" />
)}
Check for new rounds
</button>
<button
onClick={() => {
setEvalImportingRound("latest");
importLatestMutation.mutate();
}}
disabled={importLatestMutation.isPending || importRoundMutation.isPending}
className="flex items-center gap-2 rounded-lg bg-orange-600 px-3 py-2 text-sm font-medium text-white hover:bg-orange-500 disabled:opacity-50 transition-colors"
>
{importLatestMutation.isPending && evalImportingRound === "latest" ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<Sparkles className="h-4 w-4" />
)}
Import Latest Round
</button>
</div>
</div>
{/* Disclaimer callout */}
<div className="mb-5 flex items-start gap-3 rounded-lg border border-yellow-500/30 bg-yellow-900/10 p-4">
<AlertTriangle className="h-5 w-5 text-yellow-400 flex-shrink-0 mt-0.5" />
<div className="text-sm">
<p className="font-medium text-yellow-400 mb-1">Lab environment data — validation required</p>
<p className="text-gray-400">
These results reflect CrowdStrike Falcon performance in a controlled MITRE lab against
simulated adversaries. They do <strong className="text-white">not</strong> represent
your organisation's actual detection capability. All imported tests require Blue Lead
validation before contributing to programme coverage.
</p>
<a
href="https://evals.mitre.org"
target="_blank"
rel="noopener noreferrer"
className="mt-2 inline-flex items-center gap-1 text-xs text-orange-400 hover:text-orange-300 transition-colors"
>
<ExternalLink className="h-3 w-3" />
View official evaluation reports at evals.mitre.org
</a>
</div>
</div>
{/* New round check result */}
{evalCheckResult && (
<div className={`mb-4 rounded-lg border p-3 ${
evalCheckResult.error
? "border-red-500/30 bg-red-900/20"
: evalCheckResult.new_round_available
? "border-green-500/30 bg-green-900/20"
: "border-gray-700 bg-gray-800/50"
}`}>
{evalCheckResult.error ? (
<div className="flex items-center gap-2 text-sm text-red-400">
<XCircle className="h-4 w-4 flex-shrink-0" />
<span>Check failed: {evalCheckResult.error}</span>
</div>
) : evalCheckResult.new_round_available ? (
<div className="flex items-center gap-2 text-sm">
<CheckCircle className="h-4 w-4 text-green-400 flex-shrink-0" />
<span className="text-green-400 font-medium">
New round available: {evalCheckResult.latest_round?.display_name} (Round {evalCheckResult.latest_round?.eval_round})
</span>
</div>
) : (
<div className="flex items-center gap-2 text-sm">
<CalendarCheck className="h-4 w-4 text-gray-400 flex-shrink-0" />
<span className="text-gray-400">
Up to date latest round ({evalCheckResult.latest_round?.display_name}) is already imported.
</span>
</div>
)}
</div>
)}
{/* Import result feedback */}
{evalImportResult && (
<div className="mb-4 rounded-lg border border-green-500/30 bg-green-900/20 p-4">
<div className="flex items-center gap-2 mb-2">
<CheckCircle className="h-4 w-4 text-green-400" />
<span className="text-sm font-medium text-green-400">Import complete</span>
</div>
<div className="grid grid-cols-3 gap-3 text-center text-sm">
<div>
<p className="text-xl font-bold text-white">{evalImportResult.created}</p>
<p className="text-xs text-gray-400">Tests created</p>
</div>
<div>
<p className="text-xl font-bold text-white">{evalImportResult.techniques_covered}</p>
<p className="text-xs text-gray-400">Techniques covered</p>
</div>
<div>
<p className="text-sm font-medium text-orange-400 truncate">{evalImportResult.adversary}</p>
<p className="text-xs text-gray-400">Round {evalImportResult.eval_round}</p>
</div>
</div>
<p className="mt-3 text-xs text-yellow-400 flex items-center gap-1">
<AlertTriangle className="h-3 w-3" />
All tests are in <strong className="ml-1">Review Queue</strong> Blue Leads must validate before they count as coverage.
</p>
</div>
)}
{/* Import error */}
{(importLatestMutation.isError || importRoundMutation.isError) && (
<div className="mb-4 rounded-lg border border-red-500/30 bg-red-900/20 p-3">
<div className="flex items-center gap-2 text-sm text-red-400">
<XCircle className="h-4 w-4 flex-shrink-0" />
<span>
{((importLatestMutation.error || importRoundMutation.error) as Error)?.message ??
"Import failed. This round may already be imported."}
</span>
</div>
</div>
)}
{/* Rounds table */}
<div>
<p className="mb-3 text-xs font-semibold uppercase tracking-wider text-gray-500">
Available Rounds
</p>
{evalRoundsLoading ? (
<div className="flex items-center justify-center py-8">
<Loader2 className="h-6 w-6 animate-spin text-orange-400" />
</div>
) : evalRounds && evalRounds.length > 0 ? (
<div className="overflow-x-auto">
<table className="w-full text-left text-sm">
<thead>
<tr className="border-b border-gray-800">
<th className="pb-3 pr-4 font-medium text-gray-400">Round</th>
<th className="pb-3 px-4 font-medium text-gray-400">Adversary</th>
<th className="pb-3 px-4 font-medium text-gray-400">Status</th>
<th className="pb-3 px-4 font-medium text-gray-400">Imported</th>
<th className="pb-3 px-4 font-medium text-gray-400">Tests</th>
<th className="pb-3 pl-4 font-medium text-gray-400">Action</th>
</tr>
</thead>
<tbody>
{evalRounds.map((round) => (
<tr
key={round.name}
className="border-b border-gray-800/50 hover:bg-gray-800/30 transition-colors"
>
<td className="py-3 pr-4">
<span className="inline-flex items-center justify-center rounded-full border border-orange-500/30 bg-orange-900/20 px-2.5 py-0.5 text-xs font-bold text-orange-400">
R{round.eval_round}
</span>
</td>
<td className="py-3 px-4">
<p className="font-medium text-gray-200">{round.display_name}</p>
<p className="text-xs text-gray-500 font-mono">{round.name}</p>
</td>
<td className="py-3 px-4">
{round.imported ? (
<span className="inline-flex items-center gap-1.5 rounded-full border border-green-500/30 bg-green-900/20 px-2.5 py-0.5 text-xs font-medium text-green-400">
<CheckCircle className="h-3 w-3" />
Imported
</span>
) : (
<span className="inline-flex items-center gap-1.5 rounded-full border border-gray-600 bg-gray-800/50 px-2.5 py-0.5 text-xs font-medium text-gray-400">
Not imported
</span>
)}
</td>
<td className="py-3 px-4 text-xs text-gray-400">
{round.imported_at
? new Date(round.imported_at).toLocaleDateString("en-GB", {
day: "numeric",
month: "short",
year: "numeric",
})
: "—"}
</td>
<td className="py-3 px-4 text-center">
{round.imported ? (
<div className="text-center">
<p className="text-sm font-medium text-gray-200">{round.tests_created ?? "—"}</p>
<p className="text-xs text-gray-500">{round.techniques_covered ?? ""} techniques</p>
</div>
) : (
<span className="text-gray-600"></span>
)}
</td>
<td className="py-3 pl-4">
{round.imported ? (
<span className="text-xs text-gray-600 italic">Already imported</span>
) : (
<button
onClick={() => {
setEvalImportingRound(round.name);
importRoundMutation.mutate({
adversary_name: round.name,
adversary_display: round.display_name,
eval_round: round.eval_round,
});
}}
disabled={
importLatestMutation.isPending ||
importRoundMutation.isPending
}
className="flex items-center gap-1.5 rounded-lg border border-orange-500/30 bg-orange-900/20 px-3 py-1.5 text-xs font-medium text-orange-400 hover:bg-orange-900/40 disabled:opacity-50 transition-colors"
>
{importRoundMutation.isPending && evalImportingRound === round.name ? (
<Loader2 className="h-3.5 w-3.5 animate-spin" />
) : (
<Download className="h-3.5 w-3.5" />
)}
Import
</button>
)}
</td>
</tr>
))}
</tbody>
</table>
</div>
) : (
<div className="flex items-center justify-center gap-2 py-8 text-gray-500 text-sm">
<AlertCircle className="h-4 w-4" />
<span>Unable to load evaluation rounds. Check network connectivity to evals.mitre.org.</span>
</div>
)}
</div>
</div>
{/* ──────────────────────────────────────────────────────────────── {/* ────────────────────────────────────────────────────────────────
TEMPLATE ADMINISTRATION (T-124) TEMPLATE ADMINISTRATION (T-124)
──────────────────────────────────────────────────────────────── */} ──────────────────────────────────────────────────────────────── */}