feat(evaluations): ATT&CK Evaluations importer for CrowdStrike Falcon [FASE-6.1]
Some checks failed
Aegis CI / lint-and-test (push) Has been cancelled
Some checks failed
Aegis CI / lint-and-test (push) Has been cancelled
- Migration b048: evaluation_imports table (adversary, round, status, tests_created) - EvaluationImport SQLAlchemy model - attck_evaluations_service: fetch rounds from evals.mitre.org API, import per-technique detection results (Technique/Tactic/Telemetry -> detected/partially/not_detected) - All imported tests land in in_review state with lab-environment disclaimer - Idempotency guard prevents duplicate round imports - 4 new endpoints: list rounds, import specific, import latest, check-new - Weekly APScheduler cron (Mon 06:00) auto-checks and imports new rounds - SystemPage UI: rounds table, import buttons, check-new, result feedback - Disclaimer callout reminding admins these are lab results not org coverage Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
39
backend/alembic/versions/b048_evaluation_imports.py
Normal file
39
backend/alembic/versions/b048_evaluation_imports.py
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
"""Add evaluation_imports table.
|
||||||
|
|
||||||
|
Revision ID: b048
|
||||||
|
Revises: b047
|
||||||
|
Create Date: 2026-06-05
|
||||||
|
"""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID
|
||||||
|
|
||||||
|
revision = "b048"
|
||||||
|
down_revision = "b047"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"evaluation_imports",
|
||||||
|
sa.Column("id", UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("adversary_name", sa.String, nullable=False),
|
||||||
|
sa.Column("adversary_display", sa.String, nullable=False),
|
||||||
|
sa.Column("eval_round", sa.Integer, nullable=False),
|
||||||
|
sa.Column("imported_at", sa.DateTime, nullable=False),
|
||||||
|
sa.Column("imported_by", UUID(as_uuid=True), sa.ForeignKey("users.id"), nullable=True),
|
||||||
|
sa.Column("tests_created", sa.Integer, default=0),
|
||||||
|
sa.Column("techniques_covered", sa.Integer, default=0),
|
||||||
|
sa.Column("status", sa.String, default="completed"),
|
||||||
|
sa.Column("notes", sa.Text, nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index("ix_evaluation_imports_adversary", "evaluation_imports", ["adversary_name"])
|
||||||
|
op.create_index("ix_evaluation_imports_round", "evaluation_imports", ["eval_round"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_evaluation_imports_round", table_name="evaluation_imports")
|
||||||
|
op.drop_index("ix_evaluation_imports_adversary", table_name="evaluation_imports")
|
||||||
|
op.drop_table("evaluation_imports")
|
||||||
@@ -204,6 +204,83 @@ def _run_intel_scan() -> None:
|
|||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _run_evaluation_round_check() -> None:
|
||||||
|
"""Weekly job: check if a new ATT&CK Evaluation round is available.
|
||||||
|
|
||||||
|
If a new round is found it is imported automatically and an admin
|
||||||
|
notification is created so the team knows new baseline data is available.
|
||||||
|
"""
|
||||||
|
logger.info("ATT&CK Evaluations new-round check starting...")
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
from app.services.attck_evaluations_service import check_for_new_round, import_evaluation_round
|
||||||
|
from app.models.user import User as UserModel
|
||||||
|
|
||||||
|
result = check_for_new_round(db)
|
||||||
|
if result.get("error"):
|
||||||
|
logger.warning("ATT&CK Evaluations check failed: %s", result["error"])
|
||||||
|
return
|
||||||
|
|
||||||
|
if not result.get("new_round_available"):
|
||||||
|
logger.info(
|
||||||
|
"ATT&CK Evaluations check — latest round '%s' already imported",
|
||||||
|
result.get("latest_round", {}).get("display_name", "?"),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
latest = result["latest_round"]
|
||||||
|
logger.info(
|
||||||
|
"New ATT&CK Evaluation round detected: %s (round %d) — starting auto-import",
|
||||||
|
latest["display_name"], latest["eval_round"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Use the first admin user as the importer (system action)
|
||||||
|
admin = db.query(UserModel).filter(UserModel.role == "admin").first()
|
||||||
|
if not admin:
|
||||||
|
logger.warning("ATT&CK Evaluations auto-import: no admin user found — skipping")
|
||||||
|
return
|
||||||
|
|
||||||
|
summary = import_evaluation_round(
|
||||||
|
db,
|
||||||
|
latest["name"],
|
||||||
|
latest["display_name"],
|
||||||
|
latest["eval_round"],
|
||||||
|
admin,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"ATT&CK Evaluations auto-import complete — round %d (%s): %d tests created",
|
||||||
|
latest["eval_round"], latest["display_name"], summary["created"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Notify all admins
|
||||||
|
try:
|
||||||
|
from app.services.notification_service import create_notification
|
||||||
|
admins = db.query(UserModel).filter(UserModel.role == "admin").all()
|
||||||
|
for adm in admins:
|
||||||
|
create_notification(
|
||||||
|
db,
|
||||||
|
user_id=adm.id,
|
||||||
|
title="New ATT&CK Evaluation round imported",
|
||||||
|
message=(
|
||||||
|
f"Round {latest['eval_round']} — {latest['display_name']} — "
|
||||||
|
f"has been automatically imported. "
|
||||||
|
f"{summary['created']} tests created in In Review state. "
|
||||||
|
f"Blue Leads must validate each result before it counts as coverage."
|
||||||
|
),
|
||||||
|
notification_type="eval_import",
|
||||||
|
entity_type="evaluation",
|
||||||
|
entity_id=None,
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Failed to send eval import notifications", exc_info=True)
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
logger.exception("ATT&CK Evaluations round check job failed")
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
def _run_osint_enrichment() -> None:
|
def _run_osint_enrichment() -> None:
|
||||||
"""Execute weekly OSINT enrichment inside its own DB session."""
|
"""Execute weekly OSINT enrichment inside its own DB session."""
|
||||||
logger.info("Scheduled OSINT enrichment job starting...")
|
logger.info("Scheduled OSINT enrichment job starting...")
|
||||||
@@ -463,6 +540,16 @@ def start_scheduler() -> None:
|
|||||||
name="Operational alert evaluation (hourly)",
|
name="Operational alert evaluation (hourly)",
|
||||||
replace_existing=True,
|
replace_existing=True,
|
||||||
)
|
)
|
||||||
|
scheduler.add_job(
|
||||||
|
_run_evaluation_round_check,
|
||||||
|
trigger="cron",
|
||||||
|
day_of_week="mon",
|
||||||
|
hour=6,
|
||||||
|
minute=0,
|
||||||
|
id="attck_evaluation_check",
|
||||||
|
name="ATT&CK Evaluations new-round check (Mondays 06:00)",
|
||||||
|
replace_existing=True,
|
||||||
|
)
|
||||||
scheduler.start()
|
scheduler.start()
|
||||||
logger.info(
|
logger.info(
|
||||||
"Background scheduler started — mitre_sync (24h), intel_scan (7d), "
|
"Background scheduler started — mitre_sync (24h), intel_scan (7d), "
|
||||||
@@ -470,5 +557,5 @@ def start_scheduler() -> None:
|
|||||||
"recurring_campaigns (daily), jira_sync (1h), "
|
"recurring_campaigns (daily), jira_sync (1h), "
|
||||||
"osint_enrichment (weekly), stale_detection (daily), "
|
"osint_enrichment (weekly), stale_detection (daily), "
|
||||||
"retention_policies (daily), data_sources_sync (6h), "
|
"retention_policies (daily), data_sources_sync (6h), "
|
||||||
"alert_evaluation (1h)"
|
"alert_evaluation (1h), attck_evaluation_check (Mondays 06:00)"
|
||||||
)
|
)
|
||||||
|
|||||||
34
backend/app/models/evaluation_import.py
Normal file
34
backend/app/models/evaluation_import.py
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
"""SQLAlchemy model for tracking imported ATT&CK Evaluation rounds."""
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import Column, String, Integer, DateTime, Text, ForeignKey, Index
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID
|
||||||
|
|
||||||
|
from app.database import Base
|
||||||
|
|
||||||
|
|
||||||
|
class EvaluationImport(Base):
|
||||||
|
"""Tracks which ATT&CK Evaluation rounds have been imported into the platform.
|
||||||
|
|
||||||
|
Each row represents one vendor+adversary combination that has been processed
|
||||||
|
and turned into Test records. Used to avoid duplicate imports and to show
|
||||||
|
the admin panel which rounds are available vs imported.
|
||||||
|
"""
|
||||||
|
__tablename__ = "evaluation_imports"
|
||||||
|
|
||||||
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
adversary_name = Column(String, nullable=False) # "apt29", "turla"
|
||||||
|
adversary_display = Column(String, nullable=False) # "APT29", "Turla"
|
||||||
|
eval_round = Column(Integer, nullable=False) # 1, 2, 3 …
|
||||||
|
imported_at = Column(DateTime, nullable=False, default=datetime.utcnow)
|
||||||
|
imported_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
|
||||||
|
tests_created = Column(Integer, default=0)
|
||||||
|
techniques_covered = Column(Integer, default=0)
|
||||||
|
status = Column(String, default="completed") # "completed" | "failed"
|
||||||
|
notes = Column(Text, nullable=True)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_evaluation_imports_adversary", "adversary_name"),
|
||||||
|
Index("ix_evaluation_imports_round", "eval_round"),
|
||||||
|
)
|
||||||
@@ -501,6 +501,133 @@ def update_email_config(
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# ATT&CK Evaluations endpoints (admin only)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/attck-evaluations/rounds")
|
||||||
|
def list_evaluation_rounds(
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current_user: User = Depends(require_role("admin")),
|
||||||
|
):
|
||||||
|
"""Return all public CrowdStrike ENTERPRISE evaluation rounds with import status.
|
||||||
|
|
||||||
|
Each entry includes whether it has already been imported into this platform.
|
||||||
|
"""
|
||||||
|
from app.services.attck_evaluations_service import fetch_available_rounds
|
||||||
|
from app.models.evaluation_import import EvaluationImport
|
||||||
|
|
||||||
|
try:
|
||||||
|
rounds = fetch_available_rounds()
|
||||||
|
except Exception as exc:
|
||||||
|
raise HTTPException(status_code=502, detail=f"Could not reach MITRE Evaluations API: {exc}")
|
||||||
|
|
||||||
|
imported = {
|
||||||
|
row.adversary_name.lower(): row
|
||||||
|
for row in db.query(EvaluationImport).filter(EvaluationImport.status == "completed").all()
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"name": r["name"],
|
||||||
|
"display_name": r.get("display_name", r["name"]),
|
||||||
|
"eval_round": r["eval_round"],
|
||||||
|
"imported": r["name"].lower() in imported,
|
||||||
|
"imported_at": imported[r["name"].lower()].imported_at.isoformat()
|
||||||
|
if r["name"].lower() in imported else None,
|
||||||
|
"tests_created": imported[r["name"].lower()].tests_created
|
||||||
|
if r["name"].lower() in imported else None,
|
||||||
|
"techniques_covered": imported[r["name"].lower()].techniques_covered
|
||||||
|
if r["name"].lower() in imported else None,
|
||||||
|
}
|
||||||
|
for r in rounds
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/attck-evaluations/import")
|
||||||
|
def import_evaluation_round(
|
||||||
|
payload: dict,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current_user: User = Depends(require_role("admin")),
|
||||||
|
):
|
||||||
|
"""Import a specific ATT&CK Evaluation round for CrowdStrike.
|
||||||
|
|
||||||
|
Body: { "adversary_name": "apt29", "adversary_display": "APT29", "eval_round": 2 }
|
||||||
|
|
||||||
|
Creates tests in ``in_review`` state — Blue Leads must validate each
|
||||||
|
result before it counts as real coverage.
|
||||||
|
"""
|
||||||
|
from app.services.attck_evaluations_service import import_evaluation_round as _import
|
||||||
|
|
||||||
|
adversary_name = payload.get("adversary_name", "")
|
||||||
|
adversary_display = payload.get("adversary_display", adversary_name)
|
||||||
|
eval_round = payload.get("eval_round", 0)
|
||||||
|
|
||||||
|
if not adversary_name or not eval_round:
|
||||||
|
raise HTTPException(status_code=400, detail="adversary_name and eval_round are required")
|
||||||
|
|
||||||
|
try:
|
||||||
|
summary = _import(db, adversary_name, adversary_display, eval_round, current_user)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(status_code=409, detail=str(exc))
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("ATT&CK Evaluation import failed: %s", exc, exc_info=True)
|
||||||
|
raise HTTPException(status_code=502, detail=f"Import failed: {exc}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"message": f"Import complete — {summary['created']} tests created",
|
||||||
|
**summary,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/attck-evaluations/import-latest")
|
||||||
|
def import_latest_evaluation(
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current_user: User = Depends(require_role("admin")),
|
||||||
|
):
|
||||||
|
"""Import the latest available CrowdStrike evaluation round automatically.
|
||||||
|
|
||||||
|
Returns 409 if the latest round was already imported.
|
||||||
|
"""
|
||||||
|
from app.services.attck_evaluations_service import get_latest_round, import_evaluation_round as _import
|
||||||
|
|
||||||
|
try:
|
||||||
|
latest = get_latest_round()
|
||||||
|
except Exception as exc:
|
||||||
|
raise HTTPException(status_code=502, detail=f"Could not reach MITRE Evaluations API: {exc}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
summary = _import(
|
||||||
|
db,
|
||||||
|
latest["name"],
|
||||||
|
latest.get("display_name", latest["name"]),
|
||||||
|
latest["eval_round"],
|
||||||
|
current_user,
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise HTTPException(status_code=409, detail=str(exc))
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("ATT&CK Evaluation import failed: %s", exc, exc_info=True)
|
||||||
|
raise HTTPException(status_code=502, detail=f"Import failed: {exc}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"message": f"Import complete — {summary['created']} tests created",
|
||||||
|
**summary,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/attck-evaluations/check-new")
|
||||||
|
def check_new_evaluation_round(
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current_user: User = Depends(require_role("admin")),
|
||||||
|
):
|
||||||
|
"""Check if a new ATT&CK Evaluation round is available that hasn't been imported yet."""
|
||||||
|
from app.services.attck_evaluations_service import check_for_new_round
|
||||||
|
|
||||||
|
return check_for_new_round(db)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/email-test")
|
@router.post("/email-test")
|
||||||
def send_test_email(
|
def send_test_email(
|
||||||
payload: EmailTestRequest,
|
payload: EmailTestRequest,
|
||||||
|
|||||||
401
backend/app/services/attck_evaluations_service.py
Normal file
401
backend/app/services/attck_evaluations_service.py
Normal file
@@ -0,0 +1,401 @@
|
|||||||
|
"""ATT&CK Evaluations importer — fetches real CrowdStrike detection results
|
||||||
|
from MITRE Engenuity's public API and seeds the platform with validated tests.
|
||||||
|
|
||||||
|
Data source
|
||||||
|
-----------
|
||||||
|
https://evals.mitre.org/api/
|
||||||
|
- /participants/ → list of vendors + rounds they completed
|
||||||
|
- /results/?participant=crowdstrike&domain=ENTERPRISE
|
||||||
|
→ per-substep detection results per adversary
|
||||||
|
|
||||||
|
Detection level mapping (MITRE → Aegis)
|
||||||
|
---------------------------------------
|
||||||
|
Technique / Specific Behavior → detected (correctly identified ATT&CK technique)
|
||||||
|
Tactic → partially_detected (behavior noted but not categorized)
|
||||||
|
General / IOC / MSSP → partially_detected (anomaly detected, not ATT&CK-mapped)
|
||||||
|
Telemetry → partially_detected (raw data only — marginal detection)
|
||||||
|
None / N/A → not_detected
|
||||||
|
|
||||||
|
All imported tests are created in ``in_review`` state so Blue Leads must
|
||||||
|
confirm each result before it counts as real coverage for the organisation.
|
||||||
|
|
||||||
|
Important caveats stored in every test's description
|
||||||
|
------------------------------------------------------
|
||||||
|
"Source: MITRE ATT&CK Evaluation (Round N — Adversary). Results reflect
|
||||||
|
CrowdStrike Falcon in a controlled lab environment, NOT this organisation's
|
||||||
|
deployment. Validate detection in your own environment before approving."
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.models.enums import TestState, TestResult
|
||||||
|
from app.models.evaluation_import import EvaluationImport
|
||||||
|
from app.models.technique import Technique
|
||||||
|
from app.models.test import Test
|
||||||
|
from app.models.user import User
|
||||||
|
from app.services.audit_service import log_action
|
||||||
|
from app.services.status_service import recalculate_technique_status
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BASE = "https://evals.mitre.org"
|
||||||
|
_TIMEOUT = 30 # seconds per HTTP call
|
||||||
|
_VENDOR = "crowdstrike"
|
||||||
|
_DOMAIN = "ENTERPRISE"
|
||||||
|
|
||||||
|
# Detection type → quality score (higher = better)
|
||||||
|
_DETECTION_SCORE: dict[str, int] = {
|
||||||
|
"none": 0,
|
||||||
|
"n/a": 0,
|
||||||
|
"telemetry": 1,
|
||||||
|
"mssp": 2,
|
||||||
|
"general": 2,
|
||||||
|
"ioc": 2,
|
||||||
|
"tactic": 3,
|
||||||
|
"technique": 4,
|
||||||
|
"specific behavior": 4,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _score(detection_type: str) -> int:
|
||||||
|
key = (detection_type or "").lower().strip()
|
||||||
|
for pattern, score in _DETECTION_SCORE.items():
|
||||||
|
if pattern in key:
|
||||||
|
return score
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _score_to_result(score: int) -> TestResult:
|
||||||
|
if score >= 4:
|
||||||
|
return TestResult.detected
|
||||||
|
if score >= 1:
|
||||||
|
return TestResult.partially_detected
|
||||||
|
return TestResult.not_detected
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Public API helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_available_rounds() -> list[dict[str, Any]]:
|
||||||
|
"""Return all evaluation rounds CrowdStrike has completed (ENTERPRISE only).
|
||||||
|
|
||||||
|
Each dict has: name, display_name, eval_round.
|
||||||
|
Sorted by eval_round ascending.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
resp = requests.get(f"{_BASE}/api/participants/", timeout=_TIMEOUT)
|
||||||
|
resp.raise_for_status()
|
||||||
|
participants = resp.json()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("Failed to fetch ATT&CK Evaluations participants: %s", exc)
|
||||||
|
raise
|
||||||
|
|
||||||
|
crowdstrike = next(
|
||||||
|
(p for p in participants if p.get("name", "").lower() == _VENDOR),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if not crowdstrike:
|
||||||
|
raise ValueError(f"Vendor '{_VENDOR}' not found in evaluations participants list")
|
||||||
|
|
||||||
|
rounds = [
|
||||||
|
adv
|
||||||
|
for adv in crowdstrike.get("adversaries_completed", [])
|
||||||
|
if adv.get("domain", "").upper() == _DOMAIN
|
||||||
|
and adv.get("status", "").upper() == "PUBLIC"
|
||||||
|
]
|
||||||
|
rounds.sort(key=lambda x: x.get("eval_round", 0))
|
||||||
|
return rounds
|
||||||
|
|
||||||
|
|
||||||
|
def get_latest_round() -> dict[str, Any]:
|
||||||
|
"""Return the most recent PUBLIC ENTERPRISE round CrowdStrike participated in."""
|
||||||
|
rounds = fetch_available_rounds()
|
||||||
|
if not rounds:
|
||||||
|
raise ValueError("No public Enterprise evaluation rounds found for CrowdStrike")
|
||||||
|
return rounds[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_results_for_adversary(adversary_name: str) -> list[dict[str, Any]]:
|
||||||
|
"""Fetch all per-substep detection results for a specific adversary round.
|
||||||
|
|
||||||
|
Returns a flat list of substep dicts, each containing:
|
||||||
|
technique_id, technique_name, tactic_id, best_score, detection_type, note.
|
||||||
|
"""
|
||||||
|
url = f"{_BASE}/api/results/?participant={_VENDOR}&domain={_DOMAIN}"
|
||||||
|
try:
|
||||||
|
resp = requests.get(url, timeout=_TIMEOUT)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("Failed to fetch ATT&CK Evaluations results: %s", exc)
|
||||||
|
raise
|
||||||
|
|
||||||
|
# Find the adversary in the response
|
||||||
|
adversaries = data.get("adversaries", [])
|
||||||
|
target = next(
|
||||||
|
(a for a in adversaries if a.get("Adversary_Name", "").lower() == adversary_name.lower()),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if not target:
|
||||||
|
raise ValueError(
|
||||||
|
f"Adversary '{adversary_name}' not found in results. "
|
||||||
|
f"Available: {[a.get('Adversary_Name') for a in adversaries]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
substeps: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
scenarios = target.get("Detections_By_Step", {})
|
||||||
|
for _scenario_name, scenario_data in scenarios.items():
|
||||||
|
for step in scenario_data.get("Steps", []):
|
||||||
|
for substep in step.get("Substeps", []):
|
||||||
|
# Prefer sub-technique over technique
|
||||||
|
sub = substep.get("Subtechnique") or {}
|
||||||
|
tech = substep.get("Technique") or {}
|
||||||
|
tactic = substep.get("Tactic") or {}
|
||||||
|
|
||||||
|
technique_id = (
|
||||||
|
sub.get("Subtechnique_Id")
|
||||||
|
or tech.get("Technique_Id")
|
||||||
|
or ""
|
||||||
|
).strip()
|
||||||
|
technique_name = (
|
||||||
|
sub.get("Subtechnique_Name")
|
||||||
|
or tech.get("Technique_Name")
|
||||||
|
or "Unknown"
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
if not technique_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
detections = substep.get("Detections", [])
|
||||||
|
best_score = 0
|
||||||
|
best_type = "None"
|
||||||
|
best_note = ""
|
||||||
|
for det in detections:
|
||||||
|
dtype = det.get("Detection_Type", "None")
|
||||||
|
s = _score(dtype)
|
||||||
|
if s > best_score:
|
||||||
|
best_score = s
|
||||||
|
best_type = dtype
|
||||||
|
best_note = det.get("Detection_Note", "")
|
||||||
|
|
||||||
|
substeps.append(
|
||||||
|
{
|
||||||
|
"technique_id": technique_id,
|
||||||
|
"technique_name": technique_name,
|
||||||
|
"tactic_id": tactic.get("Tactic_Id", ""),
|
||||||
|
"tactic_name": tactic.get("Tactic_Name", ""),
|
||||||
|
"best_score": best_score,
|
||||||
|
"detection_type": best_type,
|
||||||
|
"note": best_note,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return substeps
|
||||||
|
|
||||||
|
|
||||||
|
def _aggregate_by_technique(substeps: list[dict]) -> dict[str, dict]:
|
||||||
|
"""Aggregate substep results per technique — keep best detection score."""
|
||||||
|
by_technique: dict[str, dict] = {}
|
||||||
|
for sub in substeps:
|
||||||
|
tid = sub["technique_id"]
|
||||||
|
if tid not in by_technique or sub["best_score"] > by_technique[tid]["best_score"]:
|
||||||
|
by_technique[tid] = sub
|
||||||
|
return by_technique
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Main import function
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def import_evaluation_round(
|
||||||
|
db: Session,
|
||||||
|
adversary_name: str,
|
||||||
|
adversary_display: str,
|
||||||
|
eval_round: int,
|
||||||
|
current_user: User,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Import a single ATT&CK Evaluation round for CrowdStrike into the platform.
|
||||||
|
|
||||||
|
Creates one Test per unique technique with the best detection result
|
||||||
|
observed across all substeps for that technique. All tests land in
|
||||||
|
``in_review`` state — Blue Leads must confirm before they count as coverage.
|
||||||
|
|
||||||
|
Returns a summary dict: created, skipped, techniques_covered.
|
||||||
|
Raises if the round was already imported (idempotency guard).
|
||||||
|
"""
|
||||||
|
# Idempotency — refuse duplicate imports
|
||||||
|
existing = (
|
||||||
|
db.query(EvaluationImport)
|
||||||
|
.filter(
|
||||||
|
EvaluationImport.adversary_name == adversary_name.lower(),
|
||||||
|
EvaluationImport.status == "completed",
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
raise ValueError(
|
||||||
|
f"Round '{adversary_display}' (round {eval_round}) was already imported "
|
||||||
|
f"on {existing.imported_at.date()}. Re-import is not allowed."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fetch and aggregate substep results
|
||||||
|
substeps = fetch_results_for_adversary(adversary_name)
|
||||||
|
by_technique = _aggregate_by_technique(substeps)
|
||||||
|
|
||||||
|
created = 0
|
||||||
|
skipped = 0
|
||||||
|
affected_technique_ids: set = set()
|
||||||
|
|
||||||
|
for mitre_id, agg in by_technique.items():
|
||||||
|
# Look up the technique in our DB
|
||||||
|
technique = (
|
||||||
|
db.query(Technique)
|
||||||
|
.filter(Technique.mitre_id == mitre_id.upper())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if technique is None:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
detection_result = _score_to_result(agg["best_score"])
|
||||||
|
|
||||||
|
description = (
|
||||||
|
f"Source: MITRE ATT&CK Evaluation — Round {eval_round} ({adversary_display}).\n"
|
||||||
|
f"Vendor: CrowdStrike Falcon.\n"
|
||||||
|
f"Detection type achieved: {agg['detection_type']}.\n\n"
|
||||||
|
f"⚠️ IMPORTANT: These results reflect CrowdStrike Falcon performance in a "
|
||||||
|
f"controlled MITRE lab environment against a simulated {adversary_display} "
|
||||||
|
f"adversary. They do NOT represent your organisation's actual detection "
|
||||||
|
f"capability. Validate in your own environment before approving."
|
||||||
|
)
|
||||||
|
if agg["note"]:
|
||||||
|
description += f"\n\nMITRE note: {agg['note']}"
|
||||||
|
|
||||||
|
red_summary = (
|
||||||
|
f"MITRE ATT&CK Evaluation — Round {eval_round} ({adversary_display})\n"
|
||||||
|
f"Vendor: CrowdStrike Falcon\n"
|
||||||
|
f"Best detection level: {agg['detection_type']}\n"
|
||||||
|
f"Tactic: {agg['tactic_name']} ({agg['tactic_id']})"
|
||||||
|
)
|
||||||
|
|
||||||
|
test = Test(
|
||||||
|
technique_id=technique.id,
|
||||||
|
name=f"[EVAL R{eval_round}] {adversary_display} — {technique.name}",
|
||||||
|
description=description,
|
||||||
|
platform=None,
|
||||||
|
procedure_text=(
|
||||||
|
f"MITRE ATT&CK Evaluation simulation using {adversary_display} TTPs. "
|
||||||
|
f"See evaluation report at https://evals.mitre.org for full details."
|
||||||
|
),
|
||||||
|
created_by=current_user.id,
|
||||||
|
state=TestState.in_review,
|
||||||
|
attack_success=True,
|
||||||
|
red_summary=red_summary,
|
||||||
|
red_validation_status="approved",
|
||||||
|
red_validated_by=current_user.id,
|
||||||
|
red_validated_at=datetime.utcnow(),
|
||||||
|
detection_result=detection_result,
|
||||||
|
blue_validation_status=None,
|
||||||
|
execution_date=datetime.utcnow(),
|
||||||
|
created_at=datetime.utcnow(),
|
||||||
|
)
|
||||||
|
db.add(test)
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
log_action(
|
||||||
|
db,
|
||||||
|
user_id=current_user.id,
|
||||||
|
action="eval_import_test",
|
||||||
|
entity_type="test",
|
||||||
|
entity_id=test.id,
|
||||||
|
details={
|
||||||
|
"adversary": adversary_name,
|
||||||
|
"eval_round": eval_round,
|
||||||
|
"mitre_id": mitre_id,
|
||||||
|
"detection_type": agg["detection_type"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
affected_technique_ids.add(technique.id)
|
||||||
|
created += 1
|
||||||
|
|
||||||
|
# Recalculate coverage for all touched techniques
|
||||||
|
for tech_id in affected_technique_ids:
|
||||||
|
tech = db.query(Technique).filter(Technique.id == tech_id).first()
|
||||||
|
if tech:
|
||||||
|
recalculate_technique_status(db, tech)
|
||||||
|
|
||||||
|
# Record the import
|
||||||
|
record = EvaluationImport(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
adversary_name=adversary_name.lower(),
|
||||||
|
adversary_display=adversary_display,
|
||||||
|
eval_round=eval_round,
|
||||||
|
imported_at=datetime.utcnow(),
|
||||||
|
imported_by=current_user.id,
|
||||||
|
tests_created=created,
|
||||||
|
techniques_covered=len(affected_technique_ids),
|
||||||
|
status="completed",
|
||||||
|
notes=f"Skipped {skipped} techniques not found in local DB.",
|
||||||
|
)
|
||||||
|
db.add(record)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"ATT&CK Evaluation import complete — round %d (%s): %d tests created, %d skipped",
|
||||||
|
eval_round, adversary_display, created, skipped,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"created": created,
|
||||||
|
"skipped": skipped,
|
||||||
|
"techniques_covered": len(affected_technique_ids),
|
||||||
|
"adversary": adversary_display,
|
||||||
|
"eval_round": eval_round,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# New-round check (used by the weekly scheduler)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def check_for_new_round(db: Session) -> dict[str, Any]:
|
||||||
|
"""Check if a new evaluation round is available that hasn't been imported yet.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{"new_round_available": bool, "latest_round": dict | None, "already_imported": bool}
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
latest = get_latest_round()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Could not check for new ATT&CK Evaluation round: %s", exc)
|
||||||
|
return {"new_round_available": False, "latest_round": None, "error": str(exc)}
|
||||||
|
|
||||||
|
already = (
|
||||||
|
db.query(EvaluationImport)
|
||||||
|
.filter(
|
||||||
|
EvaluationImport.adversary_name == latest["name"].lower(),
|
||||||
|
EvaluationImport.status == "completed",
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"new_round_available": already is None,
|
||||||
|
"already_imported": already is not None,
|
||||||
|
"latest_round": {
|
||||||
|
"name": latest["name"],
|
||||||
|
"display_name": latest.get("display_name", latest["name"]),
|
||||||
|
"eval_round": latest["eval_round"],
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -40,3 +40,59 @@ export async function getSchedulerStatus(): Promise<SchedulerStatusResponse> {
|
|||||||
const { data } = await client.get<SchedulerStatusResponse>("/system/scheduler-status");
|
const { data } = await client.get<SchedulerStatusResponse>("/system/scheduler-status");
|
||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── ATT&CK Evaluations ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
export interface EvaluationRound {
|
||||||
|
name: string;
|
||||||
|
display_name: string;
|
||||||
|
eval_round: number;
|
||||||
|
imported: boolean;
|
||||||
|
imported_at: string | null;
|
||||||
|
tests_created: number | null;
|
||||||
|
techniques_covered: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EvaluationImportResult {
|
||||||
|
message: string;
|
||||||
|
created: number;
|
||||||
|
skipped: number;
|
||||||
|
techniques_covered: number;
|
||||||
|
adversary: string;
|
||||||
|
eval_round: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NewRoundCheckResult {
|
||||||
|
new_round_available: boolean;
|
||||||
|
already_imported: boolean;
|
||||||
|
latest_round: { name: string; display_name: string; eval_round: number } | null;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** List all public CrowdStrike evaluation rounds with import status. */
|
||||||
|
export async function listEvaluationRounds(): Promise<EvaluationRound[]> {
|
||||||
|
const { data } = await client.get<EvaluationRound[]>("/system/attck-evaluations/rounds");
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Import a specific evaluation round. */
|
||||||
|
export async function importEvaluationRound(payload: {
|
||||||
|
adversary_name: string;
|
||||||
|
adversary_display: string;
|
||||||
|
eval_round: number;
|
||||||
|
}): Promise<EvaluationImportResult> {
|
||||||
|
const { data } = await client.post<EvaluationImportResult>("/system/attck-evaluations/import", payload);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Import the latest available round automatically. */
|
||||||
|
export async function importLatestEvaluation(): Promise<EvaluationImportResult> {
|
||||||
|
const { data } = await client.post<EvaluationImportResult>("/system/attck-evaluations/import-latest");
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Check if a new round is available. */
|
||||||
|
export async function checkNewEvaluationRound(): Promise<NewRoundCheckResult> {
|
||||||
|
const { data } = await client.get<NewRoundCheckResult>("/system/attck-evaluations/check-new");
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|||||||
@@ -21,14 +21,26 @@ import {
|
|||||||
Download,
|
Download,
|
||||||
Upload,
|
Upload,
|
||||||
PackageOpen,
|
PackageOpen,
|
||||||
|
Swords,
|
||||||
|
Sparkles,
|
||||||
|
AlertTriangle,
|
||||||
|
ExternalLink,
|
||||||
|
CalendarCheck,
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
import client from "../api/client";
|
import client from "../api/client";
|
||||||
import {
|
import {
|
||||||
triggerMitreSync,
|
triggerMitreSync,
|
||||||
triggerIntelScan,
|
triggerIntelScan,
|
||||||
getSchedulerStatus,
|
getSchedulerStatus,
|
||||||
|
listEvaluationRounds,
|
||||||
|
importEvaluationRound,
|
||||||
|
importLatestEvaluation,
|
||||||
|
checkNewEvaluationRound,
|
||||||
type SyncMitreResponse,
|
type SyncMitreResponse,
|
||||||
type IntelScanResponse,
|
type IntelScanResponse,
|
||||||
|
type EvaluationRound,
|
||||||
|
type EvaluationImportResult,
|
||||||
|
type NewRoundCheckResult,
|
||||||
} from "../api/system";
|
} from "../api/system";
|
||||||
import {
|
import {
|
||||||
getTemplateStats,
|
getTemplateStats,
|
||||||
@@ -51,6 +63,11 @@ export default function SystemPage() {
|
|||||||
const [bulkConfirm, setBulkConfirm] = useState<"activate" | "deactivate" | null>(null);
|
const [bulkConfirm, setBulkConfirm] = useState<"activate" | "deactivate" | null>(null);
|
||||||
const [selectedTemplateId, setSelectedTemplateId] = useState<string | null>(null);
|
const [selectedTemplateId, setSelectedTemplateId] = useState<string | null>(null);
|
||||||
|
|
||||||
|
// ── ATT&CK Evaluations state ─────────────────────────────────────
|
||||||
|
const [evalImportResult, setEvalImportResult] = useState<EvaluationImportResult | null>(null);
|
||||||
|
const [evalCheckResult, setEvalCheckResult] = useState<NewRoundCheckResult | null>(null);
|
||||||
|
const [evalImportingRound, setEvalImportingRound] = useState<string | null>(null);
|
||||||
|
|
||||||
// ── Existing queries ─────────────────────────────────────────────
|
// ── Existing queries ─────────────────────────────────────────────
|
||||||
const {
|
const {
|
||||||
data: schedulerStatus,
|
data: schedulerStatus,
|
||||||
@@ -145,6 +162,53 @@ export default function SystemPage() {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── ATT&CK Evaluations queries & mutations ───────────────────────
|
||||||
|
const {
|
||||||
|
data: evalRounds,
|
||||||
|
isLoading: evalRoundsLoading,
|
||||||
|
refetch: refetchEvalRounds,
|
||||||
|
} = useQuery<EvaluationRound[]>({
|
||||||
|
queryKey: ["eval-rounds"],
|
||||||
|
queryFn: listEvaluationRounds,
|
||||||
|
});
|
||||||
|
|
||||||
|
const checkNewRoundMutation = useMutation({
|
||||||
|
mutationFn: checkNewEvaluationRound,
|
||||||
|
onSuccess: (data) => {
|
||||||
|
setEvalCheckResult(data);
|
||||||
|
refetchEvalRounds();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const importLatestMutation = useMutation({
|
||||||
|
mutationFn: importLatestEvaluation,
|
||||||
|
onSuccess: (data) => {
|
||||||
|
setEvalImportResult(data);
|
||||||
|
setEvalImportingRound(null);
|
||||||
|
refetchEvalRounds();
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["techniques"] });
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["metrics"] });
|
||||||
|
},
|
||||||
|
onError: () => {
|
||||||
|
setEvalImportingRound(null);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const importRoundMutation = useMutation({
|
||||||
|
mutationFn: (payload: { adversary_name: string; adversary_display: string; eval_round: number }) =>
|
||||||
|
importEvaluationRound(payload),
|
||||||
|
onSuccess: (data) => {
|
||||||
|
setEvalImportResult(data);
|
||||||
|
setEvalImportingRound(null);
|
||||||
|
refetchEvalRounds();
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["techniques"] });
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["metrics"] });
|
||||||
|
},
|
||||||
|
onError: () => {
|
||||||
|
setEvalImportingRound(null);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
const formatNextRun = (dateStr: string | null) => {
|
const formatNextRun = (dateStr: string | null) => {
|
||||||
if (!dateStr) return "Not scheduled";
|
if (!dateStr) return "Not scheduled";
|
||||||
const date = new Date(dateStr);
|
const date = new Date(dateStr);
|
||||||
@@ -294,6 +358,264 @@ export default function SystemPage() {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* ────────────────────────────────────────────────────────────────
|
||||||
|
ATT&CK EVALUATIONS — CrowdStrike
|
||||||
|
──────────────────────────────────────────────────────────────── */}
|
||||||
|
<div className="rounded-xl border border-orange-500/30 bg-gray-900 p-6">
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex flex-wrap items-start justify-between gap-4 mb-5">
|
||||||
|
<div className="flex items-start gap-4">
|
||||||
|
<div className="rounded-lg bg-orange-500/10 p-3 mt-0.5">
|
||||||
|
<Swords className="h-6 w-6 text-orange-400" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<h2 className="text-lg font-semibold text-white">
|
||||||
|
ATT&CK Evaluations — CrowdStrike Falcon
|
||||||
|
</h2>
|
||||||
|
<p className="mt-1 text-sm text-gray-400 max-w-2xl">
|
||||||
|
Seed the platform with real detection data from MITRE Engenuity's public ATT&CK
|
||||||
|
Evaluations. Results are imported as <span className="text-yellow-400 font-medium">In Review</span> tests —
|
||||||
|
Blue Leads must validate each one before it counts as coverage.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2 flex-shrink-0">
|
||||||
|
<button
|
||||||
|
onClick={() => checkNewRoundMutation.mutate()}
|
||||||
|
disabled={checkNewRoundMutation.isPending || evalRoundsLoading}
|
||||||
|
className="flex items-center gap-2 rounded-lg border border-gray-700 bg-gray-800 px-3 py-2 text-sm font-medium text-gray-300 hover:bg-gray-700 disabled:opacity-50 transition-colors"
|
||||||
|
>
|
||||||
|
{checkNewRoundMutation.isPending ? (
|
||||||
|
<Loader2 className="h-4 w-4 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<RefreshCw className="h-4 w-4" />
|
||||||
|
)}
|
||||||
|
Check for new rounds
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={() => {
|
||||||
|
setEvalImportingRound("latest");
|
||||||
|
importLatestMutation.mutate();
|
||||||
|
}}
|
||||||
|
disabled={importLatestMutation.isPending || importRoundMutation.isPending}
|
||||||
|
className="flex items-center gap-2 rounded-lg bg-orange-600 px-3 py-2 text-sm font-medium text-white hover:bg-orange-500 disabled:opacity-50 transition-colors"
|
||||||
|
>
|
||||||
|
{importLatestMutation.isPending && evalImportingRound === "latest" ? (
|
||||||
|
<Loader2 className="h-4 w-4 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<Sparkles className="h-4 w-4" />
|
||||||
|
)}
|
||||||
|
Import Latest Round
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Disclaimer callout */}
|
||||||
|
<div className="mb-5 flex items-start gap-3 rounded-lg border border-yellow-500/30 bg-yellow-900/10 p-4">
|
||||||
|
<AlertTriangle className="h-5 w-5 text-yellow-400 flex-shrink-0 mt-0.5" />
|
||||||
|
<div className="text-sm">
|
||||||
|
<p className="font-medium text-yellow-400 mb-1">Lab environment data — validation required</p>
|
||||||
|
<p className="text-gray-400">
|
||||||
|
These results reflect CrowdStrike Falcon performance in a controlled MITRE lab against
|
||||||
|
simulated adversaries. They do <strong className="text-white">not</strong> represent
|
||||||
|
your organisation's actual detection capability. All imported tests require Blue Lead
|
||||||
|
validation before contributing to programme coverage.
|
||||||
|
</p>
|
||||||
|
<a
|
||||||
|
href="https://evals.mitre.org"
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
className="mt-2 inline-flex items-center gap-1 text-xs text-orange-400 hover:text-orange-300 transition-colors"
|
||||||
|
>
|
||||||
|
<ExternalLink className="h-3 w-3" />
|
||||||
|
View official evaluation reports at evals.mitre.org
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* New round check result */}
|
||||||
|
{evalCheckResult && (
|
||||||
|
<div className={`mb-4 rounded-lg border p-3 ${
|
||||||
|
evalCheckResult.error
|
||||||
|
? "border-red-500/30 bg-red-900/20"
|
||||||
|
: evalCheckResult.new_round_available
|
||||||
|
? "border-green-500/30 bg-green-900/20"
|
||||||
|
: "border-gray-700 bg-gray-800/50"
|
||||||
|
}`}>
|
||||||
|
{evalCheckResult.error ? (
|
||||||
|
<div className="flex items-center gap-2 text-sm text-red-400">
|
||||||
|
<XCircle className="h-4 w-4 flex-shrink-0" />
|
||||||
|
<span>Check failed: {evalCheckResult.error}</span>
|
||||||
|
</div>
|
||||||
|
) : evalCheckResult.new_round_available ? (
|
||||||
|
<div className="flex items-center gap-2 text-sm">
|
||||||
|
<CheckCircle className="h-4 w-4 text-green-400 flex-shrink-0" />
|
||||||
|
<span className="text-green-400 font-medium">
|
||||||
|
New round available: {evalCheckResult.latest_round?.display_name} (Round {evalCheckResult.latest_round?.eval_round})
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="flex items-center gap-2 text-sm">
|
||||||
|
<CalendarCheck className="h-4 w-4 text-gray-400 flex-shrink-0" />
|
||||||
|
<span className="text-gray-400">
|
||||||
|
Up to date — latest round ({evalCheckResult.latest_round?.display_name}) is already imported.
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Import result feedback */}
|
||||||
|
{evalImportResult && (
|
||||||
|
<div className="mb-4 rounded-lg border border-green-500/30 bg-green-900/20 p-4">
|
||||||
|
<div className="flex items-center gap-2 mb-2">
|
||||||
|
<CheckCircle className="h-4 w-4 text-green-400" />
|
||||||
|
<span className="text-sm font-medium text-green-400">Import complete</span>
|
||||||
|
</div>
|
||||||
|
<div className="grid grid-cols-3 gap-3 text-center text-sm">
|
||||||
|
<div>
|
||||||
|
<p className="text-xl font-bold text-white">{evalImportResult.created}</p>
|
||||||
|
<p className="text-xs text-gray-400">Tests created</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-xl font-bold text-white">{evalImportResult.techniques_covered}</p>
|
||||||
|
<p className="text-xs text-gray-400">Techniques covered</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium text-orange-400 truncate">{evalImportResult.adversary}</p>
|
||||||
|
<p className="text-xs text-gray-400">Round {evalImportResult.eval_round}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p className="mt-3 text-xs text-yellow-400 flex items-center gap-1">
|
||||||
|
<AlertTriangle className="h-3 w-3" />
|
||||||
|
All tests are in <strong className="ml-1">Review Queue</strong> — Blue Leads must validate before they count as coverage.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Import error */}
|
||||||
|
{(importLatestMutation.isError || importRoundMutation.isError) && (
|
||||||
|
<div className="mb-4 rounded-lg border border-red-500/30 bg-red-900/20 p-3">
|
||||||
|
<div className="flex items-center gap-2 text-sm text-red-400">
|
||||||
|
<XCircle className="h-4 w-4 flex-shrink-0" />
|
||||||
|
<span>
|
||||||
|
{((importLatestMutation.error || importRoundMutation.error) as Error)?.message ??
|
||||||
|
"Import failed. This round may already be imported."}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Rounds table */}
|
||||||
|
<div>
|
||||||
|
<p className="mb-3 text-xs font-semibold uppercase tracking-wider text-gray-500">
|
||||||
|
Available Rounds
|
||||||
|
</p>
|
||||||
|
{evalRoundsLoading ? (
|
||||||
|
<div className="flex items-center justify-center py-8">
|
||||||
|
<Loader2 className="h-6 w-6 animate-spin text-orange-400" />
|
||||||
|
</div>
|
||||||
|
) : evalRounds && evalRounds.length > 0 ? (
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full text-left text-sm">
|
||||||
|
<thead>
|
||||||
|
<tr className="border-b border-gray-800">
|
||||||
|
<th className="pb-3 pr-4 font-medium text-gray-400">Round</th>
|
||||||
|
<th className="pb-3 px-4 font-medium text-gray-400">Adversary</th>
|
||||||
|
<th className="pb-3 px-4 font-medium text-gray-400">Status</th>
|
||||||
|
<th className="pb-3 px-4 font-medium text-gray-400">Imported</th>
|
||||||
|
<th className="pb-3 px-4 font-medium text-gray-400">Tests</th>
|
||||||
|
<th className="pb-3 pl-4 font-medium text-gray-400">Action</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{evalRounds.map((round) => (
|
||||||
|
<tr
|
||||||
|
key={round.name}
|
||||||
|
className="border-b border-gray-800/50 hover:bg-gray-800/30 transition-colors"
|
||||||
|
>
|
||||||
|
<td className="py-3 pr-4">
|
||||||
|
<span className="inline-flex items-center justify-center rounded-full border border-orange-500/30 bg-orange-900/20 px-2.5 py-0.5 text-xs font-bold text-orange-400">
|
||||||
|
R{round.eval_round}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4">
|
||||||
|
<p className="font-medium text-gray-200">{round.display_name}</p>
|
||||||
|
<p className="text-xs text-gray-500 font-mono">{round.name}</p>
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4">
|
||||||
|
{round.imported ? (
|
||||||
|
<span className="inline-flex items-center gap-1.5 rounded-full border border-green-500/30 bg-green-900/20 px-2.5 py-0.5 text-xs font-medium text-green-400">
|
||||||
|
<CheckCircle className="h-3 w-3" />
|
||||||
|
Imported
|
||||||
|
</span>
|
||||||
|
) : (
|
||||||
|
<span className="inline-flex items-center gap-1.5 rounded-full border border-gray-600 bg-gray-800/50 px-2.5 py-0.5 text-xs font-medium text-gray-400">
|
||||||
|
Not imported
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4 text-xs text-gray-400">
|
||||||
|
{round.imported_at
|
||||||
|
? new Date(round.imported_at).toLocaleDateString("en-GB", {
|
||||||
|
day: "numeric",
|
||||||
|
month: "short",
|
||||||
|
year: "numeric",
|
||||||
|
})
|
||||||
|
: "—"}
|
||||||
|
</td>
|
||||||
|
<td className="py-3 px-4 text-center">
|
||||||
|
{round.imported ? (
|
||||||
|
<div className="text-center">
|
||||||
|
<p className="text-sm font-medium text-gray-200">{round.tests_created ?? "—"}</p>
|
||||||
|
<p className="text-xs text-gray-500">{round.techniques_covered ?? ""} techniques</p>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<span className="text-gray-600">—</span>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
<td className="py-3 pl-4">
|
||||||
|
{round.imported ? (
|
||||||
|
<span className="text-xs text-gray-600 italic">Already imported</span>
|
||||||
|
) : (
|
||||||
|
<button
|
||||||
|
onClick={() => {
|
||||||
|
setEvalImportingRound(round.name);
|
||||||
|
importRoundMutation.mutate({
|
||||||
|
adversary_name: round.name,
|
||||||
|
adversary_display: round.display_name,
|
||||||
|
eval_round: round.eval_round,
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
disabled={
|
||||||
|
importLatestMutation.isPending ||
|
||||||
|
importRoundMutation.isPending
|
||||||
|
}
|
||||||
|
className="flex items-center gap-1.5 rounded-lg border border-orange-500/30 bg-orange-900/20 px-3 py-1.5 text-xs font-medium text-orange-400 hover:bg-orange-900/40 disabled:opacity-50 transition-colors"
|
||||||
|
>
|
||||||
|
{importRoundMutation.isPending && evalImportingRound === round.name ? (
|
||||||
|
<Loader2 className="h-3.5 w-3.5 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<Download className="h-3.5 w-3.5" />
|
||||||
|
)}
|
||||||
|
Import
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="flex items-center justify-center gap-2 py-8 text-gray-500 text-sm">
|
||||||
|
<AlertCircle className="h-4 w-4" />
|
||||||
|
<span>Unable to load evaluation rounds. Check network connectivity to evals.mitre.org.</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
{/* ────────────────────────────────────────────────────────────────
|
{/* ────────────────────────────────────────────────────────────────
|
||||||
TEMPLATE ADMINISTRATION (T-124)
|
TEMPLATE ADMINISTRATION (T-124)
|
||||||
──────────────────────────────────────────────────────────────── */}
|
──────────────────────────────────────────────────────────────── */}
|
||||||
|
|||||||
Reference in New Issue
Block a user