Compare commits
7 Commits
0b65f51d1c
...
93fde55389
| Author | SHA1 | Date | |
|---|---|---|---|
| 93fde55389 | |||
| 560fc0c9f0 | |||
| d305db8794 | |||
| 25fddad17c | |||
| 8d5c5fa80e | |||
| 42a9f4dcd4 | |||
| 2b6d9090c9 |
File diff suppressed because it is too large
Load Diff
@@ -46,6 +46,8 @@ class TechniqueEntity:
|
|||||||
status_global: TechniqueStatus = TechniqueStatus.not_evaluated
|
status_global: TechniqueStatus = TechniqueStatus.not_evaluated
|
||||||
review_required: bool = False
|
review_required: bool = False
|
||||||
last_review_date: datetime | None = None
|
last_review_date: datetime | None = None
|
||||||
|
mitre_version: str | None = None
|
||||||
|
mitre_last_modified: datetime | None = None
|
||||||
|
|
||||||
# -- Factory -----------------------------------------------------------
|
# -- Factory -----------------------------------------------------------
|
||||||
|
|
||||||
@@ -77,11 +79,12 @@ class TechniqueEntity:
|
|||||||
def from_orm(cls, model: Any) -> TechniqueEntity:
|
def from_orm(cls, model: Any) -> TechniqueEntity:
|
||||||
"""Build a TechniqueEntity from a SQLAlchemy Technique model."""
|
"""Build a TechniqueEntity from a SQLAlchemy Technique model."""
|
||||||
raw_status = model.status_global
|
raw_status = model.status_global
|
||||||
status = (
|
if raw_status is None:
|
||||||
raw_status
|
status = TechniqueStatus.not_evaluated
|
||||||
if isinstance(raw_status, TechniqueStatus)
|
elif isinstance(raw_status, TechniqueStatus):
|
||||||
else TechniqueStatus(raw_status)
|
status = raw_status
|
||||||
)
|
else:
|
||||||
|
status = TechniqueStatus(raw_status)
|
||||||
return cls(
|
return cls(
|
||||||
id=model.id,
|
id=model.id,
|
||||||
mitre_id=model.mitre_id,
|
mitre_id=model.mitre_id,
|
||||||
@@ -94,6 +97,8 @@ class TechniqueEntity:
|
|||||||
status_global=status,
|
status_global=status,
|
||||||
review_required=model.review_required or False,
|
review_required=model.review_required or False,
|
||||||
last_review_date=model.last_review_date,
|
last_review_date=model.last_review_date,
|
||||||
|
mitre_version=getattr(model, "mitre_version", None),
|
||||||
|
mitre_last_modified=getattr(model, "mitre_last_modified", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
def apply_to(self, model: Any) -> None:
|
def apply_to(self, model: Any) -> None:
|
||||||
|
|||||||
@@ -163,6 +163,8 @@ class SATechniqueRepository:
|
|||||||
existing.platforms = technique.platforms
|
existing.platforms = technique.platforms
|
||||||
existing.is_subtechnique = technique.is_subtechnique
|
existing.is_subtechnique = technique.is_subtechnique
|
||||||
existing.parent_mitre_id = technique.parent_mitre_id
|
existing.parent_mitre_id = technique.parent_mitre_id
|
||||||
|
existing.mitre_version = technique.mitre_version
|
||||||
|
existing.mitre_last_modified = technique.mitre_last_modified
|
||||||
self._session.flush()
|
self._session.flush()
|
||||||
return TechniqueMapper.to_entity(existing)
|
return TechniqueMapper.to_entity(existing)
|
||||||
else:
|
else:
|
||||||
@@ -178,6 +180,8 @@ class SATechniqueRepository:
|
|||||||
status_global=technique.status_global,
|
status_global=technique.status_global,
|
||||||
review_required=technique.review_required,
|
review_required=technique.review_required,
|
||||||
last_review_date=technique.last_review_date,
|
last_review_date=technique.last_review_date,
|
||||||
|
mitre_version=technique.mitre_version,
|
||||||
|
mitre_last_modified=technique.mitre_last_modified,
|
||||||
)
|
)
|
||||||
self._session.add(model)
|
self._session.add(model)
|
||||||
self._session.flush()
|
self._session.flush()
|
||||||
|
|||||||
@@ -1,29 +1,24 @@
|
|||||||
"""Compliance endpoints — framework status, reports, and gap analysis.
|
"""Compliance endpoints — framework status, reports, and gap analysis.
|
||||||
|
|
||||||
|
Thin HTTP adapter: delegates all data logic to compliance_service.
|
||||||
|
|
||||||
Provides compliance posture assessment by mapping MITRE ATT&CK technique
|
Provides compliance posture assessment by mapping MITRE ATT&CK technique
|
||||||
coverage to compliance framework controls.
|
coverage to compliance framework controls.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import csv
|
from fastapi import APIRouter, Depends
|
||||||
import io
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
from sqlalchemy.orm import Session, joinedload
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.dependencies.auth import get_current_user, require_role
|
from app.dependencies.auth import get_current_user, require_role
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.models.compliance import (
|
from app.services.compliance_service import (
|
||||||
ComplianceFramework,
|
list_frameworks,
|
||||||
ComplianceControl,
|
get_framework_status,
|
||||||
ComplianceControlMapping,
|
build_framework_report_csv,
|
||||||
|
get_framework_gaps,
|
||||||
)
|
)
|
||||||
from app.models.technique import Technique
|
|
||||||
from app.models.test_template import TestTemplate
|
|
||||||
from app.models.threat_actor import ThreatActorTechnique
|
|
||||||
from app.services.scoring_service import calculate_technique_score
|
|
||||||
from app.services.compliance_import_service import (
|
from app.services.compliance_import_service import (
|
||||||
import_nist_800_53_mappings,
|
import_nist_800_53_mappings,
|
||||||
import_cis_controls_v8_mappings,
|
import_cis_controls_v8_mappings,
|
||||||
@@ -32,126 +27,16 @@ from app.services.compliance_import_service import (
|
|||||||
router = APIRouter(prefix="/compliance", tags=["compliance"])
|
router = APIRouter(prefix="/compliance", tags=["compliance"])
|
||||||
|
|
||||||
|
|
||||||
# ── Helpers ───────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
|
|
||||||
def _classify_control(technique_scores: list[float]) -> str:
|
|
||||||
"""Classify a control status based on its technique scores."""
|
|
||||||
if not technique_scores:
|
|
||||||
return "not_evaluated"
|
|
||||||
|
|
||||||
all_above_70 = all(s >= 70 for s in technique_scores)
|
|
||||||
any_above_30 = any(s >= 30 for s in technique_scores)
|
|
||||||
all_below_30 = all(s < 30 for s in technique_scores)
|
|
||||||
all_zero = all(s == 0 for s in technique_scores)
|
|
||||||
|
|
||||||
if all_zero:
|
|
||||||
return "not_evaluated"
|
|
||||||
if all_above_70:
|
|
||||||
return "covered"
|
|
||||||
if all_below_30:
|
|
||||||
return "not_covered"
|
|
||||||
if any_above_30:
|
|
||||||
return "partially_covered"
|
|
||||||
return "not_covered"
|
|
||||||
|
|
||||||
|
|
||||||
def _get_control_status(control: ComplianceControl, db: Session) -> dict:
|
|
||||||
"""Compute the status and score for a single control."""
|
|
||||||
mappings = (
|
|
||||||
db.query(ComplianceControlMapping)
|
|
||||||
.filter(ComplianceControlMapping.compliance_control_id == control.id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
if not mappings:
|
|
||||||
return {
|
|
||||||
"control_id": control.control_id,
|
|
||||||
"title": control.title,
|
|
||||||
"category": control.category,
|
|
||||||
"status": "not_evaluated",
|
|
||||||
"score": 0,
|
|
||||||
"techniques_count": 0,
|
|
||||||
"techniques_covered": 0,
|
|
||||||
"techniques": [],
|
|
||||||
}
|
|
||||||
|
|
||||||
technique_ids = [m.technique_id for m in mappings]
|
|
||||||
techniques = (
|
|
||||||
db.query(Technique)
|
|
||||||
.filter(Technique.id.in_(technique_ids))
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
tech_details = []
|
|
||||||
scores = []
|
|
||||||
covered_count = 0
|
|
||||||
|
|
||||||
for tech in techniques:
|
|
||||||
result = calculate_technique_score(tech, db)
|
|
||||||
score = result["total_score"]
|
|
||||||
scores.append(score)
|
|
||||||
if score >= 50:
|
|
||||||
covered_count += 1
|
|
||||||
|
|
||||||
tech_details.append({
|
|
||||||
"mitre_id": tech.mitre_id,
|
|
||||||
"name": tech.name,
|
|
||||||
"score": score,
|
|
||||||
"status": tech.status_global.value if tech.status_global else "not_evaluated",
|
|
||||||
})
|
|
||||||
|
|
||||||
# Sort techniques by score ascending (worst first for priority)
|
|
||||||
tech_details.sort(key=lambda t: t["score"])
|
|
||||||
|
|
||||||
avg_score = round(sum(scores) / len(scores), 1) if scores else 0
|
|
||||||
status = _classify_control(scores)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"control_id": control.control_id,
|
|
||||||
"title": control.title,
|
|
||||||
"category": control.category,
|
|
||||||
"status": status,
|
|
||||||
"score": avg_score,
|
|
||||||
"techniques_count": len(techniques),
|
|
||||||
"techniques_covered": covered_count,
|
|
||||||
"techniques": tech_details,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ── GET /compliance/frameworks ────────────────────────────────────────
|
# ── GET /compliance/frameworks ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
@router.get("/frameworks")
|
@router.get("/frameworks")
|
||||||
def list_frameworks(
|
def list_frameworks_endpoint(
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""List all available compliance frameworks."""
|
"""List all available compliance frameworks."""
|
||||||
frameworks = (
|
return list_frameworks(db)
|
||||||
db.query(ComplianceFramework)
|
|
||||||
.filter(ComplianceFramework.is_active == True)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
result = []
|
|
||||||
for fw in frameworks:
|
|
||||||
control_count = (
|
|
||||||
db.query(ComplianceControl)
|
|
||||||
.filter(ComplianceControl.framework_id == fw.id)
|
|
||||||
.count()
|
|
||||||
)
|
|
||||||
result.append({
|
|
||||||
"id": str(fw.id),
|
|
||||||
"name": fw.name,
|
|
||||||
"version": fw.version,
|
|
||||||
"description": fw.description,
|
|
||||||
"url": fw.url,
|
|
||||||
"is_active": fw.is_active,
|
|
||||||
"controls_count": control_count,
|
|
||||||
})
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ── GET /compliance/frameworks/{id}/status ────────────────────────────
|
# ── GET /compliance/frameworks/{id}/status ────────────────────────────
|
||||||
@@ -164,55 +49,7 @@ def framework_status(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Get compliance status for each control in a framework."""
|
"""Get compliance status for each control in a framework."""
|
||||||
framework = (
|
return get_framework_status(db, framework_id)
|
||||||
db.query(ComplianceFramework)
|
|
||||||
.filter(ComplianceFramework.id == framework_id)
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
if not framework:
|
|
||||||
raise HTTPException(status_code=404, detail="Framework not found")
|
|
||||||
|
|
||||||
controls = (
|
|
||||||
db.query(ComplianceControl)
|
|
||||||
.filter(ComplianceControl.framework_id == framework.id)
|
|
||||||
.order_by(ComplianceControl.control_id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
control_statuses = []
|
|
||||||
summary = {
|
|
||||||
"total_controls": len(controls),
|
|
||||||
"covered": 0,
|
|
||||||
"partially_covered": 0,
|
|
||||||
"not_covered": 0,
|
|
||||||
"not_evaluated": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
for control in controls:
|
|
||||||
status_data = _get_control_status(control, db)
|
|
||||||
control_statuses.append(status_data)
|
|
||||||
|
|
||||||
status = status_data["status"]
|
|
||||||
if status in summary:
|
|
||||||
summary[status] += 1
|
|
||||||
|
|
||||||
# Compliance percentage: (covered + partially_covered*0.5) / total * 100
|
|
||||||
total = summary["total_controls"]
|
|
||||||
if total > 0:
|
|
||||||
compliance_pct = round(
|
|
||||||
(summary["covered"] + summary["partially_covered"] * 0.5) / total * 100,
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
compliance_pct = 0
|
|
||||||
|
|
||||||
summary["compliance_percentage"] = compliance_pct
|
|
||||||
|
|
||||||
return {
|
|
||||||
"framework": {"id": str(framework.id), "name": framework.name},
|
|
||||||
"summary": summary,
|
|
||||||
"controls": control_statuses,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ── GET /compliance/frameworks/{id}/report ────────────────────────────
|
# ── GET /compliance/frameworks/{id}/report ────────────────────────────
|
||||||
@@ -225,7 +62,7 @@ def framework_report(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Get the full compliance report (same as status but marked as report)."""
|
"""Get the full compliance report (same as status but marked as report)."""
|
||||||
return framework_status(framework_id, db=db, current_user=current_user)
|
return get_framework_status(db, framework_id)
|
||||||
|
|
||||||
|
|
||||||
# ── GET /compliance/frameworks/{id}/report/csv ────────────────────────
|
# ── GET /compliance/frameworks/{id}/report/csv ────────────────────────
|
||||||
@@ -238,53 +75,9 @@ def framework_report_csv(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Export compliance report as CSV."""
|
"""Export compliance report as CSV."""
|
||||||
framework = (
|
csv_bytes, filename = build_framework_report_csv(db, framework_id)
|
||||||
db.query(ComplianceFramework)
|
|
||||||
.filter(ComplianceFramework.id == framework_id)
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
if not framework:
|
|
||||||
raise HTTPException(status_code=404, detail="Framework not found")
|
|
||||||
|
|
||||||
controls = (
|
|
||||||
db.query(ComplianceControl)
|
|
||||||
.filter(ComplianceControl.framework_id == framework.id)
|
|
||||||
.order_by(ComplianceControl.control_id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
output = io.StringIO()
|
|
||||||
writer = csv.writer(output)
|
|
||||||
writer.writerow([
|
|
||||||
"control_id",
|
|
||||||
"title",
|
|
||||||
"category",
|
|
||||||
"status",
|
|
||||||
"score",
|
|
||||||
"techniques_total",
|
|
||||||
"techniques_covered",
|
|
||||||
"technique_ids",
|
|
||||||
])
|
|
||||||
|
|
||||||
for control in controls:
|
|
||||||
status_data = _get_control_status(control, db)
|
|
||||||
technique_ids = ",".join(t["mitre_id"] for t in status_data["techniques"])
|
|
||||||
writer.writerow([
|
|
||||||
status_data["control_id"],
|
|
||||||
status_data["title"],
|
|
||||||
status_data["category"] or "",
|
|
||||||
status_data["status"],
|
|
||||||
status_data["score"],
|
|
||||||
status_data["techniques_count"],
|
|
||||||
status_data["techniques_covered"],
|
|
||||||
technique_ids,
|
|
||||||
])
|
|
||||||
|
|
||||||
output.seek(0)
|
|
||||||
filename = f"compliance_{framework.name.replace(' ', '_')}.csv"
|
|
||||||
|
|
||||||
return StreamingResponse(
|
return StreamingResponse(
|
||||||
io.BytesIO(output.getvalue().encode("utf-8")),
|
iter([csv_bytes]),
|
||||||
media_type="text/csv",
|
media_type="text/csv",
|
||||||
headers={
|
headers={
|
||||||
"Content-Disposition": f"attachment; filename={filename}",
|
"Content-Disposition": f"attachment; filename={filename}",
|
||||||
@@ -302,75 +95,10 @@ def framework_gaps(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Get controls with techniques that are not adequately covered."""
|
"""Get controls with techniques that are not adequately covered."""
|
||||||
framework = (
|
return get_framework_gaps(db, framework_id)
|
||||||
db.query(ComplianceFramework)
|
|
||||||
.filter(ComplianceFramework.id == framework_id)
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
if not framework:
|
|
||||||
raise HTTPException(status_code=404, detail="Framework not found")
|
|
||||||
|
|
||||||
controls = (
|
|
||||||
db.query(ComplianceControl)
|
|
||||||
.filter(ComplianceControl.framework_id == framework.id)
|
|
||||||
.order_by(ComplianceControl.control_id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
gaps = []
|
|
||||||
for control in controls:
|
|
||||||
status_data = _get_control_status(control, db)
|
|
||||||
|
|
||||||
if status_data["status"] in ("not_covered", "partially_covered"):
|
|
||||||
# Find uncovered techniques
|
|
||||||
uncovered_techniques = []
|
|
||||||
for tech_info in status_data["techniques"]:
|
|
||||||
if tech_info["score"] < 70:
|
|
||||||
# Count available templates
|
|
||||||
template_count = (
|
|
||||||
db.query(TestTemplate)
|
|
||||||
.filter(TestTemplate.mitre_technique_id == tech_info["mitre_id"])
|
|
||||||
.count()
|
|
||||||
)
|
|
||||||
|
|
||||||
# Count threat actors using this technique
|
|
||||||
technique = (
|
|
||||||
db.query(Technique)
|
|
||||||
.filter(Technique.mitre_id == tech_info["mitre_id"])
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
actor_count = 0
|
|
||||||
if technique:
|
|
||||||
actor_count = (
|
|
||||||
db.query(ThreatActorTechnique)
|
|
||||||
.filter(ThreatActorTechnique.technique_id == technique.id)
|
|
||||||
.count()
|
|
||||||
)
|
|
||||||
|
|
||||||
uncovered_techniques.append({
|
|
||||||
**tech_info,
|
|
||||||
"templates_available": template_count,
|
|
||||||
"threat_actors_using": actor_count,
|
|
||||||
})
|
|
||||||
|
|
||||||
if uncovered_techniques:
|
|
||||||
gaps.append({
|
|
||||||
"control_id": status_data["control_id"],
|
|
||||||
"title": status_data["title"],
|
|
||||||
"category": status_data["category"],
|
|
||||||
"status": status_data["status"],
|
|
||||||
"score": status_data["score"],
|
|
||||||
"uncovered_techniques": uncovered_techniques,
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
"framework": {"id": str(framework.id), "name": framework.name},
|
|
||||||
"total_gaps": len(gaps),
|
|
||||||
"gaps": gaps,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ── POST /compliance/import/nist-800-53 ──────────────────────────────
|
# ── POST /compliance/import/... ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
@router.post("/import/nist-800-53")
|
@router.post("/import/nist-800-53")
|
||||||
|
|||||||
@@ -1,31 +1,32 @@
|
|||||||
"""Detection rules endpoints — listing, filtering, and template association.
|
"""Detection rules endpoints — listing, filtering, and template association.
|
||||||
|
|
||||||
|
Thin HTTP adapter: delegates all query and business logic to detection_rule_service.
|
||||||
|
|
||||||
Provides endpoints for browsing detection rules, querying rules by technique,
|
Provides endpoints for browsing detection rules, querying rules by technique,
|
||||||
and managing the template ↔ detection rule associations.
|
and managing the template ↔ detection rule associations.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
from fastapi import APIRouter, Depends, Query
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import func
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.dependencies.auth import get_current_user, require_role, require_any_role
|
from app.dependencies.auth import get_current_user, require_role, require_any_role
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.models.detection_rule import DetectionRule
|
from app.services.detection_rule_service import (
|
||||||
from app.models.test_template import TestTemplate
|
list_rules,
|
||||||
from app.models.test_template_detection_rule import TestTemplateDetectionRule
|
get_rules_for_template,
|
||||||
from app.models.test_detection_result import TestDetectionResult
|
auto_associate_rules,
|
||||||
|
get_rules_for_test,
|
||||||
|
evaluate_rule,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ── Pydantic schemas for request validation ────────────────────────────
|
||||||
# Pydantic schemas for request validation
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
class DetectionRuleEvaluate(BaseModel):
|
class DetectionRuleEvaluate(BaseModel):
|
||||||
"""Payload for evaluating a detection rule against a test."""
|
"""Payload for evaluating a detection rule against a test."""
|
||||||
@@ -34,14 +35,12 @@ class DetectionRuleEvaluate(BaseModel):
|
|||||||
triggered: Optional[bool] = None
|
triggered: Optional[bool] = None
|
||||||
notes: Optional[str] = None
|
notes: Optional[str] = None
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/detection-rules", tags=["detection-rules"])
|
router = APIRouter(prefix="/detection-rules", tags=["detection-rules"])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ── GET /detection-rules — List with filters ───────────────────────────
|
||||||
# GET /detection-rules — List with filters
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_detection_rules(
|
def list_detection_rules(
|
||||||
@@ -55,55 +54,20 @@ def list_detection_rules(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""List detection rules with optional filters and pagination."""
|
"""List detection rules with optional filters and pagination."""
|
||||||
query = db.query(DetectionRule).filter(DetectionRule.is_active == True) # noqa: E712
|
return list_rules(
|
||||||
|
db,
|
||||||
if technique:
|
technique=technique,
|
||||||
query = query.filter(DetectionRule.mitre_technique_id == technique)
|
source=source,
|
||||||
|
severity=severity,
|
||||||
if source:
|
search=search,
|
||||||
query = query.filter(DetectionRule.source == source)
|
offset=offset,
|
||||||
|
limit=limit,
|
||||||
if severity:
|
|
||||||
query = query.filter(DetectionRule.severity == severity)
|
|
||||||
|
|
||||||
if search:
|
|
||||||
from app.utils import escape_like
|
|
||||||
pattern = f"%{escape_like(search)}%"
|
|
||||||
query = query.filter(
|
|
||||||
DetectionRule.title.ilike(pattern)
|
|
||||||
| DetectionRule.description.ilike(pattern)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
total = query.count()
|
|
||||||
items = query.order_by(DetectionRule.mitre_technique_id, DetectionRule.title).offset(offset).limit(limit).all()
|
|
||||||
|
|
||||||
return {
|
# ── GET /detection-rules/for-template/{template_id} ────────────────────
|
||||||
"total": total,
|
|
||||||
"offset": offset,
|
|
||||||
"limit": limit,
|
|
||||||
"items": [
|
|
||||||
{
|
|
||||||
"id": str(r.id),
|
|
||||||
"mitre_technique_id": r.mitre_technique_id,
|
|
||||||
"title": r.title,
|
|
||||||
"description": r.description,
|
|
||||||
"source": r.source,
|
|
||||||
"source_url": r.source_url,
|
|
||||||
"rule_format": r.rule_format,
|
|
||||||
"severity": r.severity,
|
|
||||||
"platforms": r.platforms or [],
|
|
||||||
"log_sources": r.log_sources,
|
|
||||||
"is_active": r.is_active,
|
|
||||||
}
|
|
||||||
for r in items
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /test-templates/{id}/detection-rules — Rules for a template
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.get("/for-template/{template_id}")
|
@router.get("/for-template/{template_id}")
|
||||||
def get_detection_rules_for_template(
|
def get_detection_rules_for_template(
|
||||||
template_id: str,
|
template_id: str,
|
||||||
@@ -111,46 +75,11 @@ def get_detection_rules_for_template(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Get detection rules associated with a test template."""
|
"""Get detection rules associated with a test template."""
|
||||||
template = db.query(TestTemplate).filter(TestTemplate.id == template_id).first()
|
return get_rules_for_template(db, template_id)
|
||||||
if not template:
|
|
||||||
raise HTTPException(status_code=404, detail="Test template not found")
|
|
||||||
|
|
||||||
associations = (
|
|
||||||
db.query(TestTemplateDetectionRule)
|
|
||||||
.filter(TestTemplateDetectionRule.test_template_id == template_id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
rules = []
|
|
||||||
for assoc in associations:
|
|
||||||
r = assoc.detection_rule
|
|
||||||
rules.append({
|
|
||||||
"id": str(r.id),
|
|
||||||
"mitre_technique_id": r.mitre_technique_id,
|
|
||||||
"title": r.title,
|
|
||||||
"description": r.description,
|
|
||||||
"source": r.source,
|
|
||||||
"source_url": r.source_url,
|
|
||||||
"rule_content": r.rule_content,
|
|
||||||
"rule_format": r.rule_format,
|
|
||||||
"severity": r.severity,
|
|
||||||
"platforms": r.platforms or [],
|
|
||||||
"log_sources": r.log_sources,
|
|
||||||
"is_primary": assoc.is_primary,
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
"template_id": str(template.id),
|
|
||||||
"template_name": template.name,
|
|
||||||
"mitre_technique_id": template.mitre_technique_id,
|
|
||||||
"rules": rules,
|
|
||||||
"total": len(rules),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ── POST /detection-rules/auto-associate ────────────────────────────────
|
||||||
# POST /detection-rules/auto-associate — Auto-link templates ↔ rules
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.post("/auto-associate")
|
@router.post("/auto-associate")
|
||||||
def auto_associate_detection_rules(
|
def auto_associate_detection_rules(
|
||||||
@@ -163,60 +92,11 @@ def auto_associate_detection_rules(
|
|||||||
technique and create associations. Rules with severity >= high are marked
|
technique and create associations. Rules with severity >= high are marked
|
||||||
as primary.
|
as primary.
|
||||||
"""
|
"""
|
||||||
templates = db.query(TestTemplate).filter(TestTemplate.is_active == True).all() # noqa: E712
|
return auto_associate_rules(db)
|
||||||
rules = db.query(DetectionRule).filter(DetectionRule.is_active == True).all() # noqa: E712
|
|
||||||
|
|
||||||
# Index rules by technique
|
|
||||||
rules_by_technique: dict[str, list] = {}
|
|
||||||
for rule in rules:
|
|
||||||
tid = rule.mitre_technique_id
|
|
||||||
if tid not in rules_by_technique:
|
|
||||||
rules_by_technique[tid] = []
|
|
||||||
rules_by_technique[tid].append(rule)
|
|
||||||
|
|
||||||
created = 0
|
|
||||||
skipped = 0
|
|
||||||
high_severities = {"high", "critical"}
|
|
||||||
|
|
||||||
for template in templates:
|
|
||||||
matching_rules = rules_by_technique.get(template.mitre_technique_id, [])
|
|
||||||
for rule in matching_rules:
|
|
||||||
# Check if association already exists
|
|
||||||
existing = (
|
|
||||||
db.query(TestTemplateDetectionRule)
|
|
||||||
.filter(
|
|
||||||
TestTemplateDetectionRule.test_template_id == template.id,
|
|
||||||
TestTemplateDetectionRule.detection_rule_id == rule.id,
|
|
||||||
)
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
if existing:
|
|
||||||
skipped += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
is_primary = (rule.severity or "").lower() in high_severities
|
|
||||||
|
|
||||||
assoc = TestTemplateDetectionRule(
|
|
||||||
test_template_id=template.id,
|
|
||||||
detection_rule_id=rule.id,
|
|
||||||
is_primary=is_primary,
|
|
||||||
)
|
|
||||||
db.add(assoc)
|
|
||||||
created += 1
|
|
||||||
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
total = db.query(TestTemplateDetectionRule).count()
|
|
||||||
return {
|
|
||||||
"created": created,
|
|
||||||
"skipped": skipped,
|
|
||||||
"total_associations": total,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ── GET /detection-rules/for-test/{test_id} ──────────────────────────────
|
||||||
# GET /detection-rules/for-test/{test_id} — Rules + results for a test
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.get("/for-test/{test_id}")
|
@router.get("/for-test/{test_id}")
|
||||||
def get_detection_rules_for_test(
|
def get_detection_rules_for_test(
|
||||||
@@ -229,83 +109,11 @@ def get_detection_rules_for_test(
|
|||||||
Finds rules by matching the test's technique_id to detection rules,
|
Finds rules by matching the test's technique_id to detection rules,
|
||||||
and returns any existing evaluation results.
|
and returns any existing evaluation results.
|
||||||
"""
|
"""
|
||||||
from app.models.test import Test
|
return get_rules_for_test(db, test_id)
|
||||||
from app.models.technique import Technique
|
|
||||||
|
|
||||||
test = db.query(Test).filter(Test.id == test_id).first()
|
|
||||||
if not test:
|
|
||||||
raise HTTPException(status_code=404, detail="Test not found")
|
|
||||||
|
|
||||||
technique = db.query(Technique).filter(Technique.id == test.technique_id).first()
|
|
||||||
if not technique:
|
|
||||||
raise HTTPException(status_code=404, detail="Technique not found")
|
|
||||||
|
|
||||||
# Get detection rules for this technique
|
|
||||||
rules = (
|
|
||||||
db.query(DetectionRule)
|
|
||||||
.filter(
|
|
||||||
DetectionRule.mitre_technique_id == technique.mitre_id,
|
|
||||||
DetectionRule.is_active == True, # noqa: E712
|
|
||||||
)
|
|
||||||
.order_by(DetectionRule.severity.desc(), DetectionRule.title)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
# Get existing results for this test
|
|
||||||
existing_results = (
|
|
||||||
db.query(TestDetectionResult)
|
|
||||||
.filter(TestDetectionResult.test_id == test_id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
results_map = {str(r.detection_rule_id): r for r in existing_results}
|
|
||||||
|
|
||||||
items = []
|
|
||||||
triggered_count = 0
|
|
||||||
evaluated_count = 0
|
|
||||||
|
|
||||||
for rule in rules:
|
|
||||||
result = results_map.get(str(rule.id))
|
|
||||||
triggered = result.triggered if result else None
|
|
||||||
notes = result.notes if result else None
|
|
||||||
evaluated_at = result.evaluated_at.isoformat() if result and result.evaluated_at else None
|
|
||||||
|
|
||||||
if triggered is not None:
|
|
||||||
evaluated_count += 1
|
|
||||||
if triggered:
|
|
||||||
triggered_count += 1
|
|
||||||
|
|
||||||
items.append({
|
|
||||||
"id": str(rule.id),
|
|
||||||
"mitre_technique_id": rule.mitre_technique_id,
|
|
||||||
"title": rule.title,
|
|
||||||
"description": rule.description,
|
|
||||||
"source": rule.source,
|
|
||||||
"source_url": rule.source_url,
|
|
||||||
"rule_content": rule.rule_content,
|
|
||||||
"rule_format": rule.rule_format,
|
|
||||||
"severity": rule.severity,
|
|
||||||
"platforms": rule.platforms or [],
|
|
||||||
"log_sources": rule.log_sources,
|
|
||||||
"triggered": triggered,
|
|
||||||
"notes": notes,
|
|
||||||
"evaluated_at": evaluated_at,
|
|
||||||
"result_id": str(result.id) if result else None,
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
"test_id": str(test.id),
|
|
||||||
"mitre_technique_id": technique.mitre_id,
|
|
||||||
"rules": items,
|
|
||||||
"total": len(items),
|
|
||||||
"evaluated": evaluated_count,
|
|
||||||
"triggered": triggered_count,
|
|
||||||
"detection_rate": round(triggered_count / evaluated_count * 100, 1) if evaluated_count > 0 else 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ── POST /detection-rules/evaluate ──────────────────────────────────────
|
||||||
# POST /detection-rules/evaluate — Save detection result for a rule
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.post("/evaluate")
|
@router.post("/evaluate")
|
||||||
def evaluate_detection_rule(
|
def evaluate_detection_rule(
|
||||||
@@ -314,60 +122,11 @@ def evaluate_detection_rule(
|
|||||||
current_user: User = Depends(require_any_role("blue_tech", "blue_lead")),
|
current_user: User = Depends(require_any_role("blue_tech", "blue_lead")),
|
||||||
):
|
):
|
||||||
"""Save or update the evaluation result for a detection rule on a test."""
|
"""Save or update the evaluation result for a detection rule on a test."""
|
||||||
test_id = payload.test_id
|
return evaluate_rule(
|
||||||
detection_rule_id = payload.detection_rule_id
|
db,
|
||||||
triggered = payload.triggered
|
test_id=payload.test_id,
|
||||||
notes = payload.notes
|
detection_rule_id=payload.detection_rule_id,
|
||||||
|
triggered=payload.triggered,
|
||||||
# Check test exists
|
notes=payload.notes,
|
||||||
from app.models.test import Test
|
evaluator_id=current_user.id,
|
||||||
test = db.query(Test).filter(Test.id == test_id).first()
|
|
||||||
if not test:
|
|
||||||
raise HTTPException(status_code=404, detail="Test not found")
|
|
||||||
|
|
||||||
# Check rule exists
|
|
||||||
rule = db.query(DetectionRule).filter(DetectionRule.id == detection_rule_id).first()
|
|
||||||
if not rule:
|
|
||||||
raise HTTPException(status_code=404, detail="Detection rule not found")
|
|
||||||
|
|
||||||
# Upsert result
|
|
||||||
existing = (
|
|
||||||
db.query(TestDetectionResult)
|
|
||||||
.filter(
|
|
||||||
TestDetectionResult.test_id == test_id,
|
|
||||||
TestDetectionResult.detection_rule_id == detection_rule_id,
|
|
||||||
)
|
)
|
||||||
.first()
|
|
||||||
)
|
|
||||||
|
|
||||||
if existing:
|
|
||||||
existing.triggered = triggered
|
|
||||||
existing.notes = notes
|
|
||||||
existing.evaluated_by = current_user.id
|
|
||||||
existing.evaluated_at = datetime.utcnow()
|
|
||||||
db.commit()
|
|
||||||
db.refresh(existing)
|
|
||||||
return {
|
|
||||||
"id": str(existing.id),
|
|
||||||
"triggered": existing.triggered,
|
|
||||||
"notes": existing.notes,
|
|
||||||
"evaluated_at": existing.evaluated_at.isoformat() if existing.evaluated_at else None,
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
result = TestDetectionResult(
|
|
||||||
test_id=test_id,
|
|
||||||
detection_rule_id=detection_rule_id,
|
|
||||||
triggered=triggered,
|
|
||||||
notes=notes,
|
|
||||||
evaluated_by=current_user.id,
|
|
||||||
evaluated_at=datetime.utcnow(),
|
|
||||||
)
|
|
||||||
db.add(result)
|
|
||||||
db.commit()
|
|
||||||
db.refresh(result)
|
|
||||||
return {
|
|
||||||
"id": str(result.id),
|
|
||||||
"triggered": result.triggered,
|
|
||||||
"notes": result.notes,
|
|
||||||
"evaluated_at": result.evaluated_at.isoformat() if result.evaluated_at else None,
|
|
||||||
}
|
|
||||||
|
|||||||
+18
-216
@@ -3,19 +3,15 @@
|
|||||||
Provides aggregated views of MITRE ATT&CK technique coverage for
|
Provides aggregated views of MITRE ATT&CK technique coverage for
|
||||||
dashboards and reporting. V2 adds pipeline, team-activity, and
|
dashboards and reporting. V2 adds pipeline, team-activity, and
|
||||||
validation-rate endpoints for the Red/Blue workflow.
|
validation-rate endpoints for the Red/Blue workflow.
|
||||||
|
|
||||||
|
Thin HTTP adapter: delegates all data logic to metrics_query_service.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from collections import defaultdict
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
from sqlalchemy import func
|
from sqlalchemy.orm import Session
|
||||||
from sqlalchemy.orm import Session, joinedload
|
|
||||||
|
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.dependencies.auth import get_current_user
|
from app.dependencies.auth import get_current_user
|
||||||
from app.models.enums import TechniqueStatus, TestState
|
|
||||||
from app.models.technique import Technique
|
|
||||||
from app.models.test import Test
|
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.schemas.metrics import (
|
from app.schemas.metrics import (
|
||||||
CoverageSummary,
|
CoverageSummary,
|
||||||
@@ -25,6 +21,14 @@ from app.schemas.metrics import (
|
|||||||
TestPipelineCounts,
|
TestPipelineCounts,
|
||||||
ValidationRate,
|
ValidationRate,
|
||||||
)
|
)
|
||||||
|
from app.services.metrics_query_service import (
|
||||||
|
get_coverage_by_tactic,
|
||||||
|
get_coverage_summary,
|
||||||
|
get_recent_tests,
|
||||||
|
get_team_activity,
|
||||||
|
get_test_pipeline_counts,
|
||||||
|
get_validation_rate,
|
||||||
|
)
|
||||||
|
|
||||||
router = APIRouter(prefix="/metrics", tags=["metrics"])
|
router = APIRouter(prefix="/metrics", tags=["metrics"])
|
||||||
|
|
||||||
@@ -40,37 +44,7 @@ def coverage_summary(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Return a global coverage summary across all techniques."""
|
"""Return a global coverage summary across all techniques."""
|
||||||
|
return get_coverage_summary(db)
|
||||||
rows = (
|
|
||||||
db.query(
|
|
||||||
Technique.status_global,
|
|
||||||
func.count(Technique.id).label("cnt"),
|
|
||||||
)
|
|
||||||
.group_by(Technique.status_global)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
counts: dict[str, int] = {s.value: 0 for s in TechniqueStatus}
|
|
||||||
for status, cnt in rows:
|
|
||||||
counts[status.value] = cnt
|
|
||||||
|
|
||||||
total = sum(counts.values())
|
|
||||||
validated = counts["validated"]
|
|
||||||
partial = counts["partial"]
|
|
||||||
|
|
||||||
coverage_pct = (
|
|
||||||
round((validated + partial) / total * 100, 2) if total > 0 else 0.0
|
|
||||||
)
|
|
||||||
|
|
||||||
return CoverageSummary(
|
|
||||||
total_techniques=total,
|
|
||||||
validated=validated,
|
|
||||||
partial=partial,
|
|
||||||
not_covered=counts["not_covered"],
|
|
||||||
in_progress=counts["in_progress"],
|
|
||||||
not_evaluated=counts["not_evaluated"],
|
|
||||||
coverage_percentage=coverage_pct,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -83,49 +57,8 @@ def coverage_by_tactic(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Return coverage breakdown grouped by tactic.
|
"""Return coverage breakdown grouped by tactic."""
|
||||||
|
return get_coverage_by_tactic(db)
|
||||||
Since a technique can belong to multiple tactics (stored as a
|
|
||||||
comma-separated string), the technique is counted once per tactic
|
|
||||||
it belongs to.
|
|
||||||
"""
|
|
||||||
|
|
||||||
techniques = db.query(
|
|
||||||
Technique.tactic, Technique.status_global
|
|
||||||
).all()
|
|
||||||
|
|
||||||
# Accumulate per-tactic counters. A technique with tactic
|
|
||||||
# "persistence, privilege-escalation" is counted in both.
|
|
||||||
tactic_data: dict[str, dict[str, int]] = defaultdict(
|
|
||||||
lambda: {s.value: 0 for s in TechniqueStatus}
|
|
||||||
)
|
|
||||||
|
|
||||||
for tactic_str, status in techniques:
|
|
||||||
if not tactic_str:
|
|
||||||
tactics = ["unknown"]
|
|
||||||
else:
|
|
||||||
tactics = [t.strip() for t in tactic_str.split(",")]
|
|
||||||
|
|
||||||
for tactic in tactics:
|
|
||||||
tactic_data[tactic][status.value] += 1
|
|
||||||
|
|
||||||
result = []
|
|
||||||
for tactic in sorted(tactic_data):
|
|
||||||
counts = tactic_data[tactic]
|
|
||||||
total = sum(counts.values())
|
|
||||||
result.append(
|
|
||||||
TacticCoverage(
|
|
||||||
tactic=tactic,
|
|
||||||
total=total,
|
|
||||||
validated=counts["validated"],
|
|
||||||
partial=counts["partial"],
|
|
||||||
not_covered=counts["not_covered"],
|
|
||||||
not_evaluated=counts["not_evaluated"],
|
|
||||||
in_progress=counts["in_progress"],
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -139,28 +72,7 @@ def test_pipeline(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Return how many tests are in each pipeline state."""
|
"""Return how many tests are in each pipeline state."""
|
||||||
|
return get_test_pipeline_counts(db)
|
||||||
rows = (
|
|
||||||
db.query(Test.state, func.count(Test.id).label("cnt"))
|
|
||||||
.group_by(Test.state)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
state_counts: dict[str, int] = {s.value: 0 for s in TestState}
|
|
||||||
for state, cnt in rows:
|
|
||||||
state_counts[state.value] = cnt
|
|
||||||
|
|
||||||
total = sum(state_counts.values())
|
|
||||||
|
|
||||||
return TestPipelineCounts(
|
|
||||||
draft=state_counts["draft"],
|
|
||||||
red_executing=state_counts["red_executing"],
|
|
||||||
blue_evaluating=state_counts["blue_evaluating"],
|
|
||||||
in_review=state_counts["in_review"],
|
|
||||||
validated=state_counts["validated"],
|
|
||||||
rejected=state_counts["rejected"],
|
|
||||||
total=total,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -174,54 +86,7 @@ def team_activity(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Return activity summary for Red and Blue teams."""
|
"""Return activity summary for Red and Blue teams."""
|
||||||
|
return get_team_activity(db)
|
||||||
# Red Team: completed = tests past red_executing; pending = draft + red_executing
|
|
||||||
red_completed = (
|
|
||||||
db.query(func.count(Test.id))
|
|
||||||
.filter(Test.state.in_([
|
|
||||||
TestState.blue_evaluating,
|
|
||||||
TestState.in_review,
|
|
||||||
TestState.validated,
|
|
||||||
TestState.rejected,
|
|
||||||
]))
|
|
||||||
.scalar()
|
|
||||||
) or 0
|
|
||||||
|
|
||||||
red_pending = (
|
|
||||||
db.query(func.count(Test.id))
|
|
||||||
.filter(Test.state.in_([TestState.draft, TestState.red_executing]))
|
|
||||||
.scalar()
|
|
||||||
) or 0
|
|
||||||
|
|
||||||
# Blue Team: completed = tests past blue_evaluating; pending = blue_evaluating
|
|
||||||
blue_completed = (
|
|
||||||
db.query(func.count(Test.id))
|
|
||||||
.filter(Test.state.in_([
|
|
||||||
TestState.in_review,
|
|
||||||
TestState.validated,
|
|
||||||
TestState.rejected,
|
|
||||||
]))
|
|
||||||
.scalar()
|
|
||||||
) or 0
|
|
||||||
|
|
||||||
blue_pending = (
|
|
||||||
db.query(func.count(Test.id))
|
|
||||||
.filter(Test.state == TestState.blue_evaluating)
|
|
||||||
.scalar()
|
|
||||||
) or 0
|
|
||||||
|
|
||||||
return [
|
|
||||||
TeamActivity(
|
|
||||||
team="Red Team",
|
|
||||||
tests_completed=red_completed,
|
|
||||||
tests_pending=red_pending,
|
|
||||||
),
|
|
||||||
TeamActivity(
|
|
||||||
team="Blue Team",
|
|
||||||
tests_completed=blue_completed,
|
|
||||||
tests_pending=blue_pending,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -235,51 +100,7 @@ def validation_rate(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Return approval and rejection rates for Red Lead and Blue Lead."""
|
"""Return approval and rejection rates for Red Lead and Blue Lead."""
|
||||||
|
return get_validation_rate(db)
|
||||||
# Red Lead validations
|
|
||||||
red_approved = (
|
|
||||||
db.query(func.count(Test.id))
|
|
||||||
.filter(Test.red_validation_status == "approved")
|
|
||||||
.scalar()
|
|
||||||
) or 0
|
|
||||||
red_rejected = (
|
|
||||||
db.query(func.count(Test.id))
|
|
||||||
.filter(Test.red_validation_status == "rejected")
|
|
||||||
.scalar()
|
|
||||||
) or 0
|
|
||||||
red_total = red_approved + red_rejected
|
|
||||||
red_rate = round(red_approved / red_total * 100, 1) if red_total > 0 else 0.0
|
|
||||||
|
|
||||||
# Blue Lead validations
|
|
||||||
blue_approved = (
|
|
||||||
db.query(func.count(Test.id))
|
|
||||||
.filter(Test.blue_validation_status == "approved")
|
|
||||||
.scalar()
|
|
||||||
) or 0
|
|
||||||
blue_rejected = (
|
|
||||||
db.query(func.count(Test.id))
|
|
||||||
.filter(Test.blue_validation_status == "rejected")
|
|
||||||
.scalar()
|
|
||||||
) or 0
|
|
||||||
blue_total = blue_approved + blue_rejected
|
|
||||||
blue_rate = round(blue_approved / blue_total * 100, 1) if blue_total > 0 else 0.0
|
|
||||||
|
|
||||||
return [
|
|
||||||
ValidationRate(
|
|
||||||
role="red_lead",
|
|
||||||
total_reviewed=red_total,
|
|
||||||
approved=red_approved,
|
|
||||||
rejected=red_rejected,
|
|
||||||
approval_rate=red_rate,
|
|
||||||
),
|
|
||||||
ValidationRate(
|
|
||||||
role="blue_lead",
|
|
||||||
total_reviewed=blue_total,
|
|
||||||
approved=blue_approved,
|
|
||||||
rejected=blue_rejected,
|
|
||||||
approval_rate=blue_rate,
|
|
||||||
),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -293,23 +114,4 @@ def recent_tests(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Return the 10 most recently created tests."""
|
"""Return the 10 most recently created tests."""
|
||||||
|
return get_recent_tests(db, limit=10)
|
||||||
tests = (
|
|
||||||
db.query(Test)
|
|
||||||
.options(joinedload(Test.technique))
|
|
||||||
.order_by(Test.created_at.desc())
|
|
||||||
.limit(10)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
return [
|
|
||||||
RecentTestItem(
|
|
||||||
id=str(t.id),
|
|
||||||
name=t.name,
|
|
||||||
state=t.state.value,
|
|
||||||
technique_mitre_id=t.technique.mitre_id if t.technique else None,
|
|
||||||
technique_name=t.technique.name if t.technique else None,
|
|
||||||
created_at=t.created_at,
|
|
||||||
)
|
|
||||||
for t in tests
|
|
||||||
]
|
|
||||||
|
|||||||
+16
-199
@@ -1,5 +1,7 @@
|
|||||||
"""Reports endpoints — export coverage summaries and test results.
|
"""Reports endpoints — export coverage summaries and test results.
|
||||||
|
|
||||||
|
Thin HTTP adapter: delegates all data logic to coverage_report_service.
|
||||||
|
|
||||||
Endpoints
|
Endpoints
|
||||||
---------
|
---------
|
||||||
GET /reports/coverage-summary — full coverage JSON report
|
GET /reports/coverage-summary — full coverage JSON report
|
||||||
@@ -15,24 +17,21 @@ from typing import Optional
|
|||||||
|
|
||||||
from fastapi import APIRouter, Depends, Query
|
from fastapi import APIRouter, Depends, Query
|
||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
from sqlalchemy import func
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.dependencies.auth import get_current_user
|
from app.dependencies.auth import get_current_user
|
||||||
from app.models.enums import TestState
|
|
||||||
from app.models.technique import Technique
|
|
||||||
from app.models.test import Test
|
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
|
from app.services.coverage_report_service import (
|
||||||
|
build_coverage_csv_rows,
|
||||||
|
build_coverage_summary,
|
||||||
|
build_remediation_status_report,
|
||||||
|
build_test_results_report,
|
||||||
|
)
|
||||||
|
|
||||||
router = APIRouter(prefix="/reports", tags=["reports"])
|
router = APIRouter(prefix="/reports", tags=["reports"])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /reports/coverage-summary
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/coverage-summary")
|
@router.get("/coverage-summary")
|
||||||
def coverage_summary(
|
def coverage_summary(
|
||||||
tactic: Optional[str] = Query(None, description="Filter by tactic"),
|
tactic: Optional[str] = Query(None, description="Filter by tactic"),
|
||||||
@@ -41,63 +40,7 @@ def coverage_summary(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Full coverage report as JSON — technique-by-technique with test counts."""
|
"""Full coverage report as JSON — technique-by-technique with test counts."""
|
||||||
query = db.query(Technique)
|
return build_coverage_summary(db, tactic=tactic, platform=platform)
|
||||||
if tactic:
|
|
||||||
from app.utils import escape_like
|
|
||||||
query = query.filter(Technique.tactic.ilike(f"%{escape_like(tactic)}%"))
|
|
||||||
|
|
||||||
techniques = query.order_by(Technique.mitre_id).all()
|
|
||||||
|
|
||||||
rows = []
|
|
||||||
for t in techniques:
|
|
||||||
# Count tests per state for this technique
|
|
||||||
test_counts = (
|
|
||||||
db.query(Test.state, func.count(Test.id))
|
|
||||||
.filter(Test.technique_id == t.id)
|
|
||||||
.group_by(Test.state)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
counts = {str(state): count for state, count in test_counts}
|
|
||||||
|
|
||||||
# Filter by platform if requested (check if technique platforms contain it)
|
|
||||||
if platform and platform.lower() not in [p.lower() for p in (t.platforms or [])]:
|
|
||||||
continue
|
|
||||||
|
|
||||||
rows.append({
|
|
||||||
"mitre_id": t.mitre_id,
|
|
||||||
"name": t.name,
|
|
||||||
"tactic": t.tactic,
|
|
||||||
"platforms": t.platforms,
|
|
||||||
"status_global": t.status_global,
|
|
||||||
"total_tests": sum(counts.values()),
|
|
||||||
"tests_by_state": counts,
|
|
||||||
})
|
|
||||||
|
|
||||||
total = len(rows)
|
|
||||||
validated = sum(1 for r in rows if r["status_global"] == "validated")
|
|
||||||
partial = sum(1 for r in rows if r["status_global"] == "partial")
|
|
||||||
not_covered = sum(1 for r in rows if r["status_global"] == "not_covered")
|
|
||||||
in_progress = sum(1 for r in rows if r["status_global"] == "in_progress")
|
|
||||||
not_evaluated = sum(1 for r in rows if r["status_global"] == "not_evaluated")
|
|
||||||
|
|
||||||
return {
|
|
||||||
"generated_at": datetime.utcnow().isoformat(),
|
|
||||||
"summary": {
|
|
||||||
"total_techniques": total,
|
|
||||||
"validated": validated,
|
|
||||||
"partial": partial,
|
|
||||||
"not_covered": not_covered,
|
|
||||||
"in_progress": in_progress,
|
|
||||||
"not_evaluated": not_evaluated,
|
|
||||||
"coverage_percentage": round((validated / total * 100) if total > 0 else 0, 1),
|
|
||||||
},
|
|
||||||
"techniques": rows,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /reports/coverage-csv
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/coverage-csv")
|
@router.get("/coverage-csv")
|
||||||
@@ -108,57 +51,22 @@ def coverage_csv(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Export coverage as a downloadable CSV."""
|
"""Export coverage as a downloadable CSV."""
|
||||||
query = db.query(Technique)
|
rows = build_coverage_csv_rows(db, tactic=tactic, platform=platform)
|
||||||
if tactic:
|
|
||||||
from app.utils import escape_like
|
|
||||||
query = query.filter(Technique.tactic.ilike(f"%{escape_like(tactic)}%"))
|
|
||||||
|
|
||||||
techniques = query.order_by(Technique.mitre_id).all()
|
|
||||||
|
|
||||||
output = io.StringIO()
|
output = io.StringIO()
|
||||||
writer = csv.writer(output)
|
writer = csv.writer(output)
|
||||||
writer.writerow([
|
for row in rows:
|
||||||
"MITRE ID", "Name", "Tactic", "Platforms", "Status",
|
writer.writerow(row)
|
||||||
"Total Tests", "Validated", "In Progress", "Not Covered",
|
|
||||||
])
|
|
||||||
|
|
||||||
for t in techniques:
|
|
||||||
if platform and platform.lower() not in [p.lower() for p in (t.platforms or [])]:
|
|
||||||
continue
|
|
||||||
|
|
||||||
test_counts = (
|
|
||||||
db.query(Test.state, func.count(Test.id))
|
|
||||||
.filter(Test.technique_id == t.id)
|
|
||||||
.group_by(Test.state)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
counts = {str(state): count for state, count in test_counts}
|
|
||||||
|
|
||||||
writer.writerow([
|
|
||||||
t.mitre_id,
|
|
||||||
t.name,
|
|
||||||
t.tactic,
|
|
||||||
", ".join(t.platforms or []),
|
|
||||||
t.status_global,
|
|
||||||
sum(counts.values()),
|
|
||||||
counts.get("validated", 0),
|
|
||||||
sum(counts.get(s, 0) for s in ["draft", "red_executing", "blue_evaluating", "in_review"]),
|
|
||||||
counts.get("rejected", 0),
|
|
||||||
])
|
|
||||||
|
|
||||||
output.seek(0)
|
output.seek(0)
|
||||||
|
filename = f"aegis_coverage_{datetime.utcnow().strftime('%Y%m%d')}.csv"
|
||||||
return StreamingResponse(
|
return StreamingResponse(
|
||||||
iter([output.getvalue()]),
|
iter([output.getvalue()]),
|
||||||
media_type="text/csv",
|
media_type="text/csv",
|
||||||
headers={"Content-Disposition": f"attachment; filename=aegis_coverage_{datetime.utcnow().strftime('%Y%m%d')}.csv"},
|
headers={"Content-Disposition": f"attachment; filename={filename}"},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /reports/test-results
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/test-results")
|
@router.get("/test-results")
|
||||||
def test_results(
|
def test_results(
|
||||||
state: Optional[str] = Query(None),
|
state: Optional[str] = Query(None),
|
||||||
@@ -168,68 +76,7 @@ def test_results(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Report of test results with optional filters."""
|
"""Report of test results with optional filters."""
|
||||||
query = db.query(Test)
|
return build_test_results_report(db, state=state, date_from=date_from, date_to=date_to)
|
||||||
|
|
||||||
if state:
|
|
||||||
query = query.filter(Test.state == state)
|
|
||||||
if date_from:
|
|
||||||
try:
|
|
||||||
dt = datetime.fromisoformat(date_from)
|
|
||||||
query = query.filter(Test.created_at >= dt)
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
if date_to:
|
|
||||||
try:
|
|
||||||
dt = datetime.fromisoformat(date_to)
|
|
||||||
query = query.filter(Test.created_at <= dt)
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
tests = query.order_by(Test.created_at.desc()).all()
|
|
||||||
|
|
||||||
# Summary
|
|
||||||
total = len(tests)
|
|
||||||
by_state = {}
|
|
||||||
by_result = {}
|
|
||||||
for t in tests:
|
|
||||||
s = t.state.value if hasattr(t.state, "value") else str(t.state)
|
|
||||||
by_state[s] = by_state.get(s, 0) + 1
|
|
||||||
if t.detection_result:
|
|
||||||
r = t.detection_result.value if hasattr(t.detection_result, "value") else str(t.detection_result)
|
|
||||||
by_result[r] = by_result.get(r, 0) + 1
|
|
||||||
|
|
||||||
return {
|
|
||||||
"generated_at": datetime.utcnow().isoformat(),
|
|
||||||
"filters": {"state": state, "date_from": date_from, "date_to": date_to},
|
|
||||||
"summary": {
|
|
||||||
"total_tests": total,
|
|
||||||
"by_state": by_state,
|
|
||||||
"by_detection_result": by_result,
|
|
||||||
},
|
|
||||||
"tests": [
|
|
||||||
{
|
|
||||||
"id": str(t.id),
|
|
||||||
"name": t.name,
|
|
||||||
"technique_id": str(t.technique_id),
|
|
||||||
"state": t.state.value if hasattr(t.state, "value") else str(t.state),
|
|
||||||
"platform": t.platform,
|
|
||||||
"attack_success": t.attack_success,
|
|
||||||
"detection_result": (
|
|
||||||
t.detection_result.value if t.detection_result and hasattr(t.detection_result, "value")
|
|
||||||
else str(t.detection_result) if t.detection_result else None
|
|
||||||
),
|
|
||||||
"red_validation_status": t.red_validation_status,
|
|
||||||
"blue_validation_status": t.blue_validation_status,
|
|
||||||
"created_at": t.created_at.isoformat() if t.created_at else None,
|
|
||||||
}
|
|
||||||
for t in tests
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /reports/remediation-status
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/remediation-status")
|
@router.get("/remediation-status")
|
||||||
@@ -239,34 +86,4 @@ def remediation_status(
|
|||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Report of remediation status across all tests."""
|
"""Report of remediation status across all tests."""
|
||||||
query = db.query(Test).filter(Test.remediation_steps.isnot(None))
|
return build_remediation_status_report(db, status=status)
|
||||||
|
|
||||||
if status:
|
|
||||||
query = query.filter(Test.remediation_status == status)
|
|
||||||
|
|
||||||
tests = query.order_by(Test.created_at.desc()).all()
|
|
||||||
|
|
||||||
by_status = {}
|
|
||||||
for t in tests:
|
|
||||||
s = t.remediation_status or "unset"
|
|
||||||
by_status[s] = by_status.get(s, 0) + 1
|
|
||||||
|
|
||||||
return {
|
|
||||||
"generated_at": datetime.utcnow().isoformat(),
|
|
||||||
"summary": {
|
|
||||||
"total_with_remediation": len(tests),
|
|
||||||
"by_status": by_status,
|
|
||||||
},
|
|
||||||
"tests": [
|
|
||||||
{
|
|
||||||
"id": str(t.id),
|
|
||||||
"name": t.name,
|
|
||||||
"technique_id": str(t.technique_id),
|
|
||||||
"state": t.state.value if hasattr(t.state, "value") else str(t.state),
|
|
||||||
"remediation_status": t.remediation_status,
|
|
||||||
"remediation_steps": t.remediation_steps,
|
|
||||||
"remediation_assignee": str(t.remediation_assignee) if t.remediation_assignee else None,
|
|
||||||
}
|
|
||||||
for t in tests
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,13 +1,23 @@
|
|||||||
"""CRUD router for MITRE ATT&CK Techniques."""
|
"""CRUD router for MITRE ATT&CK Techniques.
|
||||||
|
|
||||||
from datetime import datetime
|
Uses the TechniqueRepository for data access and domain exceptions
|
||||||
|
for error signaling. The error_handler middleware maps domain
|
||||||
|
exceptions to HTTP responses automatically.
|
||||||
|
"""
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
from fastapi import APIRouter, Depends, Query, status
|
||||||
from sqlalchemy.orm import Session, joinedload
|
from sqlalchemy.orm import Session, joinedload
|
||||||
|
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.dependencies.auth import get_current_user, require_role, require_any_role
|
from app.dependencies.auth import get_current_user, require_role, require_any_role
|
||||||
from app.models.enums import TechniqueStatus
|
from app.dependencies.repositories import get_technique_repository
|
||||||
|
from app.domain.entities.technique import TechniqueEntity
|
||||||
|
from app.domain.errors import DuplicateEntityError, EntityNotFoundError
|
||||||
|
from app.domain.enums import TechniqueStatus
|
||||||
|
from app.domain.unit_of_work import UnitOfWork
|
||||||
|
from app.infrastructure.persistence.repositories.sa_technique_repository import (
|
||||||
|
SATechniqueRepository,
|
||||||
|
)
|
||||||
from app.models.technique import Technique
|
from app.models.technique import Technique
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.schemas.technique import (
|
from app.schemas.technique import (
|
||||||
@@ -34,24 +44,19 @@ def list_techniques(
|
|||||||
None, alias="status", description="Filter by global status"
|
None, alias="status", description="Filter by global status"
|
||||||
),
|
),
|
||||||
review_required: bool | None = Query(None, description="Filter by review flag"),
|
review_required: bool | None = Query(None, description="Filter by review flag"),
|
||||||
db: Session = Depends(get_db),
|
repo: SATechniqueRepository = Depends(get_technique_repository),
|
||||||
current_user: User = Depends(get_current_user),
|
current_user: User = Depends(get_current_user),
|
||||||
):
|
):
|
||||||
"""Return a lightweight list of techniques, optionally filtered."""
|
"""Return a lightweight list of techniques, optionally filtered."""
|
||||||
query = db.query(Technique)
|
return repo.list_all(
|
||||||
|
tactic=tactic,
|
||||||
if tactic is not None:
|
status=status_global,
|
||||||
query = query.filter(Technique.tactic == tactic)
|
review_required=review_required,
|
||||||
if status_global is not None:
|
)
|
||||||
query = query.filter(Technique.status_global == status_global)
|
|
||||||
if review_required is not None:
|
|
||||||
query = query.filter(Technique.review_required == review_required)
|
|
||||||
|
|
||||||
return query.order_by(Technique.mitre_id).all()
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# GET /techniques/{mitre_id} — detail (with tests)
|
# GET /techniques/{mitre_id} — detail (with tests + D3FEND)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@@ -70,12 +75,8 @@ def get_technique(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if technique is None:
|
if technique is None:
|
||||||
raise HTTPException(
|
raise EntityNotFoundError("Technique", mitre_id)
|
||||||
status_code=status.HTTP_404_NOT_FOUND,
|
|
||||||
detail=f"Technique {mitre_id} not found",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Build response dict manually to include D3FEND defenses
|
|
||||||
defenses = get_defenses_for_technique(db, technique.id)
|
defenses = get_defenses_for_technique(db, technique.id)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -120,34 +121,35 @@ def get_technique(
|
|||||||
def create_technique(
|
def create_technique(
|
||||||
payload: TechniqueCreate,
|
payload: TechniqueCreate,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
|
repo: SATechniqueRepository = Depends(get_technique_repository),
|
||||||
current_user: User = Depends(require_role("admin")),
|
current_user: User = Depends(require_role("admin")),
|
||||||
):
|
):
|
||||||
"""Create a new technique manually."""
|
"""Create a new technique manually."""
|
||||||
# Ensure mitre_id is unique
|
if repo.exists_by_mitre_id(payload.mitre_id):
|
||||||
existing = (
|
raise DuplicateEntityError("Technique", "mitre_id", payload.mitre_id)
|
||||||
db.query(Technique).filter(Technique.mitre_id == payload.mitre_id).first()
|
|
||||||
)
|
entity = TechniqueEntity.create(
|
||||||
if existing is not None:
|
mitre_id=payload.mitre_id,
|
||||||
raise HTTPException(
|
name=payload.name,
|
||||||
status_code=status.HTTP_409_CONFLICT,
|
description=payload.description,
|
||||||
detail=f"Technique with mitre_id '{payload.mitre_id}' already exists",
|
tactic=payload.tactic,
|
||||||
|
platforms=payload.platforms,
|
||||||
)
|
)
|
||||||
|
|
||||||
technique = Technique(**payload.model_dump())
|
with UnitOfWork(db) as uow:
|
||||||
db.add(technique)
|
saved = repo.save(entity)
|
||||||
db.commit()
|
uow.commit()
|
||||||
db.refresh(technique)
|
|
||||||
|
|
||||||
log_action(
|
log_action(
|
||||||
db,
|
db,
|
||||||
user_id=current_user.id,
|
user_id=current_user.id,
|
||||||
action="create_technique",
|
action="create_technique",
|
||||||
entity_type="technique",
|
entity_type="technique",
|
||||||
entity_id=technique.id,
|
entity_id=saved.id,
|
||||||
details={"mitre_id": technique.mitre_id, "name": technique.name},
|
details={"mitre_id": saved.mitre_id, "name": saved.name},
|
||||||
)
|
)
|
||||||
|
|
||||||
return technique
|
return saved
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -160,36 +162,32 @@ def update_technique(
|
|||||||
mitre_id: str,
|
mitre_id: str,
|
||||||
payload: TechniqueUpdate,
|
payload: TechniqueUpdate,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
|
repo: SATechniqueRepository = Depends(get_technique_repository),
|
||||||
current_user: User = Depends(require_role("admin")),
|
current_user: User = Depends(require_role("admin")),
|
||||||
):
|
):
|
||||||
"""Update one or more fields of an existing technique."""
|
"""Update one or more fields of an existing technique."""
|
||||||
technique = (
|
entity = repo.find_by_mitre_id(mitre_id)
|
||||||
db.query(Technique).filter(Technique.mitre_id == mitre_id).first()
|
if entity is None:
|
||||||
)
|
raise EntityNotFoundError("Technique", mitre_id)
|
||||||
|
|
||||||
if technique is None:
|
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_404_NOT_FOUND,
|
|
||||||
detail=f"Technique {mitre_id} not found",
|
|
||||||
)
|
|
||||||
|
|
||||||
update_data = payload.model_dump(exclude_unset=True)
|
update_data = payload.model_dump(exclude_unset=True)
|
||||||
for field, value in update_data.items():
|
for field, value in update_data.items():
|
||||||
setattr(technique, field, value)
|
setattr(entity, field, value)
|
||||||
|
|
||||||
db.commit()
|
with UnitOfWork(db) as uow:
|
||||||
db.refresh(technique)
|
saved = repo.save(entity)
|
||||||
|
uow.commit()
|
||||||
|
|
||||||
log_action(
|
log_action(
|
||||||
db,
|
db,
|
||||||
user_id=current_user.id,
|
user_id=current_user.id,
|
||||||
action="update_technique",
|
action="update_technique",
|
||||||
entity_type="technique",
|
entity_type="technique",
|
||||||
entity_id=technique.id,
|
entity_id=saved.id,
|
||||||
details={"mitre_id": mitre_id, "updated_fields": list(update_data.keys())},
|
details={"mitre_id": mitre_id, "updated_fields": list(update_data.keys())},
|
||||||
)
|
)
|
||||||
|
|
||||||
return technique
|
return saved
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -201,6 +199,7 @@ def update_technique(
|
|||||||
def review_technique(
|
def review_technique(
|
||||||
mitre_id: str,
|
mitre_id: str,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
|
repo: SATechniqueRepository = Depends(get_technique_repository),
|
||||||
current_user: User = Depends(require_any_role("red_lead", "blue_lead")),
|
current_user: User = Depends(require_any_role("red_lead", "blue_lead")),
|
||||||
):
|
):
|
||||||
"""Mark a technique as reviewed.
|
"""Mark a technique as reviewed.
|
||||||
@@ -208,29 +207,23 @@ def review_technique(
|
|||||||
Sets ``review_required`` to *False* and records the current timestamp
|
Sets ``review_required`` to *False* and records the current timestamp
|
||||||
in ``last_review_date``.
|
in ``last_review_date``.
|
||||||
"""
|
"""
|
||||||
technique = (
|
entity = repo.find_by_mitre_id(mitre_id)
|
||||||
db.query(Technique).filter(Technique.mitre_id == mitre_id).first()
|
if entity is None:
|
||||||
)
|
raise EntityNotFoundError("Technique", mitre_id)
|
||||||
|
|
||||||
if technique is None:
|
entity.mark_reviewed()
|
||||||
raise HTTPException(
|
|
||||||
status_code=status.HTTP_404_NOT_FOUND,
|
|
||||||
detail=f"Technique {mitre_id} not found",
|
|
||||||
)
|
|
||||||
|
|
||||||
technique.review_required = False
|
with UnitOfWork(db) as uow:
|
||||||
technique.last_review_date = datetime.utcnow()
|
saved = repo.save(entity)
|
||||||
|
uow.commit()
|
||||||
db.commit()
|
|
||||||
db.refresh(technique)
|
|
||||||
|
|
||||||
log_action(
|
log_action(
|
||||||
db,
|
db,
|
||||||
user_id=current_user.id,
|
user_id=current_user.id,
|
||||||
action="review_technique",
|
action="review_technique",
|
||||||
entity_type="technique",
|
entity_type="technique",
|
||||||
entity_id=technique.id,
|
entity_id=saved.id,
|
||||||
details={"mitre_id": mitre_id},
|
details={"mitre_id": mitre_id},
|
||||||
)
|
)
|
||||||
|
|
||||||
return technique
|
return saved
|
||||||
|
|||||||
@@ -7,28 +7,24 @@ threat actor profiles imported from MITRE CTI.
|
|||||||
import logging
|
import logging
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
from fastapi import APIRouter, Depends, Query
|
||||||
from sqlalchemy import func, or_
|
from sqlalchemy.orm import Session
|
||||||
from sqlalchemy.orm import Session, joinedload
|
|
||||||
|
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.dependencies.auth import get_current_user
|
from app.dependencies.auth import get_current_user
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
from app.models.threat_actor import ThreatActor, ThreatActorTechnique
|
from app.services.threat_actor_service import (
|
||||||
from app.models.technique import Technique
|
get_actor_coverage,
|
||||||
from app.models.test import Test
|
get_actor_detail,
|
||||||
from app.models.test_template import TestTemplate
|
get_actor_gaps,
|
||||||
from app.models.enums import TechniqueStatus
|
list_actors,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
router = APIRouter(prefix="/threat-actors", tags=["threat-actors"])
|
router = APIRouter(prefix="/threat-actors", tags=["threat-actors"])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /threat-actors — Listado con filtros
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_threat_actors(
|
def list_threat_actors(
|
||||||
search: Optional[str] = Query(None),
|
search: Optional[str] = Query(None),
|
||||||
@@ -45,92 +41,17 @@ def list_threat_actors(
|
|||||||
|
|
||||||
**Requires** authentication (any role).
|
**Requires** authentication (any role).
|
||||||
"""
|
"""
|
||||||
query = db.query(ThreatActor)
|
return list_actors(
|
||||||
|
db,
|
||||||
# Filters
|
search=search,
|
||||||
if search:
|
country=country,
|
||||||
from app.utils import escape_like
|
motivation=motivation,
|
||||||
pattern = f"%{escape_like(search)}%"
|
sophistication=sophistication,
|
||||||
query = query.filter(
|
target_sectors=target_sectors,
|
||||||
or_(
|
offset=offset,
|
||||||
ThreatActor.name.ilike(pattern),
|
limit=limit,
|
||||||
ThreatActor.description.ilike(pattern),
|
|
||||||
func.cast(ThreatActor.aliases, func.text()).ilike(pattern),
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if country:
|
|
||||||
query = query.filter(ThreatActor.country == country)
|
|
||||||
|
|
||||||
if motivation:
|
|
||||||
query = query.filter(ThreatActor.motivation == motivation)
|
|
||||||
|
|
||||||
if sophistication:
|
|
||||||
query = query.filter(ThreatActor.sophistication == sophistication)
|
|
||||||
|
|
||||||
if target_sectors:
|
|
||||||
from app.utils import escape_like
|
|
||||||
# JSONB contains check
|
|
||||||
query = query.filter(
|
|
||||||
func.cast(ThreatActor.target_sectors, func.text()).ilike(f"%{escape_like(target_sectors)}%")
|
|
||||||
)
|
|
||||||
|
|
||||||
# Total count
|
|
||||||
total = query.count()
|
|
||||||
|
|
||||||
# Paginate
|
|
||||||
actors = query.order_by(ThreatActor.name).offset(offset).limit(limit).all()
|
|
||||||
|
|
||||||
# For each actor, count techniques and calculate basic coverage
|
|
||||||
results = []
|
|
||||||
for actor in actors:
|
|
||||||
tech_count = (
|
|
||||||
db.query(ThreatActorTechnique)
|
|
||||||
.filter(ThreatActorTechnique.threat_actor_id == actor.id)
|
|
||||||
.count()
|
|
||||||
)
|
|
||||||
|
|
||||||
# Quick coverage calculation
|
|
||||||
covered = (
|
|
||||||
db.query(ThreatActorTechnique)
|
|
||||||
.join(Technique, ThreatActorTechnique.technique_id == Technique.id)
|
|
||||||
.filter(ThreatActorTechnique.threat_actor_id == actor.id)
|
|
||||||
.filter(Technique.status_global.in_([
|
|
||||||
TechniqueStatus.validated,
|
|
||||||
TechniqueStatus.partial,
|
|
||||||
]))
|
|
||||||
.count()
|
|
||||||
)
|
|
||||||
|
|
||||||
coverage_pct = round((covered / tech_count * 100), 1) if tech_count > 0 else 0.0
|
|
||||||
|
|
||||||
results.append({
|
|
||||||
"id": str(actor.id),
|
|
||||||
"mitre_id": actor.mitre_id,
|
|
||||||
"name": actor.name,
|
|
||||||
"aliases": actor.aliases or [],
|
|
||||||
"country": actor.country,
|
|
||||||
"target_sectors": actor.target_sectors or [],
|
|
||||||
"target_regions": actor.target_regions or [],
|
|
||||||
"motivation": actor.motivation,
|
|
||||||
"sophistication": actor.sophistication,
|
|
||||||
"mitre_url": actor.mitre_url,
|
|
||||||
"technique_count": tech_count,
|
|
||||||
"coverage_pct": coverage_pct,
|
|
||||||
"is_active": actor.is_active,
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
"total": total,
|
|
||||||
"offset": offset,
|
|
||||||
"limit": limit,
|
|
||||||
"items": results,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /threat-actors/{id} — Detalle
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.get("/{actor_id}")
|
@router.get("/{actor_id}")
|
||||||
def get_threat_actor(
|
def get_threat_actor(
|
||||||
@@ -142,54 +63,8 @@ def get_threat_actor(
|
|||||||
|
|
||||||
**Requires** authentication (any role).
|
**Requires** authentication (any role).
|
||||||
"""
|
"""
|
||||||
actor = db.query(ThreatActor).filter(ThreatActor.id == actor_id).first()
|
return get_actor_detail(db, actor_id)
|
||||||
if not actor:
|
|
||||||
raise HTTPException(status_code=404, detail="Threat actor not found")
|
|
||||||
|
|
||||||
# Get associated techniques with their coverage status
|
|
||||||
actor_techniques = (
|
|
||||||
db.query(ThreatActorTechnique, Technique)
|
|
||||||
.join(Technique, ThreatActorTechnique.technique_id == Technique.id)
|
|
||||||
.filter(ThreatActorTechnique.threat_actor_id == actor.id)
|
|
||||||
.order_by(Technique.mitre_id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
techniques_list = []
|
|
||||||
for at, tech in actor_techniques:
|
|
||||||
techniques_list.append({
|
|
||||||
"technique_id": str(tech.id),
|
|
||||||
"mitre_id": tech.mitre_id,
|
|
||||||
"name": tech.name,
|
|
||||||
"tactic": tech.tactic,
|
|
||||||
"status_global": tech.status_global.value if tech.status_global else None,
|
|
||||||
"usage_description": at.usage_description,
|
|
||||||
"first_seen_using": at.first_seen_using,
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
"id": str(actor.id),
|
|
||||||
"mitre_id": actor.mitre_id,
|
|
||||||
"name": actor.name,
|
|
||||||
"aliases": actor.aliases or [],
|
|
||||||
"description": actor.description,
|
|
||||||
"country": actor.country,
|
|
||||||
"target_sectors": actor.target_sectors or [],
|
|
||||||
"target_regions": actor.target_regions or [],
|
|
||||||
"motivation": actor.motivation,
|
|
||||||
"sophistication": actor.sophistication,
|
|
||||||
"first_seen": actor.first_seen,
|
|
||||||
"last_seen": actor.last_seen,
|
|
||||||
"references": actor.references or [],
|
|
||||||
"mitre_url": actor.mitre_url,
|
|
||||||
"is_active": actor.is_active,
|
|
||||||
"techniques": techniques_list,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /threat-actors/{id}/coverage — Cobertura
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.get("/{actor_id}/coverage")
|
@router.get("/{actor_id}/coverage")
|
||||||
def get_threat_actor_coverage(
|
def get_threat_actor_coverage(
|
||||||
@@ -204,49 +79,8 @@ def get_threat_actor_coverage(
|
|||||||
Returns the percentage of the actor's techniques that have been
|
Returns the percentage of the actor's techniques that have been
|
||||||
validated or partially validated, along with a breakdown.
|
validated or partially validated, along with a breakdown.
|
||||||
"""
|
"""
|
||||||
actor = db.query(ThreatActor).filter(ThreatActor.id == actor_id).first()
|
return get_actor_coverage(db, actor_id)
|
||||||
if not actor:
|
|
||||||
raise HTTPException(status_code=404, detail="Threat actor not found")
|
|
||||||
|
|
||||||
# Get all techniques for this actor
|
|
||||||
actor_techniques = (
|
|
||||||
db.query(Technique)
|
|
||||||
.join(ThreatActorTechnique, ThreatActorTechnique.technique_id == Technique.id)
|
|
||||||
.filter(ThreatActorTechnique.threat_actor_id == actor.id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
total = len(actor_techniques)
|
|
||||||
if total == 0:
|
|
||||||
return {
|
|
||||||
"actor_id": str(actor.id),
|
|
||||||
"actor_name": actor.name,
|
|
||||||
"total_techniques": 0,
|
|
||||||
"coverage_pct": 0.0,
|
|
||||||
"breakdown": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
breakdown = {}
|
|
||||||
for tech in actor_techniques:
|
|
||||||
status = tech.status_global.value if tech.status_global else "not_evaluated"
|
|
||||||
breakdown[status] = breakdown.get(status, 0) + 1
|
|
||||||
|
|
||||||
covered = breakdown.get("validated", 0) + breakdown.get("partial", 0)
|
|
||||||
coverage_pct = round((covered / total * 100), 1)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"actor_id": str(actor.id),
|
|
||||||
"actor_name": actor.name,
|
|
||||||
"total_techniques": total,
|
|
||||||
"covered": covered,
|
|
||||||
"coverage_pct": coverage_pct,
|
|
||||||
"breakdown": breakdown,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# GET /threat-actors/{id}/gaps — Gap analysis
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@router.get("/{actor_id}/gaps")
|
@router.get("/{actor_id}/gaps")
|
||||||
def get_threat_actor_gaps(
|
def get_threat_actor_gaps(
|
||||||
@@ -260,52 +94,4 @@ def get_threat_actor_gaps(
|
|||||||
|
|
||||||
Returns list of gap techniques with available templates.
|
Returns list of gap techniques with available templates.
|
||||||
"""
|
"""
|
||||||
actor = db.query(ThreatActor).filter(ThreatActor.id == actor_id).first()
|
return get_actor_gaps(db, actor_id)
|
||||||
if not actor:
|
|
||||||
raise HTTPException(status_code=404, detail="Threat actor not found")
|
|
||||||
|
|
||||||
# Get techniques NOT validated
|
|
||||||
gap_techniques = (
|
|
||||||
db.query(Technique, ThreatActorTechnique)
|
|
||||||
.join(ThreatActorTechnique, ThreatActorTechnique.technique_id == Technique.id)
|
|
||||||
.filter(ThreatActorTechnique.threat_actor_id == actor.id)
|
|
||||||
.filter(Technique.status_global != TechniqueStatus.validated)
|
|
||||||
.order_by(Technique.mitre_id)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
gaps = []
|
|
||||||
for tech, at in gap_techniques:
|
|
||||||
# Count available templates for this technique
|
|
||||||
template_count = (
|
|
||||||
db.query(TestTemplate)
|
|
||||||
.filter(TestTemplate.mitre_technique_id == tech.mitre_id)
|
|
||||||
.filter(TestTemplate.is_active == True)
|
|
||||||
.count()
|
|
||||||
)
|
|
||||||
|
|
||||||
# Count existing tests
|
|
||||||
test_count = (
|
|
||||||
db.query(Test)
|
|
||||||
.filter(Test.technique_id == tech.id)
|
|
||||||
.count()
|
|
||||||
)
|
|
||||||
|
|
||||||
gaps.append({
|
|
||||||
"technique_id": str(tech.id),
|
|
||||||
"mitre_id": tech.mitre_id,
|
|
||||||
"name": tech.name,
|
|
||||||
"tactic": tech.tactic,
|
|
||||||
"status_global": tech.status_global.value if tech.status_global else None,
|
|
||||||
"usage_description": at.usage_description,
|
|
||||||
"available_templates": template_count,
|
|
||||||
"existing_tests": test_count,
|
|
||||||
"has_templates": template_count > 0,
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
"actor_id": str(actor.id),
|
|
||||||
"actor_name": actor.name,
|
|
||||||
"total_gaps": len(gaps),
|
|
||||||
"gaps": gaps,
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,327 @@
|
|||||||
|
"""Compliance data service.
|
||||||
|
|
||||||
|
Extracts query and aggregation logic from the compliance router so
|
||||||
|
that the router remains a thin HTTP adapter.
|
||||||
|
|
||||||
|
This module is framework-agnostic: no FastAPI imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.domain.errors import EntityNotFoundError
|
||||||
|
from app.models.compliance import (
|
||||||
|
ComplianceFramework,
|
||||||
|
ComplianceControl,
|
||||||
|
ComplianceControlMapping,
|
||||||
|
)
|
||||||
|
from app.models.technique import Technique
|
||||||
|
from app.models.test_template import TestTemplate
|
||||||
|
from app.models.threat_actor import ThreatActorTechnique
|
||||||
|
from app.services.scoring_service import calculate_technique_score
|
||||||
|
|
||||||
|
|
||||||
|
# ── Helpers ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _classify_control(technique_scores: list[float]) -> str:
|
||||||
|
"""Classify a control status based on its technique scores."""
|
||||||
|
if not technique_scores:
|
||||||
|
return "not_evaluated"
|
||||||
|
|
||||||
|
all_above_70 = all(s >= 70 for s in technique_scores)
|
||||||
|
any_above_30 = any(s >= 30 for s in technique_scores)
|
||||||
|
all_below_30 = all(s < 30 for s in technique_scores)
|
||||||
|
all_zero = all(s == 0 for s in technique_scores)
|
||||||
|
|
||||||
|
if all_zero:
|
||||||
|
return "not_evaluated"
|
||||||
|
if all_above_70:
|
||||||
|
return "covered"
|
||||||
|
if all_below_30:
|
||||||
|
return "not_covered"
|
||||||
|
if any_above_30:
|
||||||
|
return "partially_covered"
|
||||||
|
return "not_covered"
|
||||||
|
|
||||||
|
|
||||||
|
def _get_control_status(control: ComplianceControl, db: Session) -> dict[str, Any]:
|
||||||
|
"""Compute the status and score for a single control."""
|
||||||
|
mappings = (
|
||||||
|
db.query(ComplianceControlMapping)
|
||||||
|
.filter(ComplianceControlMapping.compliance_control_id == control.id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
if not mappings:
|
||||||
|
return {
|
||||||
|
"control_id": control.control_id,
|
||||||
|
"title": control.title,
|
||||||
|
"category": control.category,
|
||||||
|
"status": "not_evaluated",
|
||||||
|
"score": 0,
|
||||||
|
"techniques_count": 0,
|
||||||
|
"techniques_covered": 0,
|
||||||
|
"techniques": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
technique_ids = [m.technique_id for m in mappings]
|
||||||
|
techniques = (
|
||||||
|
db.query(Technique)
|
||||||
|
.filter(Technique.id.in_(technique_ids))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
tech_details = []
|
||||||
|
scores = []
|
||||||
|
covered_count = 0
|
||||||
|
|
||||||
|
for tech in techniques:
|
||||||
|
result = calculate_technique_score(tech, db)
|
||||||
|
score = result["total_score"]
|
||||||
|
scores.append(score)
|
||||||
|
if score >= 50:
|
||||||
|
covered_count += 1
|
||||||
|
|
||||||
|
tech_details.append({
|
||||||
|
"mitre_id": tech.mitre_id,
|
||||||
|
"name": tech.name,
|
||||||
|
"score": score,
|
||||||
|
"status": tech.status_global.value if tech.status_global else "not_evaluated",
|
||||||
|
})
|
||||||
|
|
||||||
|
# Sort techniques by score ascending (worst first for priority)
|
||||||
|
tech_details.sort(key=lambda t: t["score"])
|
||||||
|
|
||||||
|
avg_score = round(sum(scores) / len(scores), 1) if scores else 0
|
||||||
|
status = _classify_control(scores)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"control_id": control.control_id,
|
||||||
|
"title": control.title,
|
||||||
|
"category": control.category,
|
||||||
|
"status": status,
|
||||||
|
"score": avg_score,
|
||||||
|
"techniques_count": len(techniques),
|
||||||
|
"techniques_covered": covered_count,
|
||||||
|
"techniques": tech_details,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Public service functions ───────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def list_frameworks(db: Session) -> list[dict[str, Any]]:
|
||||||
|
"""List all available compliance frameworks with control counts."""
|
||||||
|
frameworks = (
|
||||||
|
db.query(ComplianceFramework)
|
||||||
|
.filter(ComplianceFramework.is_active == True)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
result = []
|
||||||
|
for fw in frameworks:
|
||||||
|
control_count = (
|
||||||
|
db.query(ComplianceControl)
|
||||||
|
.filter(ComplianceControl.framework_id == fw.id)
|
||||||
|
.count()
|
||||||
|
)
|
||||||
|
result.append({
|
||||||
|
"id": str(fw.id),
|
||||||
|
"name": fw.name,
|
||||||
|
"version": fw.version,
|
||||||
|
"description": fw.description,
|
||||||
|
"url": fw.url,
|
||||||
|
"is_active": fw.is_active,
|
||||||
|
"controls_count": control_count,
|
||||||
|
})
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def get_framework(db: Session, framework_id: str) -> ComplianceFramework | None:
|
||||||
|
"""Get a framework by ID, or None if not found."""
|
||||||
|
return (
|
||||||
|
db.query(ComplianceFramework)
|
||||||
|
.filter(ComplianceFramework.id == framework_id)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_framework_status(db: Session, framework_id: str) -> dict[str, Any]:
|
||||||
|
"""Get compliance status for each control in a framework.
|
||||||
|
|
||||||
|
Raises EntityNotFoundError if the framework does not exist.
|
||||||
|
"""
|
||||||
|
framework = get_framework(db, framework_id)
|
||||||
|
if not framework:
|
||||||
|
raise EntityNotFoundError("Framework", framework_id)
|
||||||
|
|
||||||
|
controls = (
|
||||||
|
db.query(ComplianceControl)
|
||||||
|
.filter(ComplianceControl.framework_id == framework.id)
|
||||||
|
.order_by(ComplianceControl.control_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
control_statuses = []
|
||||||
|
summary = {
|
||||||
|
"total_controls": len(controls),
|
||||||
|
"covered": 0,
|
||||||
|
"partially_covered": 0,
|
||||||
|
"not_covered": 0,
|
||||||
|
"not_evaluated": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
for control in controls:
|
||||||
|
status_data = _get_control_status(control, db)
|
||||||
|
control_statuses.append(status_data)
|
||||||
|
|
||||||
|
status = status_data["status"]
|
||||||
|
if status in summary:
|
||||||
|
summary[status] += 1
|
||||||
|
|
||||||
|
# Compliance percentage: (covered + partially_covered*0.5) / total * 100
|
||||||
|
total = summary["total_controls"]
|
||||||
|
if total > 0:
|
||||||
|
compliance_pct = round(
|
||||||
|
(summary["covered"] + summary["partially_covered"] * 0.5) / total * 100,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
compliance_pct = 0
|
||||||
|
|
||||||
|
summary["compliance_percentage"] = compliance_pct
|
||||||
|
|
||||||
|
return {
|
||||||
|
"framework": {"id": str(framework.id), "name": framework.name},
|
||||||
|
"summary": summary,
|
||||||
|
"controls": control_statuses,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_framework_report_csv(
|
||||||
|
db: Session,
|
||||||
|
framework_id: str,
|
||||||
|
) -> tuple[bytes, str]:
|
||||||
|
"""Build the compliance report CSV content and filename.
|
||||||
|
|
||||||
|
Returns (csv_bytes, filename).
|
||||||
|
|
||||||
|
Raises EntityNotFoundError if the framework does not exist.
|
||||||
|
"""
|
||||||
|
framework = get_framework(db, framework_id)
|
||||||
|
if not framework:
|
||||||
|
raise EntityNotFoundError("Framework", framework_id)
|
||||||
|
|
||||||
|
controls = (
|
||||||
|
db.query(ComplianceControl)
|
||||||
|
.filter(ComplianceControl.framework_id == framework.id)
|
||||||
|
.order_by(ComplianceControl.control_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
output = io.StringIO()
|
||||||
|
writer = csv.writer(output)
|
||||||
|
writer.writerow([
|
||||||
|
"control_id",
|
||||||
|
"title",
|
||||||
|
"category",
|
||||||
|
"status",
|
||||||
|
"score",
|
||||||
|
"techniques_total",
|
||||||
|
"techniques_covered",
|
||||||
|
"technique_ids",
|
||||||
|
])
|
||||||
|
|
||||||
|
for control in controls:
|
||||||
|
status_data = _get_control_status(control, db)
|
||||||
|
technique_ids = ",".join(t["mitre_id"] for t in status_data["techniques"])
|
||||||
|
writer.writerow([
|
||||||
|
status_data["control_id"],
|
||||||
|
status_data["title"],
|
||||||
|
status_data["category"] or "",
|
||||||
|
status_data["status"],
|
||||||
|
status_data["score"],
|
||||||
|
status_data["techniques_count"],
|
||||||
|
status_data["techniques_covered"],
|
||||||
|
technique_ids,
|
||||||
|
])
|
||||||
|
|
||||||
|
output.seek(0)
|
||||||
|
filename = f"compliance_{framework.name.replace(' ', '_')}.csv"
|
||||||
|
return output.getvalue().encode("utf-8"), filename
|
||||||
|
|
||||||
|
|
||||||
|
def get_framework_gaps(db: Session, framework_id: str) -> dict[str, Any]:
|
||||||
|
"""Get controls with techniques that are not adequately covered.
|
||||||
|
|
||||||
|
Raises EntityNotFoundError if the framework does not exist.
|
||||||
|
"""
|
||||||
|
framework = get_framework(db, framework_id)
|
||||||
|
if not framework:
|
||||||
|
raise EntityNotFoundError("Framework", framework_id)
|
||||||
|
|
||||||
|
controls = (
|
||||||
|
db.query(ComplianceControl)
|
||||||
|
.filter(ComplianceControl.framework_id == framework.id)
|
||||||
|
.order_by(ComplianceControl.control_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
gaps = []
|
||||||
|
for control in controls:
|
||||||
|
status_data = _get_control_status(control, db)
|
||||||
|
|
||||||
|
if status_data["status"] in ("not_covered", "partially_covered"):
|
||||||
|
# Find uncovered techniques
|
||||||
|
uncovered_techniques = []
|
||||||
|
for tech_info in status_data["techniques"]:
|
||||||
|
if tech_info["score"] < 70:
|
||||||
|
# Count available templates
|
||||||
|
template_count = (
|
||||||
|
db.query(TestTemplate)
|
||||||
|
.filter(TestTemplate.mitre_technique_id == tech_info["mitre_id"])
|
||||||
|
.count()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Count threat actors using this technique
|
||||||
|
technique = (
|
||||||
|
db.query(Technique)
|
||||||
|
.filter(Technique.mitre_id == tech_info["mitre_id"])
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
actor_count = 0
|
||||||
|
if technique:
|
||||||
|
actor_count = (
|
||||||
|
db.query(ThreatActorTechnique)
|
||||||
|
.filter(ThreatActorTechnique.technique_id == technique.id)
|
||||||
|
.count()
|
||||||
|
)
|
||||||
|
|
||||||
|
uncovered_techniques.append({
|
||||||
|
**tech_info,
|
||||||
|
"templates_available": template_count,
|
||||||
|
"threat_actors_using": actor_count,
|
||||||
|
})
|
||||||
|
|
||||||
|
if uncovered_techniques:
|
||||||
|
gaps.append({
|
||||||
|
"control_id": status_data["control_id"],
|
||||||
|
"title": status_data["title"],
|
||||||
|
"category": status_data["category"],
|
||||||
|
"status": status_data["status"],
|
||||||
|
"score": status_data["score"],
|
||||||
|
"uncovered_techniques": uncovered_techniques,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"framework": {"id": str(framework.id), "name": framework.name},
|
||||||
|
"total_gaps": len(gaps),
|
||||||
|
"gaps": gaps,
|
||||||
|
}
|
||||||
@@ -0,0 +1,234 @@
|
|||||||
|
"""Coverage report data service.
|
||||||
|
|
||||||
|
Extracts query and aggregation logic from the reports router so
|
||||||
|
that the router remains a thin HTTP adapter. Fixes the N+1
|
||||||
|
technique/test-count pattern by using a single grouped query.
|
||||||
|
|
||||||
|
This module is framework-agnostic: no FastAPI imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import func
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.models.technique import Technique
|
||||||
|
from app.models.test import Test
|
||||||
|
from app.utils import escape_like
|
||||||
|
|
||||||
|
|
||||||
|
def _technique_test_counts(
|
||||||
|
db: Session,
|
||||||
|
technique_ids: list,
|
||||||
|
) -> dict:
|
||||||
|
"""Return ``{technique_id: {state_str: count}}`` in a single query."""
|
||||||
|
if not technique_ids:
|
||||||
|
return {}
|
||||||
|
rows = (
|
||||||
|
db.query(Test.technique_id, Test.state, func.count(Test.id))
|
||||||
|
.filter(Test.technique_id.in_(technique_ids))
|
||||||
|
.group_by(Test.technique_id, Test.state)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
result: dict = {}
|
||||||
|
for tid, state, count in rows:
|
||||||
|
result.setdefault(tid, {})[str(state)] = count
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def build_coverage_summary(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
tactic: str | None = None,
|
||||||
|
platform: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Build the full coverage summary report as a dict."""
|
||||||
|
query = db.query(Technique)
|
||||||
|
if tactic:
|
||||||
|
query = query.filter(Technique.tactic.ilike(f"%{escape_like(tactic)}%"))
|
||||||
|
|
||||||
|
techniques = query.order_by(Technique.mitre_id).all()
|
||||||
|
counts_map = _technique_test_counts(db, [t.id for t in techniques])
|
||||||
|
|
||||||
|
rows = []
|
||||||
|
for t in techniques:
|
||||||
|
if platform and platform.lower() not in [p.lower() for p in (t.platforms or [])]:
|
||||||
|
continue
|
||||||
|
|
||||||
|
counts = counts_map.get(t.id, {})
|
||||||
|
rows.append({
|
||||||
|
"mitre_id": t.mitre_id,
|
||||||
|
"name": t.name,
|
||||||
|
"tactic": t.tactic,
|
||||||
|
"platforms": t.platforms,
|
||||||
|
"status_global": t.status_global,
|
||||||
|
"total_tests": sum(counts.values()),
|
||||||
|
"tests_by_state": counts,
|
||||||
|
})
|
||||||
|
|
||||||
|
total = len(rows)
|
||||||
|
validated = sum(1 for r in rows if r["status_global"] == "validated")
|
||||||
|
partial = sum(1 for r in rows if r["status_global"] == "partial")
|
||||||
|
not_covered = sum(1 for r in rows if r["status_global"] == "not_covered")
|
||||||
|
in_progress = sum(1 for r in rows if r["status_global"] == "in_progress")
|
||||||
|
not_evaluated = sum(1 for r in rows if r["status_global"] == "not_evaluated")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"generated_at": datetime.utcnow().isoformat(),
|
||||||
|
"summary": {
|
||||||
|
"total_techniques": total,
|
||||||
|
"validated": validated,
|
||||||
|
"partial": partial,
|
||||||
|
"not_covered": not_covered,
|
||||||
|
"in_progress": in_progress,
|
||||||
|
"not_evaluated": not_evaluated,
|
||||||
|
"coverage_percentage": round((validated / total * 100) if total > 0 else 0, 1),
|
||||||
|
},
|
||||||
|
"techniques": rows,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_coverage_csv_rows(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
tactic: str | None = None,
|
||||||
|
platform: str | None = None,
|
||||||
|
) -> list[list]:
|
||||||
|
"""Build rows for a CSV coverage export (header + data)."""
|
||||||
|
query = db.query(Technique)
|
||||||
|
if tactic:
|
||||||
|
query = query.filter(Technique.tactic.ilike(f"%{escape_like(tactic)}%"))
|
||||||
|
|
||||||
|
techniques = query.order_by(Technique.mitre_id).all()
|
||||||
|
counts_map = _technique_test_counts(db, [t.id for t in techniques])
|
||||||
|
|
||||||
|
header = [
|
||||||
|
"MITRE ID", "Name", "Tactic", "Platforms", "Status",
|
||||||
|
"Total Tests", "Validated", "In Progress", "Not Covered",
|
||||||
|
]
|
||||||
|
rows = [header]
|
||||||
|
|
||||||
|
in_progress_states = {"draft", "red_executing", "blue_evaluating", "in_review"}
|
||||||
|
|
||||||
|
for t in techniques:
|
||||||
|
if platform and platform.lower() not in [p.lower() for p in (t.platforms or [])]:
|
||||||
|
continue
|
||||||
|
|
||||||
|
counts = counts_map.get(t.id, {})
|
||||||
|
rows.append([
|
||||||
|
t.mitre_id,
|
||||||
|
t.name,
|
||||||
|
t.tactic,
|
||||||
|
", ".join(t.platforms or []),
|
||||||
|
t.status_global,
|
||||||
|
sum(counts.values()),
|
||||||
|
counts.get("validated", 0),
|
||||||
|
sum(counts.get(s, 0) for s in in_progress_states),
|
||||||
|
counts.get("rejected", 0),
|
||||||
|
])
|
||||||
|
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def build_test_results_report(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
state: str | None = None,
|
||||||
|
date_from: str | None = None,
|
||||||
|
date_to: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Build a test results report with optional filters."""
|
||||||
|
query = db.query(Test)
|
||||||
|
|
||||||
|
if state:
|
||||||
|
query = query.filter(Test.state == state)
|
||||||
|
if date_from:
|
||||||
|
try:
|
||||||
|
query = query.filter(Test.created_at >= datetime.fromisoformat(date_from))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
if date_to:
|
||||||
|
try:
|
||||||
|
query = query.filter(Test.created_at <= datetime.fromisoformat(date_to))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
tests = query.order_by(Test.created_at.desc()).all()
|
||||||
|
|
||||||
|
by_state: dict[str, int] = {}
|
||||||
|
by_result: dict[str, int] = {}
|
||||||
|
for t in tests:
|
||||||
|
s = t.state.value if hasattr(t.state, "value") else str(t.state)
|
||||||
|
by_state[s] = by_state.get(s, 0) + 1
|
||||||
|
if t.detection_result:
|
||||||
|
r = t.detection_result.value if hasattr(t.detection_result, "value") else str(t.detection_result)
|
||||||
|
by_result[r] = by_result.get(r, 0) + 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"generated_at": datetime.utcnow().isoformat(),
|
||||||
|
"filters": {"state": state, "date_from": date_from, "date_to": date_to},
|
||||||
|
"summary": {
|
||||||
|
"total_tests": len(tests),
|
||||||
|
"by_state": by_state,
|
||||||
|
"by_detection_result": by_result,
|
||||||
|
},
|
||||||
|
"tests": [
|
||||||
|
{
|
||||||
|
"id": str(t.id),
|
||||||
|
"name": t.name,
|
||||||
|
"technique_id": str(t.technique_id),
|
||||||
|
"state": t.state.value if hasattr(t.state, "value") else str(t.state),
|
||||||
|
"platform": t.platform,
|
||||||
|
"attack_success": t.attack_success,
|
||||||
|
"detection_result": (
|
||||||
|
t.detection_result.value if t.detection_result and hasattr(t.detection_result, "value")
|
||||||
|
else str(t.detection_result) if t.detection_result else None
|
||||||
|
),
|
||||||
|
"red_validation_status": t.red_validation_status,
|
||||||
|
"blue_validation_status": t.blue_validation_status,
|
||||||
|
"created_at": t.created_at.isoformat() if t.created_at else None,
|
||||||
|
}
|
||||||
|
for t in tests
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build_remediation_status_report(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
status: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Build a remediation status report."""
|
||||||
|
query = db.query(Test).filter(Test.remediation_steps.isnot(None))
|
||||||
|
|
||||||
|
if status:
|
||||||
|
query = query.filter(Test.remediation_status == status)
|
||||||
|
|
||||||
|
tests = query.order_by(Test.created_at.desc()).all()
|
||||||
|
|
||||||
|
by_status: dict[str, int] = {}
|
||||||
|
for t in tests:
|
||||||
|
s = t.remediation_status or "unset"
|
||||||
|
by_status[s] = by_status.get(s, 0) + 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"generated_at": datetime.utcnow().isoformat(),
|
||||||
|
"summary": {
|
||||||
|
"total_with_remediation": len(tests),
|
||||||
|
"by_status": by_status,
|
||||||
|
},
|
||||||
|
"tests": [
|
||||||
|
{
|
||||||
|
"id": str(t.id),
|
||||||
|
"name": t.name,
|
||||||
|
"technique_id": str(t.technique_id),
|
||||||
|
"state": t.state.value if hasattr(t.state, "value") else str(t.state),
|
||||||
|
"remediation_status": t.remediation_status,
|
||||||
|
"remediation_steps": t.remediation_steps,
|
||||||
|
"remediation_assignee": str(t.remediation_assignee) if t.remediation_assignee else None,
|
||||||
|
}
|
||||||
|
for t in tests
|
||||||
|
],
|
||||||
|
}
|
||||||
@@ -0,0 +1,319 @@
|
|||||||
|
"""Detection rule data service.
|
||||||
|
|
||||||
|
Extracts query and business logic from the detection_rules router so
|
||||||
|
that the router remains a thin HTTP adapter.
|
||||||
|
|
||||||
|
This module is framework-agnostic: no FastAPI imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.domain.errors import EntityNotFoundError
|
||||||
|
from app.models.detection_rule import DetectionRule
|
||||||
|
from app.models.test import Test
|
||||||
|
from app.models.test_template import TestTemplate
|
||||||
|
from app.models.test_template_detection_rule import TestTemplateDetectionRule
|
||||||
|
from app.models.test_detection_result import TestDetectionResult
|
||||||
|
from app.models.technique import Technique
|
||||||
|
from app.utils import escape_like
|
||||||
|
|
||||||
|
|
||||||
|
# ── Public service functions ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def list_rules(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
technique: str | None = None,
|
||||||
|
source: str | None = None,
|
||||||
|
severity: str | None = None,
|
||||||
|
search: str | None = None,
|
||||||
|
offset: int = 0,
|
||||||
|
limit: int = 50,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""List detection rules with optional filters and pagination."""
|
||||||
|
query = db.query(DetectionRule).filter(DetectionRule.is_active == True)
|
||||||
|
|
||||||
|
if technique:
|
||||||
|
query = query.filter(DetectionRule.mitre_technique_id == technique)
|
||||||
|
if source:
|
||||||
|
query = query.filter(DetectionRule.source == source)
|
||||||
|
if severity:
|
||||||
|
query = query.filter(DetectionRule.severity == severity)
|
||||||
|
if search:
|
||||||
|
pattern = f"%{escape_like(search)}%"
|
||||||
|
query = query.filter(
|
||||||
|
DetectionRule.title.ilike(pattern)
|
||||||
|
| DetectionRule.description.ilike(pattern)
|
||||||
|
)
|
||||||
|
|
||||||
|
total = query.count()
|
||||||
|
items = (
|
||||||
|
query.order_by(DetectionRule.mitre_technique_id, DetectionRule.title)
|
||||||
|
.offset(offset)
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"offset": offset,
|
||||||
|
"limit": limit,
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"id": str(r.id),
|
||||||
|
"mitre_technique_id": r.mitre_technique_id,
|
||||||
|
"title": r.title,
|
||||||
|
"description": r.description,
|
||||||
|
"source": r.source,
|
||||||
|
"source_url": r.source_url,
|
||||||
|
"rule_format": r.rule_format,
|
||||||
|
"severity": r.severity,
|
||||||
|
"platforms": r.platforms or [],
|
||||||
|
"log_sources": r.log_sources,
|
||||||
|
"is_active": r.is_active,
|
||||||
|
}
|
||||||
|
for r in items
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_rules_for_template(db: Session, template_id: str) -> dict[str, Any]:
|
||||||
|
"""Get detection rules associated with a test template.
|
||||||
|
|
||||||
|
Raises EntityNotFoundError if the template does not exist.
|
||||||
|
"""
|
||||||
|
template = db.query(TestTemplate).filter(TestTemplate.id == template_id).first()
|
||||||
|
if not template:
|
||||||
|
raise EntityNotFoundError("Test template", template_id)
|
||||||
|
|
||||||
|
associations = (
|
||||||
|
db.query(TestTemplateDetectionRule)
|
||||||
|
.filter(TestTemplateDetectionRule.test_template_id == template_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
rules = []
|
||||||
|
for assoc in associations:
|
||||||
|
r = assoc.detection_rule
|
||||||
|
rules.append({
|
||||||
|
"id": str(r.id),
|
||||||
|
"mitre_technique_id": r.mitre_technique_id,
|
||||||
|
"title": r.title,
|
||||||
|
"description": r.description,
|
||||||
|
"source": r.source,
|
||||||
|
"source_url": r.source_url,
|
||||||
|
"rule_content": r.rule_content,
|
||||||
|
"rule_format": r.rule_format,
|
||||||
|
"severity": r.severity,
|
||||||
|
"platforms": r.platforms or [],
|
||||||
|
"log_sources": r.log_sources,
|
||||||
|
"is_primary": assoc.is_primary,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"template_id": str(template.id),
|
||||||
|
"template_name": template.name,
|
||||||
|
"mitre_technique_id": template.mitre_technique_id,
|
||||||
|
"rules": rules,
|
||||||
|
"total": len(rules),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def auto_associate_rules(db: Session) -> dict[str, Any]:
|
||||||
|
"""Auto-associate test templates with detection rules by MITRE technique ID.
|
||||||
|
|
||||||
|
For each active template, finds all active detection rules for the same
|
||||||
|
technique and creates associations. Rules with severity high/critical
|
||||||
|
are marked as primary. Performs commit internally.
|
||||||
|
"""
|
||||||
|
templates = db.query(TestTemplate).filter(TestTemplate.is_active == True).all()
|
||||||
|
rules = db.query(DetectionRule).filter(DetectionRule.is_active == True).all()
|
||||||
|
|
||||||
|
rules_by_technique: dict[str, list] = {}
|
||||||
|
for rule in rules:
|
||||||
|
tid = rule.mitre_technique_id
|
||||||
|
if tid not in rules_by_technique:
|
||||||
|
rules_by_technique[tid] = []
|
||||||
|
rules_by_technique[tid].append(rule)
|
||||||
|
|
||||||
|
created = 0
|
||||||
|
skipped = 0
|
||||||
|
high_severities = {"high", "critical"}
|
||||||
|
|
||||||
|
for template in templates:
|
||||||
|
matching_rules = rules_by_technique.get(template.mitre_technique_id, [])
|
||||||
|
for rule in matching_rules:
|
||||||
|
existing = (
|
||||||
|
db.query(TestTemplateDetectionRule)
|
||||||
|
.filter(
|
||||||
|
TestTemplateDetectionRule.test_template_id == template.id,
|
||||||
|
TestTemplateDetectionRule.detection_rule_id == rule.id,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
is_primary = (rule.severity or "").lower() in high_severities
|
||||||
|
|
||||||
|
assoc = TestTemplateDetectionRule(
|
||||||
|
test_template_id=template.id,
|
||||||
|
detection_rule_id=rule.id,
|
||||||
|
is_primary=is_primary,
|
||||||
|
)
|
||||||
|
db.add(assoc)
|
||||||
|
created += 1
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
total = db.query(TestTemplateDetectionRule).count()
|
||||||
|
return {
|
||||||
|
"created": created,
|
||||||
|
"skipped": skipped,
|
||||||
|
"total_associations": total,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_rules_for_test(db: Session, test_id: str) -> dict[str, Any]:
|
||||||
|
"""Get detection rules relevant to a test, along with their evaluation results.
|
||||||
|
|
||||||
|
Finds rules by matching the test's technique to detection rules.
|
||||||
|
Raises EntityNotFoundError if the test or its technique does not exist.
|
||||||
|
"""
|
||||||
|
test = db.query(Test).filter(Test.id == test_id).first()
|
||||||
|
if not test:
|
||||||
|
raise EntityNotFoundError("Test", str(test_id))
|
||||||
|
|
||||||
|
technique = db.query(Technique).filter(Technique.id == test.technique_id).first()
|
||||||
|
if not technique:
|
||||||
|
raise EntityNotFoundError("Technique", str(test.technique_id))
|
||||||
|
|
||||||
|
rules = (
|
||||||
|
db.query(DetectionRule)
|
||||||
|
.filter(
|
||||||
|
DetectionRule.mitre_technique_id == technique.mitre_id,
|
||||||
|
DetectionRule.is_active == True,
|
||||||
|
)
|
||||||
|
.order_by(DetectionRule.severity.desc(), DetectionRule.title)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
existing_results = (
|
||||||
|
db.query(TestDetectionResult)
|
||||||
|
.filter(TestDetectionResult.test_id == test_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
results_map = {str(r.detection_rule_id): r for r in existing_results}
|
||||||
|
|
||||||
|
items = []
|
||||||
|
triggered_count = 0
|
||||||
|
evaluated_count = 0
|
||||||
|
|
||||||
|
for rule in rules:
|
||||||
|
result = results_map.get(str(rule.id))
|
||||||
|
triggered = result.triggered if result else None
|
||||||
|
notes = result.notes if result else None
|
||||||
|
evaluated_at = result.evaluated_at.isoformat() if result and result.evaluated_at else None
|
||||||
|
|
||||||
|
if triggered is not None:
|
||||||
|
evaluated_count += 1
|
||||||
|
if triggered:
|
||||||
|
triggered_count += 1
|
||||||
|
|
||||||
|
items.append({
|
||||||
|
"id": str(rule.id),
|
||||||
|
"mitre_technique_id": rule.mitre_technique_id,
|
||||||
|
"title": rule.title,
|
||||||
|
"description": rule.description,
|
||||||
|
"source": rule.source,
|
||||||
|
"source_url": rule.source_url,
|
||||||
|
"rule_content": rule.rule_content,
|
||||||
|
"rule_format": rule.rule_format,
|
||||||
|
"severity": rule.severity,
|
||||||
|
"platforms": rule.platforms or [],
|
||||||
|
"log_sources": rule.log_sources,
|
||||||
|
"triggered": triggered,
|
||||||
|
"notes": notes,
|
||||||
|
"evaluated_at": evaluated_at,
|
||||||
|
"result_id": str(result.id) if result else None,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"test_id": str(test.id),
|
||||||
|
"mitre_technique_id": technique.mitre_id,
|
||||||
|
"rules": items,
|
||||||
|
"total": len(items),
|
||||||
|
"evaluated": evaluated_count,
|
||||||
|
"triggered": triggered_count,
|
||||||
|
"detection_rate": round(triggered_count / evaluated_count * 100, 1) if evaluated_count > 0 else 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_rule(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
test_id: Any,
|
||||||
|
detection_rule_id: Any,
|
||||||
|
triggered: bool | None,
|
||||||
|
notes: str | None,
|
||||||
|
evaluator_id: Any,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Save or update the evaluation result for a detection rule on a test.
|
||||||
|
|
||||||
|
Raises EntityNotFoundError if the test or detection rule does not exist.
|
||||||
|
"""
|
||||||
|
test = db.query(Test).filter(Test.id == test_id).first()
|
||||||
|
if not test:
|
||||||
|
raise EntityNotFoundError("Test", str(test_id))
|
||||||
|
|
||||||
|
rule = db.query(DetectionRule).filter(DetectionRule.id == detection_rule_id).first()
|
||||||
|
if not rule:
|
||||||
|
raise EntityNotFoundError("Detection rule", str(detection_rule_id))
|
||||||
|
|
||||||
|
existing = (
|
||||||
|
db.query(TestDetectionResult)
|
||||||
|
.filter(
|
||||||
|
TestDetectionResult.test_id == test_id,
|
||||||
|
TestDetectionResult.detection_rule_id == detection_rule_id,
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
if existing:
|
||||||
|
existing.triggered = triggered
|
||||||
|
existing.notes = notes
|
||||||
|
existing.evaluated_by = evaluator_id
|
||||||
|
existing.evaluated_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
db.refresh(existing)
|
||||||
|
return {
|
||||||
|
"id": str(existing.id),
|
||||||
|
"triggered": existing.triggered,
|
||||||
|
"notes": existing.notes,
|
||||||
|
"evaluated_at": existing.evaluated_at.isoformat() if existing.evaluated_at else None,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
result = TestDetectionResult(
|
||||||
|
test_id=test_id,
|
||||||
|
detection_rule_id=detection_rule_id,
|
||||||
|
triggered=triggered,
|
||||||
|
notes=notes,
|
||||||
|
evaluated_by=evaluator_id,
|
||||||
|
evaluated_at=datetime.utcnow(),
|
||||||
|
)
|
||||||
|
db.add(result)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(result)
|
||||||
|
return {
|
||||||
|
"id": str(result.id),
|
||||||
|
"triggered": result.triggered,
|
||||||
|
"notes": result.notes,
|
||||||
|
"evaluated_at": result.evaluated_at.isoformat() if result.evaluated_at else None,
|
||||||
|
}
|
||||||
@@ -0,0 +1,253 @@
|
|||||||
|
"""Metrics query service.
|
||||||
|
|
||||||
|
Extracts query and aggregation logic from the metrics router so that
|
||||||
|
the router remains a thin HTTP adapter. Provides aggregated views
|
||||||
|
of MITRE ATT&CK technique coverage for dashboards and reporting.
|
||||||
|
|
||||||
|
This module is framework-agnostic: no FastAPI imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
from sqlalchemy import func
|
||||||
|
from sqlalchemy.orm import Session, joinedload
|
||||||
|
|
||||||
|
from app.models.enums import TechniqueStatus, TestState
|
||||||
|
from app.models.technique import Technique
|
||||||
|
from app.models.test import Test
|
||||||
|
from app.schemas.metrics import (
|
||||||
|
CoverageSummary,
|
||||||
|
RecentTestItem,
|
||||||
|
TacticCoverage,
|
||||||
|
TeamActivity,
|
||||||
|
TestPipelineCounts,
|
||||||
|
ValidationRate,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_coverage_summary(db: Session) -> CoverageSummary:
|
||||||
|
"""Return a global coverage summary across all techniques."""
|
||||||
|
rows = (
|
||||||
|
db.query(
|
||||||
|
Technique.status_global,
|
||||||
|
func.count(Technique.id).label("cnt"),
|
||||||
|
)
|
||||||
|
.group_by(Technique.status_global)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
counts: dict[str, int] = {s.value: 0 for s in TechniqueStatus}
|
||||||
|
for status, cnt in rows:
|
||||||
|
counts[status.value] = cnt
|
||||||
|
|
||||||
|
total = sum(counts.values())
|
||||||
|
validated = counts["validated"]
|
||||||
|
partial = counts["partial"]
|
||||||
|
|
||||||
|
coverage_pct = (
|
||||||
|
round((validated + partial) / total * 100, 2) if total > 0 else 0.0
|
||||||
|
)
|
||||||
|
|
||||||
|
return CoverageSummary(
|
||||||
|
total_techniques=total,
|
||||||
|
validated=validated,
|
||||||
|
partial=partial,
|
||||||
|
not_covered=counts["not_covered"],
|
||||||
|
in_progress=counts["in_progress"],
|
||||||
|
not_evaluated=counts["not_evaluated"],
|
||||||
|
coverage_percentage=coverage_pct,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_coverage_by_tactic(db: Session) -> list[TacticCoverage]:
|
||||||
|
"""Return coverage breakdown grouped by tactic.
|
||||||
|
|
||||||
|
Since a technique can belong to multiple tactics (stored as a
|
||||||
|
comma-separated string), the technique is counted once per tactic
|
||||||
|
it belongs to.
|
||||||
|
"""
|
||||||
|
techniques = db.query(
|
||||||
|
Technique.tactic, Technique.status_global
|
||||||
|
).all()
|
||||||
|
|
||||||
|
# Accumulate per-tactic counters. A technique with tactic
|
||||||
|
# "persistence, privilege-escalation" is counted in both.
|
||||||
|
tactic_data: dict[str, dict[str, int]] = defaultdict(
|
||||||
|
lambda: {s.value: 0 for s in TechniqueStatus}
|
||||||
|
)
|
||||||
|
|
||||||
|
for tactic_str, status in techniques:
|
||||||
|
if not tactic_str:
|
||||||
|
tactics = ["unknown"]
|
||||||
|
else:
|
||||||
|
tactics = [t.strip() for t in tactic_str.split(",")]
|
||||||
|
|
||||||
|
for tactic in tactics:
|
||||||
|
tactic_data[tactic][status.value] += 1
|
||||||
|
|
||||||
|
result = []
|
||||||
|
for tactic in sorted(tactic_data):
|
||||||
|
counts = tactic_data[tactic]
|
||||||
|
total = sum(counts.values())
|
||||||
|
result.append(
|
||||||
|
TacticCoverage(
|
||||||
|
tactic=tactic,
|
||||||
|
total=total,
|
||||||
|
validated=counts["validated"],
|
||||||
|
partial=counts["partial"],
|
||||||
|
not_covered=counts["not_covered"],
|
||||||
|
not_evaluated=counts["not_evaluated"],
|
||||||
|
in_progress=counts["in_progress"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def get_test_pipeline_counts(db: Session) -> TestPipelineCounts:
|
||||||
|
"""Return how many tests are in each pipeline state."""
|
||||||
|
rows = (
|
||||||
|
db.query(Test.state, func.count(Test.id).label("cnt"))
|
||||||
|
.group_by(Test.state)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
state_counts: dict[str, int] = {s.value: 0 for s in TestState}
|
||||||
|
for state, cnt in rows:
|
||||||
|
state_counts[state.value] = cnt
|
||||||
|
|
||||||
|
total = sum(state_counts.values())
|
||||||
|
|
||||||
|
return TestPipelineCounts(
|
||||||
|
draft=state_counts["draft"],
|
||||||
|
red_executing=state_counts["red_executing"],
|
||||||
|
blue_evaluating=state_counts["blue_evaluating"],
|
||||||
|
in_review=state_counts["in_review"],
|
||||||
|
validated=state_counts["validated"],
|
||||||
|
rejected=state_counts["rejected"],
|
||||||
|
total=total,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_team_activity(db: Session) -> list[TeamActivity]:
|
||||||
|
"""Return activity summary for Red and Blue teams."""
|
||||||
|
# Red Team: completed = tests past red_executing; pending = draft + red_executing
|
||||||
|
red_completed = (
|
||||||
|
db.query(func.count(Test.id))
|
||||||
|
.filter(Test.state.in_([
|
||||||
|
TestState.blue_evaluating,
|
||||||
|
TestState.in_review,
|
||||||
|
TestState.validated,
|
||||||
|
TestState.rejected,
|
||||||
|
]))
|
||||||
|
.scalar()
|
||||||
|
) or 0
|
||||||
|
|
||||||
|
red_pending = (
|
||||||
|
db.query(func.count(Test.id))
|
||||||
|
.filter(Test.state.in_([TestState.draft, TestState.red_executing]))
|
||||||
|
.scalar()
|
||||||
|
) or 0
|
||||||
|
|
||||||
|
# Blue Team: completed = tests past blue_evaluating; pending = blue_evaluating
|
||||||
|
blue_completed = (
|
||||||
|
db.query(func.count(Test.id))
|
||||||
|
.filter(Test.state.in_([
|
||||||
|
TestState.in_review,
|
||||||
|
TestState.validated,
|
||||||
|
TestState.rejected,
|
||||||
|
]))
|
||||||
|
.scalar()
|
||||||
|
) or 0
|
||||||
|
|
||||||
|
blue_pending = (
|
||||||
|
db.query(func.count(Test.id))
|
||||||
|
.filter(Test.state == TestState.blue_evaluating)
|
||||||
|
.scalar()
|
||||||
|
) or 0
|
||||||
|
|
||||||
|
return [
|
||||||
|
TeamActivity(
|
||||||
|
team="Red Team",
|
||||||
|
tests_completed=red_completed,
|
||||||
|
tests_pending=red_pending,
|
||||||
|
),
|
||||||
|
TeamActivity(
|
||||||
|
team="Blue Team",
|
||||||
|
tests_completed=blue_completed,
|
||||||
|
tests_pending=blue_pending,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_validation_rate(db: Session) -> list[ValidationRate]:
|
||||||
|
"""Return approval and rejection rates for Red Lead and Blue Lead."""
|
||||||
|
# Red Lead validations
|
||||||
|
red_approved = (
|
||||||
|
db.query(func.count(Test.id))
|
||||||
|
.filter(Test.red_validation_status == "approved")
|
||||||
|
.scalar()
|
||||||
|
) or 0
|
||||||
|
red_rejected = (
|
||||||
|
db.query(func.count(Test.id))
|
||||||
|
.filter(Test.red_validation_status == "rejected")
|
||||||
|
.scalar()
|
||||||
|
) or 0
|
||||||
|
red_total = red_approved + red_rejected
|
||||||
|
red_rate = round(red_approved / red_total * 100, 1) if red_total > 0 else 0.0
|
||||||
|
|
||||||
|
# Blue Lead validations
|
||||||
|
blue_approved = (
|
||||||
|
db.query(func.count(Test.id))
|
||||||
|
.filter(Test.blue_validation_status == "approved")
|
||||||
|
.scalar()
|
||||||
|
) or 0
|
||||||
|
blue_rejected = (
|
||||||
|
db.query(func.count(Test.id))
|
||||||
|
.filter(Test.blue_validation_status == "rejected")
|
||||||
|
.scalar()
|
||||||
|
) or 0
|
||||||
|
blue_total = blue_approved + blue_rejected
|
||||||
|
blue_rate = round(blue_approved / blue_total * 100, 1) if blue_total > 0 else 0.0
|
||||||
|
|
||||||
|
return [
|
||||||
|
ValidationRate(
|
||||||
|
role="red_lead",
|
||||||
|
total_reviewed=red_total,
|
||||||
|
approved=red_approved,
|
||||||
|
rejected=red_rejected,
|
||||||
|
approval_rate=red_rate,
|
||||||
|
),
|
||||||
|
ValidationRate(
|
||||||
|
role="blue_lead",
|
||||||
|
total_reviewed=blue_total,
|
||||||
|
approved=blue_approved,
|
||||||
|
rejected=blue_rejected,
|
||||||
|
approval_rate=blue_rate,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_recent_tests(db: Session, *, limit: int = 10) -> list[RecentTestItem]:
|
||||||
|
"""Return the most recently created tests."""
|
||||||
|
tests = (
|
||||||
|
db.query(Test)
|
||||||
|
.options(joinedload(Test.technique))
|
||||||
|
.order_by(Test.created_at.desc())
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
return [
|
||||||
|
RecentTestItem(
|
||||||
|
id=str(t.id),
|
||||||
|
name=t.name,
|
||||||
|
state=t.state.value,
|
||||||
|
technique_mitre_id=t.technique.mitre_id if t.technique else None,
|
||||||
|
technique_name=t.technique.name if t.technique else None,
|
||||||
|
created_at=t.created_at,
|
||||||
|
)
|
||||||
|
for t in tests
|
||||||
|
]
|
||||||
@@ -1,47 +1,30 @@
|
|||||||
"""Service for recalculating the global status of a Technique
|
"""Service for recalculating the global status of a Technique.
|
||||||
based on the state and result of its associated tests.
|
|
||||||
|
|
||||||
V2 rules account for dual Red/Blue validation and use
|
Delegates entirely to :meth:`TechniqueEntity.recalculate_status`
|
||||||
``detection_result`` (filled by Blue Team) instead of the legacy
|
so that the business rules live in a single place (the domain entity).
|
||||||
``result`` field.
|
|
||||||
|
|
||||||
This function mutates the technique but does **not** commit.
|
This thin adapter converts ORM objects into the format the entity
|
||||||
|
expects, then writes the result back onto the ORM model.
|
||||||
|
|
||||||
|
The function mutates the technique but does **not** commit.
|
||||||
The caller is responsible for committing the session.
|
The caller is responsible for committing the session.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.models.enums import TechniqueStatus, TestState
|
from app.domain.entities.technique import TechniqueEntity
|
||||||
from app.models.technique import Technique
|
from app.models.technique import Technique
|
||||||
|
|
||||||
|
|
||||||
def recalculate_technique_status(db: Session, technique: Technique) -> None:
|
def recalculate_technique_status(db: Session, technique: Technique) -> None:
|
||||||
"""Recompute ``technique.status_global`` from its tests and commit.
|
"""Recompute ``technique.status_global`` from its tests.
|
||||||
|
|
||||||
Rules (v2)
|
``db`` is accepted for backward compatibility but is not used
|
||||||
----------
|
directly — test data comes from the ORM relationship.
|
||||||
1. No tests → ``not_evaluated``
|
|
||||||
2. All tests ``validated`` → look at detection results:
|
|
||||||
- All ``detected`` → ``validated``
|
|
||||||
- Any ``partially_detected`` → ``partial``
|
|
||||||
- Otherwise → ``not_covered``
|
|
||||||
3. Some tests ``validated``, others still in progress → ``partial``
|
|
||||||
4. All tests in intermediate states (no validated) → ``in_progress``
|
|
||||||
"""
|
"""
|
||||||
tests = technique.tests
|
entity = TechniqueEntity.from_orm(technique)
|
||||||
|
test_snapshots = [
|
||||||
if not tests:
|
(t.state, t.detection_result) for t in technique.tests
|
||||||
technique.status_global = TechniqueStatus.not_evaluated
|
]
|
||||||
elif all(t.state == TestState.validated for t in tests):
|
entity.recalculate_status(test_snapshots)
|
||||||
# All validated — inspect detection results
|
technique.status_global = entity.status_global
|
||||||
results = [t.detection_result for t in tests if t.detection_result]
|
|
||||||
if results and all(str(r) == "detected" or r == "detected" for r in results):
|
|
||||||
technique.status_global = TechniqueStatus.validated
|
|
||||||
elif any(str(r) == "partially_detected" or r == "partially_detected" for r in results):
|
|
||||||
technique.status_global = TechniqueStatus.partial
|
|
||||||
else:
|
|
||||||
technique.status_global = TechniqueStatus.not_covered
|
|
||||||
elif any(t.state == TestState.validated for t in tests):
|
|
||||||
technique.status_global = TechniqueStatus.partial
|
|
||||||
else:
|
|
||||||
technique.status_global = TechniqueStatus.in_progress
|
|
||||||
|
|||||||
@@ -0,0 +1,310 @@
|
|||||||
|
"""Threat actor data service.
|
||||||
|
|
||||||
|
Extracts query and business logic from the threat_actors router so
|
||||||
|
that the router remains a thin HTTP adapter.
|
||||||
|
|
||||||
|
This module is framework-agnostic: no FastAPI imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import case, func, or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.domain.errors import EntityNotFoundError
|
||||||
|
from app.models.enums import TechniqueStatus
|
||||||
|
from app.models.test import Test
|
||||||
|
from app.models.test_template import TestTemplate
|
||||||
|
from app.models.threat_actor import ThreatActor, ThreatActorTechnique
|
||||||
|
from app.models.technique import Technique
|
||||||
|
from app.utils import escape_like
|
||||||
|
|
||||||
|
|
||||||
|
# ── Public service functions ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def list_actors(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
search: str | None = None,
|
||||||
|
country: str | None = None,
|
||||||
|
motivation: str | None = None,
|
||||||
|
sophistication: str | None = None,
|
||||||
|
target_sectors: str | None = None,
|
||||||
|
offset: int = 0,
|
||||||
|
limit: int = 50,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""List threat actors with optional filters, pagination, and coverage stats.
|
||||||
|
|
||||||
|
Uses grouped subqueries to avoid N+1: technique counts and coverage
|
||||||
|
counts are fetched in one query per page.
|
||||||
|
"""
|
||||||
|
query = db.query(ThreatActor)
|
||||||
|
|
||||||
|
if search:
|
||||||
|
pattern = f"%{escape_like(search)}%"
|
||||||
|
query = query.filter(
|
||||||
|
or_(
|
||||||
|
ThreatActor.name.ilike(pattern),
|
||||||
|
ThreatActor.description.ilike(pattern),
|
||||||
|
func.cast(ThreatActor.aliases, func.text()).ilike(pattern),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if country:
|
||||||
|
query = query.filter(ThreatActor.country == country)
|
||||||
|
|
||||||
|
if motivation:
|
||||||
|
query = query.filter(ThreatActor.motivation == motivation)
|
||||||
|
|
||||||
|
if sophistication:
|
||||||
|
query = query.filter(ThreatActor.sophistication == sophistication)
|
||||||
|
|
||||||
|
if target_sectors:
|
||||||
|
query = query.filter(
|
||||||
|
func.cast(ThreatActor.target_sectors, func.text()).ilike(
|
||||||
|
f"%{escape_like(target_sectors)}%"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
total = query.count()
|
||||||
|
actors = (
|
||||||
|
query.order_by(ThreatActor.name).offset(offset).limit(limit).all()
|
||||||
|
)
|
||||||
|
|
||||||
|
if not actors:
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"offset": offset,
|
||||||
|
"limit": limit,
|
||||||
|
"items": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
actor_ids = [a.id for a in actors]
|
||||||
|
|
||||||
|
# Single grouped query: tech_count and covered_count per actor
|
||||||
|
counts_rows = (
|
||||||
|
db.query(
|
||||||
|
ThreatActorTechnique.threat_actor_id,
|
||||||
|
func.count(ThreatActorTechnique.id).label("tech_count"),
|
||||||
|
func.sum(
|
||||||
|
case(
|
||||||
|
(
|
||||||
|
Technique.status_global.in_([
|
||||||
|
TechniqueStatus.validated,
|
||||||
|
TechniqueStatus.partial,
|
||||||
|
]),
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
else_=0,
|
||||||
|
)
|
||||||
|
).label("covered_count"),
|
||||||
|
)
|
||||||
|
.join(Technique, ThreatActorTechnique.technique_id == Technique.id)
|
||||||
|
.filter(ThreatActorTechnique.threat_actor_id.in_(actor_ids))
|
||||||
|
.group_by(ThreatActorTechnique.threat_actor_id)
|
||||||
|
).all()
|
||||||
|
|
||||||
|
counts_map = {
|
||||||
|
str(row.threat_actor_id): {
|
||||||
|
"tech_count": row.tech_count,
|
||||||
|
"covered_count": row.covered_count or 0,
|
||||||
|
}
|
||||||
|
for row in counts_rows
|
||||||
|
}
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for actor in actors:
|
||||||
|
cnt = counts_map.get(str(actor.id), {"tech_count": 0, "covered_count": 0})
|
||||||
|
tech_count = cnt["tech_count"]
|
||||||
|
covered = cnt["covered_count"]
|
||||||
|
coverage_pct = round((covered / tech_count * 100), 1) if tech_count > 0 else 0.0
|
||||||
|
|
||||||
|
results.append({
|
||||||
|
"id": str(actor.id),
|
||||||
|
"mitre_id": actor.mitre_id,
|
||||||
|
"name": actor.name,
|
||||||
|
"aliases": actor.aliases or [],
|
||||||
|
"country": actor.country,
|
||||||
|
"target_sectors": actor.target_sectors or [],
|
||||||
|
"target_regions": actor.target_regions or [],
|
||||||
|
"motivation": actor.motivation,
|
||||||
|
"sophistication": actor.sophistication,
|
||||||
|
"mitre_url": actor.mitre_url,
|
||||||
|
"technique_count": tech_count,
|
||||||
|
"coverage_pct": coverage_pct,
|
||||||
|
"is_active": actor.is_active,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"offset": offset,
|
||||||
|
"limit": limit,
|
||||||
|
"items": results,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_actor_detail(db: Session, actor_id: str) -> dict[str, Any]:
|
||||||
|
"""Get detailed threat actor with techniques.
|
||||||
|
|
||||||
|
Raises EntityNotFoundError if the actor does not exist.
|
||||||
|
"""
|
||||||
|
actor = db.query(ThreatActor).filter(ThreatActor.id == actor_id).first()
|
||||||
|
if not actor:
|
||||||
|
raise EntityNotFoundError("Threat actor", actor_id)
|
||||||
|
|
||||||
|
actor_techniques = (
|
||||||
|
db.query(ThreatActorTechnique, Technique)
|
||||||
|
.join(Technique, ThreatActorTechnique.technique_id == Technique.id)
|
||||||
|
.filter(ThreatActorTechnique.threat_actor_id == actor.id)
|
||||||
|
.order_by(Technique.mitre_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
techniques_list = [
|
||||||
|
{
|
||||||
|
"technique_id": str(tech.id),
|
||||||
|
"mitre_id": tech.mitre_id,
|
||||||
|
"name": tech.name,
|
||||||
|
"tactic": tech.tactic,
|
||||||
|
"status_global": tech.status_global.value if tech.status_global else None,
|
||||||
|
"usage_description": at.usage_description,
|
||||||
|
"first_seen_using": at.first_seen_using,
|
||||||
|
}
|
||||||
|
for at, tech in actor_techniques
|
||||||
|
]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": str(actor.id),
|
||||||
|
"mitre_id": actor.mitre_id,
|
||||||
|
"name": actor.name,
|
||||||
|
"aliases": actor.aliases or [],
|
||||||
|
"description": actor.description,
|
||||||
|
"country": actor.country,
|
||||||
|
"target_sectors": actor.target_sectors or [],
|
||||||
|
"target_regions": actor.target_regions or [],
|
||||||
|
"motivation": actor.motivation,
|
||||||
|
"sophistication": actor.sophistication,
|
||||||
|
"first_seen": actor.first_seen,
|
||||||
|
"last_seen": actor.last_seen,
|
||||||
|
"references": actor.references or [],
|
||||||
|
"mitre_url": actor.mitre_url,
|
||||||
|
"is_active": actor.is_active,
|
||||||
|
"techniques": techniques_list,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_actor_coverage(db: Session, actor_id: str) -> dict[str, Any]:
|
||||||
|
"""Calculate coverage percentage against a specific threat actor.
|
||||||
|
|
||||||
|
Raises EntityNotFoundError if the actor does not exist.
|
||||||
|
"""
|
||||||
|
actor = db.query(ThreatActor).filter(ThreatActor.id == actor_id).first()
|
||||||
|
if not actor:
|
||||||
|
raise EntityNotFoundError("Threat actor", actor_id)
|
||||||
|
|
||||||
|
actor_techniques = (
|
||||||
|
db.query(Technique)
|
||||||
|
.join(ThreatActorTechnique, ThreatActorTechnique.technique_id == Technique.id)
|
||||||
|
.filter(ThreatActorTechnique.threat_actor_id == actor.id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
total = len(actor_techniques)
|
||||||
|
if total == 0:
|
||||||
|
return {
|
||||||
|
"actor_id": str(actor.id),
|
||||||
|
"actor_name": actor.name,
|
||||||
|
"total_techniques": 0,
|
||||||
|
"coverage_pct": 0.0,
|
||||||
|
"breakdown": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
breakdown: dict[str, int] = {}
|
||||||
|
for tech in actor_techniques:
|
||||||
|
status = tech.status_global.value if tech.status_global else "not_evaluated"
|
||||||
|
breakdown[status] = breakdown.get(status, 0) + 1
|
||||||
|
|
||||||
|
covered = breakdown.get("validated", 0) + breakdown.get("partial", 0)
|
||||||
|
coverage_pct = round((covered / total * 100), 1)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"actor_id": str(actor.id),
|
||||||
|
"actor_name": actor.name,
|
||||||
|
"total_techniques": total,
|
||||||
|
"covered": covered,
|
||||||
|
"coverage_pct": coverage_pct,
|
||||||
|
"breakdown": breakdown,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_actor_gaps(db: Session, actor_id: str) -> dict[str, Any]:
|
||||||
|
"""Identify techniques of this actor that are not fully validated.
|
||||||
|
|
||||||
|
Raises EntityNotFoundError if the actor does not exist.
|
||||||
|
"""
|
||||||
|
actor = db.query(ThreatActor).filter(ThreatActor.id == actor_id).first()
|
||||||
|
if not actor:
|
||||||
|
raise EntityNotFoundError("Threat actor", actor_id)
|
||||||
|
|
||||||
|
gap_techniques = (
|
||||||
|
db.query(Technique, ThreatActorTechnique)
|
||||||
|
.join(ThreatActorTechnique, ThreatActorTechnique.technique_id == Technique.id)
|
||||||
|
.filter(ThreatActorTechnique.threat_actor_id == actor.id)
|
||||||
|
.filter(Technique.status_global != TechniqueStatus.validated)
|
||||||
|
.order_by(Technique.mitre_id)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
if not gap_techniques:
|
||||||
|
return {
|
||||||
|
"actor_id": str(actor.id),
|
||||||
|
"actor_name": actor.name,
|
||||||
|
"total_gaps": 0,
|
||||||
|
"gaps": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
technique_ids = [tech.id for tech, _ in gap_techniques]
|
||||||
|
mitre_ids = [tech.mitre_id for tech, _ in gap_techniques]
|
||||||
|
|
||||||
|
# Batch template counts by mitre_technique_id
|
||||||
|
template_counts = (
|
||||||
|
db.query(TestTemplate.mitre_technique_id, func.count(TestTemplate.id).label("cnt"))
|
||||||
|
.filter(TestTemplate.mitre_technique_id.in_(mitre_ids))
|
||||||
|
.filter(TestTemplate.is_active == True)
|
||||||
|
.group_by(TestTemplate.mitre_technique_id)
|
||||||
|
).all()
|
||||||
|
template_map = {row.mitre_technique_id: row.cnt for row in template_counts}
|
||||||
|
|
||||||
|
# Batch test counts by technique_id
|
||||||
|
test_counts = (
|
||||||
|
db.query(Test.technique_id, func.count(Test.id).label("cnt"))
|
||||||
|
.filter(Test.technique_id.in_(technique_ids))
|
||||||
|
.group_by(Test.technique_id)
|
||||||
|
).all()
|
||||||
|
test_map = {str(row.technique_id): row.cnt for row in test_counts}
|
||||||
|
|
||||||
|
gaps = []
|
||||||
|
for tech, at in gap_techniques:
|
||||||
|
template_count = template_map.get(tech.mitre_id, 0)
|
||||||
|
test_count = test_map.get(str(tech.id), 0)
|
||||||
|
gaps.append({
|
||||||
|
"technique_id": str(tech.id),
|
||||||
|
"mitre_id": tech.mitre_id,
|
||||||
|
"name": tech.name,
|
||||||
|
"tactic": tech.tactic,
|
||||||
|
"status_global": tech.status_global.value if tech.status_global else None,
|
||||||
|
"usage_description": at.usage_description,
|
||||||
|
"available_templates": template_count,
|
||||||
|
"existing_tests": test_count,
|
||||||
|
"has_templates": template_count > 0,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"actor_id": str(actor.id),
|
||||||
|
"actor_name": actor.name,
|
||||||
|
"total_gaps": len(gaps),
|
||||||
|
"gaps": gaps,
|
||||||
|
}
|
||||||
@@ -147,8 +147,8 @@ def test_pipeline_metrics_endpoint_exists():
|
|||||||
|
|
||||||
def test_pipeline_metrics_queries_all_states():
|
def test_pipeline_metrics_queries_all_states():
|
||||||
"""Pipeline endpoint groups by all test states."""
|
"""Pipeline endpoint groups by all test states."""
|
||||||
from app.routers.metrics import test_pipeline
|
from app.services.metrics_query_service import get_test_pipeline_counts
|
||||||
source = inspect.getsource(test_pipeline)
|
source = inspect.getsource(get_test_pipeline_counts)
|
||||||
|
|
||||||
assert "Test.state" in source, "Must query Test.state"
|
assert "Test.state" in source, "Must query Test.state"
|
||||||
assert "group_by" in source, "Must group by state"
|
assert "group_by" in source, "Must group by state"
|
||||||
@@ -169,8 +169,8 @@ def test_team_activity_endpoint_exists():
|
|||||||
|
|
||||||
def test_team_activity_calculates_both_teams():
|
def test_team_activity_calculates_both_teams():
|
||||||
"""Team activity endpoint returns data for both Red and Blue teams."""
|
"""Team activity endpoint returns data for both Red and Blue teams."""
|
||||||
from app.routers.metrics import team_activity
|
from app.services.metrics_query_service import get_team_activity
|
||||||
source = inspect.getsource(team_activity)
|
source = inspect.getsource(get_team_activity)
|
||||||
|
|
||||||
assert "Red Team" in source or "red" in source.lower(), "Must include Red Team data"
|
assert "Red Team" in source or "red" in source.lower(), "Must include Red Team data"
|
||||||
assert "Blue Team" in source or "blue" in source.lower(), "Must include Blue Team data"
|
assert "Blue Team" in source or "blue" in source.lower(), "Must include Blue Team data"
|
||||||
@@ -180,8 +180,8 @@ def test_team_activity_calculates_both_teams():
|
|||||||
|
|
||||||
def test_team_activity_red_pending_states():
|
def test_team_activity_red_pending_states():
|
||||||
"""Red Team pending includes draft and red_executing."""
|
"""Red Team pending includes draft and red_executing."""
|
||||||
from app.routers.metrics import team_activity
|
from app.services.metrics_query_service import get_team_activity
|
||||||
source = inspect.getsource(team_activity)
|
source = inspect.getsource(get_team_activity)
|
||||||
|
|
||||||
assert "draft" in source, "Red pending must include draft"
|
assert "draft" in source, "Red pending must include draft"
|
||||||
assert "red_executing" in source, "Red pending must include red_executing"
|
assert "red_executing" in source, "Red pending must include red_executing"
|
||||||
@@ -189,8 +189,8 @@ def test_team_activity_red_pending_states():
|
|||||||
|
|
||||||
def test_team_activity_blue_pending_states():
|
def test_team_activity_blue_pending_states():
|
||||||
"""Blue Team pending includes blue_evaluating."""
|
"""Blue Team pending includes blue_evaluating."""
|
||||||
from app.routers.metrics import team_activity
|
from app.services.metrics_query_service import get_team_activity
|
||||||
source = inspect.getsource(team_activity)
|
source = inspect.getsource(get_team_activity)
|
||||||
|
|
||||||
assert "blue_evaluating" in source, "Blue pending must include blue_evaluating"
|
assert "blue_evaluating" in source, "Blue pending must include blue_evaluating"
|
||||||
|
|
||||||
@@ -348,8 +348,8 @@ def test_validation_rate_endpoint_exists():
|
|||||||
|
|
||||||
def test_validation_rate_queries_both_roles():
|
def test_validation_rate_queries_both_roles():
|
||||||
"""Validation rate endpoint returns data for both red_lead and blue_lead."""
|
"""Validation rate endpoint returns data for both red_lead and blue_lead."""
|
||||||
from app.routers.metrics import validation_rate
|
from app.services.metrics_query_service import get_validation_rate
|
||||||
source = inspect.getsource(validation_rate)
|
source = inspect.getsource(get_validation_rate)
|
||||||
|
|
||||||
assert "red_validation_status" in source, "Must query red_validation_status"
|
assert "red_validation_status" in source, "Must query red_validation_status"
|
||||||
assert "blue_validation_status" in source, "Must query blue_validation_status"
|
assert "blue_validation_status" in source, "Must query blue_validation_status"
|
||||||
@@ -372,11 +372,12 @@ def test_recent_tests_endpoint_exists():
|
|||||||
|
|
||||||
def test_recent_tests_limits_to_10():
|
def test_recent_tests_limits_to_10():
|
||||||
"""Recent tests endpoint limits to 10 results."""
|
"""Recent tests endpoint limits to 10 results."""
|
||||||
from app.routers.metrics import recent_tests
|
from app.services.metrics_query_service import get_recent_tests
|
||||||
source = inspect.getsource(recent_tests)
|
source = inspect.getsource(get_recent_tests)
|
||||||
|
|
||||||
assert "limit(10)" in source or ".limit(10)" in source, \
|
assert ".limit(" in source, "Must limit query results"
|
||||||
"Must limit to 10 recent tests"
|
assert "limit" in source and ("10" in source or "limit" in source), \
|
||||||
|
"Must have limit param or default 10"
|
||||||
assert "created_at" in source, "Must order by created_at"
|
assert "created_at" in source, "Must order by created_at"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+394
@@ -0,0 +1,394 @@
|
|||||||
|
# Aegis — Architecture Decision Records (ADR)
|
||||||
|
|
||||||
|
> **Date:** February 11, 2026
|
||||||
|
> **Status:** All decisions are **Accepted** and currently in effect.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Index
|
||||||
|
|
||||||
|
| ADR | Title | Status |
|
||||||
|
|-----|-------|--------|
|
||||||
|
| [ADR-001](#adr-001-fastapi-as-backend-framework) | FastAPI as Backend Framework | Accepted |
|
||||||
|
| [ADR-002](#adr-002-postgresql-with-jsonb-as-primary-database) | PostgreSQL with JSONB as Primary Database | Accepted |
|
||||||
|
| [ADR-003](#adr-003-minio-for-evidence-storage) | MinIO for Evidence Storage | Accepted |
|
||||||
|
| [ADR-004](#adr-004-docker-compose-for-deployment) | Docker Compose for Deployment | Accepted |
|
||||||
|
| [ADR-005](#adr-005-modular-monolith-over-microservices) | Modular Monolith over Microservices | Accepted |
|
||||||
|
| [ADR-006](#adr-006-apscheduler-in-process-over-external-job-system) | APScheduler In-Process over External Job System | Accepted |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ADR-001: FastAPI as Backend Framework
|
||||||
|
|
||||||
|
**Date:** Project inception
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
### Context
|
||||||
|
|
||||||
|
Aegis is an internal security platform for managing MITRE ATT&CK coverage through Red/Blue team validation workflows. The backend must:
|
||||||
|
|
||||||
|
- Expose a REST API consumed by a React SPA (21 pages, 80+ endpoints).
|
||||||
|
- Handle CRUD operations for 18+ domain entities with complex filtering and joins.
|
||||||
|
- Support file uploads (evidence) and streaming downloads (CSV/JSON exports).
|
||||||
|
- Integrate with external APIs (MITRE TAXII 2.0, GitHub REST, D3FEND REST).
|
||||||
|
- Enforce RBAC authorization across 6 roles.
|
||||||
|
- Be developed and maintained by a small team requiring fast iteration.
|
||||||
|
- Run in a containerized environment with Python as the team's primary language.
|
||||||
|
|
||||||
|
### Decision
|
||||||
|
|
||||||
|
We chose **FastAPI** as the backend framework, served by **Uvicorn** (ASGI).
|
||||||
|
|
||||||
|
Key factors:
|
||||||
|
- **Automatic OpenAPI/Swagger** generation from type hints reduces documentation burden for 80+ endpoints.
|
||||||
|
- **Pydantic integration** provides request/response validation with zero boilerplate, critical for a schema-heavy domain (test workflows, scoring payloads, compliance data).
|
||||||
|
- **`Depends()` system** provides clean dependency injection for auth, DB sessions, and role checks without a third-party DI container.
|
||||||
|
- **Async-capable** but allows synchronous route handlers, which matters because SQLAlchemy (sync) is the ORM and all external data imports are CPU/IO-bound synchronous operations.
|
||||||
|
- **Performance** is sufficient for an internal tool (< 100 concurrent users) without needing Go/Rust-level throughput.
|
||||||
|
- **Python ecosystem** gives direct access to `taxii2-client`, `pySigma`, `boto3`, `PyYAML`, and `toml` — all required for the 8 external data source integrations.
|
||||||
|
|
||||||
|
### Consequences
|
||||||
|
|
||||||
|
**Positive:**
|
||||||
|
- Swagger UI available in development (`/docs`) for rapid API exploration and testing.
|
||||||
|
- Pydantic schemas act as living documentation for the API contract.
|
||||||
|
- `Depends()` chain for `get_db` → `get_current_user` → `require_role()` is concise and composable.
|
||||||
|
- `python-jose` + `passlib` integrate naturally for JWT/bcrypt auth.
|
||||||
|
- SlowAPI integrates directly with FastAPI for rate limiting.
|
||||||
|
|
||||||
|
**Negative:**
|
||||||
|
- The `Depends()` system encourages passing `db: Session` directly into route handlers, which has led to routers containing raw SQLAlchemy queries instead of delegating to a service/repository layer (see ADR analysis — 11 of 21 routers query the DB directly).
|
||||||
|
- Synchronous route handlers block the event loop when performing long operations (MITRE sync ZIP downloads can take 30+ seconds), mitigated by Nginx proxy timeout of 300s.
|
||||||
|
- No built-in background task system beyond `BackgroundTasks` (which is request-scoped), requiring APScheduler for scheduled jobs (see ADR-006).
|
||||||
|
|
||||||
|
**Risks:**
|
||||||
|
- FastAPI's ease of putting logic in route handlers has contributed to "fat controllers" — this is a developer discipline issue, not a framework limitation.
|
||||||
|
|
||||||
|
### Alternatives Considered
|
||||||
|
|
||||||
|
| Alternative | Reason Rejected |
|
||||||
|
|------------|-----------------|
|
||||||
|
| **Django + DRF** | Heavier ORM opinions, admin panel unnecessary, slower startup. Django's ORM lacks SQLAlchemy's flexibility with JSONB and complex joins. |
|
||||||
|
| **Flask + Flask-RESTful** | No built-in validation, no auto-generated OpenAPI, manual Swagger setup. Would require marshmallow or similar for schema validation. |
|
||||||
|
| **Go (Gin/Echo)** | Team's primary expertise is Python. The 8 data source integrations rely heavily on Python libraries (pySigma, taxii2-client, PyYAML). |
|
||||||
|
| **NestJS (Node.js)** | Would split the team across two runtimes. Python libraries for STIX/TAXII and Sigma rule parsing have no mature Node.js equivalents. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ADR-002: PostgreSQL with JSONB as Primary Database
|
||||||
|
|
||||||
|
**Date:** Project inception
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
### Context
|
||||||
|
|
||||||
|
Aegis manages a complex relational domain: techniques have tests, tests belong to campaigns, threat actors map to techniques, compliance controls map to techniques, detection rules map to techniques and tests. This is a deeply relational model with 18+ tables and many-to-many relationships.
|
||||||
|
|
||||||
|
However, several entities also carry semi-structured data that varies by source:
|
||||||
|
- **Audit logs** — `details` field contains arbitrary action metadata (different structure per action type).
|
||||||
|
- **Threat actors** — `aliases`, `target_sectors`, `target_regions`, `references` are variable-length arrays/objects from STIX 2.0 bundles.
|
||||||
|
- **Detection rules** — `platforms` (array), `log_sources` (object with varying keys like `product`, `service`, `category`).
|
||||||
|
- **Data sources** — `last_sync_stats` (object with import-specific counters), `config` (source-specific configuration).
|
||||||
|
- **Techniques** — `platforms` (array of OS names from ATT&CK).
|
||||||
|
- **Campaigns** — `tags` (user-defined array).
|
||||||
|
|
||||||
|
This data is imported from external sources with varying schemas (STIX JSON, Sigma YAML, Elastic TOML) and must be stored without rigid column definitions.
|
||||||
|
|
||||||
|
### Decision
|
||||||
|
|
||||||
|
We chose **PostgreSQL 15** as the primary database, using its native **JSONB** column type for semi-structured fields alongside traditional relational columns for the core domain.
|
||||||
|
|
||||||
|
The schema is managed by **Alembic** (18 migration versions) with **SQLAlchemy** ORM using `sqlalchemy.dialects.postgresql.JSONB`.
|
||||||
|
|
||||||
|
### Consequences
|
||||||
|
|
||||||
|
**Positive:**
|
||||||
|
- Relational integrity enforced with foreign keys for the core domain (test → technique, campaign → test, evidence → test, etc.).
|
||||||
|
- JSONB columns store variable-structure data without schema migrations when external sources change their format.
|
||||||
|
- JSONB supports GIN indexing for efficient containment queries (`@>` operator) on arrays like `platforms` and `target_sectors`.
|
||||||
|
- Single database to operate — no need for a separate document store.
|
||||||
|
- PostgreSQL's mature ecosystem: `pg_dump` for backups, `pg_isready` for health checks, extensive monitoring tooling.
|
||||||
|
- SQLAlchemy's `JSONB` type allows Python dict/list access with full query support.
|
||||||
|
|
||||||
|
**Negative:**
|
||||||
|
- JSONB fields bypass ORM-level validation — the schema for `details`, `config`, `references` etc. is only enforced by application code (Pydantic schemas on input), not by the database.
|
||||||
|
- Complex queries mixing relational joins with JSONB containment can be harder to optimize and debug.
|
||||||
|
- No GIN indexes are currently defined in migrations for JSONB columns, meaning array containment queries may perform full scans on large datasets.
|
||||||
|
- JSONB fields in audit logs make structured querying across action types difficult (e.g., "find all audit entries where details.old_state = 'draft'").
|
||||||
|
|
||||||
|
**Risks:**
|
||||||
|
- As JSONB usage grows, the boundary between "should be a column" and "should be JSONB" can blur. Currently well-contained to arrays and metadata fields.
|
||||||
|
|
||||||
|
### Alternatives Considered
|
||||||
|
|
||||||
|
| Alternative | Reason Rejected |
|
||||||
|
|------------|-----------------|
|
||||||
|
| **PostgreSQL without JSONB** | Would require separate junction tables for every array field (technique_platforms, actor_aliases, actor_sectors, etc.), adding 10+ tables for data that is always read as a whole array. |
|
||||||
|
| **MongoDB** | The core domain is deeply relational (techniques ↔ tests ↔ campaigns ↔ threat actors). Modeling this in MongoDB would require denormalization, embedded documents, or manual reference integrity — trading JSONB flexibility for relational integrity loss. |
|
||||||
|
| **PostgreSQL + MongoDB (dual)** | Operational complexity of two database systems is unjustified for the current JSONB usage (~12 columns across 6 tables). |
|
||||||
|
| **MySQL 8 with JSON** | PostgreSQL's JSONB is binary-indexed and faster for containment queries. MySQL's JSON type is text-based with function-based indexing. PostgreSQL also has superior support for UUID primary keys (native type vs BINARY(16)). |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ADR-003: MinIO for Evidence Storage
|
||||||
|
|
||||||
|
**Date:** Project inception
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
### Context
|
||||||
|
|
||||||
|
The Red/Blue team validation workflow requires both teams to upload evidence files (screenshots, log files, PCAPs, documents) to support their test findings. Requirements:
|
||||||
|
|
||||||
|
- Files range from small screenshots (KB) to large PCAPs (hundreds of MB).
|
||||||
|
- Files must be associated with specific tests and teams (red/blue).
|
||||||
|
- Files must be downloadable by authorized users via the browser.
|
||||||
|
- Storage must be independent from the application database (no BLOBs in PostgreSQL).
|
||||||
|
- The platform is deployed on-premise via Docker Compose — cloud-native S3 is not available.
|
||||||
|
- The upload/download API must be simple and well-supported in Python.
|
||||||
|
|
||||||
|
### Decision
|
||||||
|
|
||||||
|
We chose **MinIO** as an S3-compatible object storage system, accessed via **boto3** (AWS S3 SDK for Python).
|
||||||
|
|
||||||
|
Implementation details:
|
||||||
|
- A single `evidence` bucket is auto-created on backend startup (`ensure_bucket_exists()`).
|
||||||
|
- Files are uploaded with `put_object()` using a generated UUID-based key.
|
||||||
|
- Downloads use presigned URLs (`generate_presigned_url()`) with 1-hour expiration.
|
||||||
|
- The MinIO client is a module-level singleton in `storage.py`.
|
||||||
|
- Evidence metadata (filename, MIME type, size, team, test association) is stored in PostgreSQL; only the binary content lives in MinIO.
|
||||||
|
|
||||||
|
### Consequences
|
||||||
|
|
||||||
|
**Positive:**
|
||||||
|
- S3-compatible API means zero code changes if migrating to AWS S3, GCS, or any S3-compatible service.
|
||||||
|
- boto3 is the most mature and well-documented S3 client library in Python.
|
||||||
|
- Presigned URLs offload download bandwidth from the backend — the browser fetches directly from MinIO.
|
||||||
|
- Binary data stays out of PostgreSQL, keeping the database lean and backups fast.
|
||||||
|
- MinIO runs as a single Docker container with a persistent volume — simple to deploy and back up.
|
||||||
|
- MinIO Console (port 9001) provides a web UI for administrators to inspect stored files.
|
||||||
|
|
||||||
|
**Negative:**
|
||||||
|
- Presigned URLs currently point to `minio:9000` (Docker internal hostname), which is not accessible from the browser in production without additional Nginx configuration or a public MinIO endpoint.
|
||||||
|
- No file virus scanning or content validation before storage.
|
||||||
|
- No lifecycle policies configured (no automatic deletion of old evidence).
|
||||||
|
- The module-level singleton client means the MinIO connection configuration cannot be changed at runtime (acceptable for the current deployment model).
|
||||||
|
|
||||||
|
**Risks:**
|
||||||
|
- If MinIO container is lost and the volume is not backed up, all evidence files are permanently lost. Evidence metadata in PostgreSQL would reference non-existent files.
|
||||||
|
|
||||||
|
### Alternatives Considered
|
||||||
|
|
||||||
|
| Alternative | Reason Rejected |
|
||||||
|
|------------|-----------------|
|
||||||
|
| **PostgreSQL BYTEA/BLOB** | Storing binary files in the database bloats backups, degrades query performance, and makes streaming large files complex. PostgreSQL is not designed as a file store. |
|
||||||
|
| **Local filesystem** | Not portable across container restarts without host volume mounts. No presigned URL support, requiring the backend to proxy all downloads. No built-in replication or management UI. |
|
||||||
|
| **AWS S3** | Requires cloud account and internet connectivity. The platform is designed for on-premise deployment where external cloud services may not be permitted. |
|
||||||
|
| **SeaweedFS** | Less mature ecosystem, smaller community. The S3-compatible layer is less complete than MinIO's. boto3 compatibility is not guaranteed. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ADR-004: Docker Compose for Deployment
|
||||||
|
|
||||||
|
**Date:** Project inception
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
### Context
|
||||||
|
|
||||||
|
Aegis is a multi-component platform deployed on-premise within organizations' security environments:
|
||||||
|
|
||||||
|
- 4 services: Frontend (Nginx), Backend (Uvicorn), PostgreSQL, MinIO.
|
||||||
|
- Target environments range from a single server to small clusters.
|
||||||
|
- Security teams typically have Docker available but may not have Kubernetes.
|
||||||
|
- The platform must be installable by a security engineer (not necessarily a DevOps specialist).
|
||||||
|
- Both development and production environments should use the same orchestration approach for consistency.
|
||||||
|
|
||||||
|
### Decision
|
||||||
|
|
||||||
|
We chose **Docker Compose** as the deployment and orchestration tool, with two compose files:
|
||||||
|
|
||||||
|
- `docker-compose.yml` — Development: source volumes mounted, dev servers, exposed ports.
|
||||||
|
- `docker-compose.prod.yml` — Production: multi-stage builds, Nginx serving static assets, only frontend port exposed, `SECRET_KEY` required.
|
||||||
|
|
||||||
|
Supporting infrastructure:
|
||||||
|
- `scripts/install.sh` — Interactive production installer that generates secrets, prompts for configuration, writes `.env`, and runs `docker compose up -d --build`.
|
||||||
|
- `scripts/init.sh` — Development setup that waits for services, runs migrations, and seeds data.
|
||||||
|
- All services connected via a `aegis-network` bridge network.
|
||||||
|
- Named volumes for PostgreSQL and MinIO data persistence.
|
||||||
|
- Health checks on PostgreSQL (`pg_isready`) and backend (`/health`).
|
||||||
|
- Service dependency ordering: backend waits for `postgres: service_healthy` and `minio: service_started`.
|
||||||
|
|
||||||
|
### Consequences
|
||||||
|
|
||||||
|
**Positive:**
|
||||||
|
- Single-command deployment: `docker compose -f docker-compose.prod.yml up -d --build`.
|
||||||
|
- The `install.sh` wizard makes production setup accessible to non-DevOps personnel.
|
||||||
|
- Consistent environments between development and production (same containers, same network topology).
|
||||||
|
- Named volumes survive container rebuilds — data persists across upgrades.
|
||||||
|
- No external dependencies beyond Docker and Docker Compose.
|
||||||
|
- Multi-stage Dockerfile for frontend produces a minimal Nginx image (~25MB) from a full Node.js build stage.
|
||||||
|
- Non-root user (`appuser`, UID 1001) in backend Dockerfile follows container security best practices.
|
||||||
|
|
||||||
|
**Negative:**
|
||||||
|
- No built-in horizontal scaling — running multiple backend instances requires manual Nginx upstream configuration and a shared token blacklist (currently in-memory).
|
||||||
|
- No rolling deployments — `docker compose up -d --build` causes brief downtime during image rebuilds.
|
||||||
|
- No built-in secrets management — secrets are in `.env` files on the host filesystem.
|
||||||
|
- No container orchestration beyond restart policies (`restart: always`).
|
||||||
|
- No centralized logging — each container logs to its own stdout/stderr.
|
||||||
|
|
||||||
|
**Risks:**
|
||||||
|
- Single point of failure: if the host machine goes down, all services go down.
|
||||||
|
- No automated backup strategy — `pg_dump` is documented but not automated.
|
||||||
|
|
||||||
|
### Alternatives Considered
|
||||||
|
|
||||||
|
| Alternative | Reason Rejected |
|
||||||
|
|------------|-----------------|
|
||||||
|
| **Kubernetes (k8s)** | Significantly higher operational complexity. Requires a cluster, kubectl expertise, Helm charts or manifests, ingress controllers, PVCs. Overkill for a single-server deployment targeting security teams. |
|
||||||
|
| **Docker Swarm** | Adds orchestration complexity with minimal benefit over Compose for < 5 services. The project does not need multi-node scheduling or service mesh. Swarm's future is uncertain compared to Compose V2. |
|
||||||
|
| **Bare metal / systemd** | Loses containerization benefits (isolation, reproducibility, dependency management). Would require manual installation of Python, Node.js, PostgreSQL, MinIO on each target system. |
|
||||||
|
| **Ansible + Docker** | Adds a configuration management layer that is unnecessary for a 4-service application. Could be valuable in the future for multi-server deployments but is premature now. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ADR-005: Modular Monolith over Microservices
|
||||||
|
|
||||||
|
**Date:** Project inception
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
### Context
|
||||||
|
|
||||||
|
Aegis has distinct functional domains that could theoretically be separate services:
|
||||||
|
- **Test Workflow** — Red/Blue validation state machine, evidence management.
|
||||||
|
- **Coverage Analytics** — Scoring engine, heatmaps, metrics, reports.
|
||||||
|
- **Data Import** — 8 external source integrations (MITRE, Sigma, Elastic, CALDERA, etc.).
|
||||||
|
- **Campaign Management** — Campaign lifecycle, scheduling, threat actor generation.
|
||||||
|
- **Compliance** — Framework mappings, gap analysis, control tracking.
|
||||||
|
- **User/Auth** — Authentication, RBAC, audit logging.
|
||||||
|
|
||||||
|
However:
|
||||||
|
- These domains share the same database and have tight data dependencies (e.g., scoring reads tests, techniques, detection rules, and D3FEND mappings in a single calculation).
|
||||||
|
- The development team is small.
|
||||||
|
- The deployment target is single-server Docker Compose.
|
||||||
|
- Latency between services would complicate the scoring engine (which aggregates across 5+ tables).
|
||||||
|
|
||||||
|
### Decision
|
||||||
|
|
||||||
|
We chose a **modular monolith** architecture: a single deployable backend process organized into internal modules (routers, services, models) rather than separate microservices.
|
||||||
|
|
||||||
|
Module boundaries:
|
||||||
|
- **Routers** (21 files) — HTTP endpoint definitions grouped by domain.
|
||||||
|
- **Services** (20 files) — Business logic grouped by capability (workflow, scoring, notifications, imports).
|
||||||
|
- **Models** (18 files) — ORM entities grouped by domain concept.
|
||||||
|
- **Schemas** (10 files) — Pydantic DTOs grouped by domain concept.
|
||||||
|
|
||||||
|
All modules share a single database, a single process, and a single deployment artifact.
|
||||||
|
|
||||||
|
### Consequences
|
||||||
|
|
||||||
|
**Positive:**
|
||||||
|
- No network overhead between domains — scoring can join 5+ tables in a single SQL query.
|
||||||
|
- Single deployment artifact simplifies CI/CD, monitoring, and debugging.
|
||||||
|
- Shared database means ACID transactions across domains (e.g., creating a test + logging the audit entry + sending a notification in one commit).
|
||||||
|
- No service discovery, API gateways, circuit breakers, or distributed tracing needed.
|
||||||
|
- Faster development iteration — change any module, rebuild one container.
|
||||||
|
|
||||||
|
**Negative:**
|
||||||
|
- All domains scale together — cannot scale the data import workers independently from the API.
|
||||||
|
- A bug in one module (e.g., a memory leak in scoring) can crash the entire application.
|
||||||
|
- Module boundaries are not enforced at the language level — routers currently import services and models freely across domains (e.g., `heatmap.py` imports 6 models from different domains).
|
||||||
|
- The monolith has grown to 21 routers and 20 services without explicit boundary enforcement, leading to "fat controllers" and cross-cutting concerns.
|
||||||
|
|
||||||
|
**Risks:**
|
||||||
|
- Without explicit module boundaries (enforced by code structure or linting rules), the modular monolith can degrade into a traditional monolith where everything depends on everything.
|
||||||
|
- The Clean Architecture refactor proposed in `ARCHITECTURAL_ANALYSIS.md` would restore module boundaries via the domain/application/infrastructure/presentation layers.
|
||||||
|
|
||||||
|
### Alternatives Considered
|
||||||
|
|
||||||
|
| Alternative | Reason Rejected |
|
||||||
|
|------------|-----------------|
|
||||||
|
| **Microservices** | The 8 data source integrations would each become a service, requiring inter-service communication for writing to the shared technique/rule tables. Scoring would need to call 3-4 services to gather data, adding latency and failure modes. Operational overhead (8+ containers, service mesh, distributed tracing) unjustified for a small team and single-server deployment. |
|
||||||
|
| **Microservices with shared DB** | Anti-pattern. Multiple services sharing a database lose the main benefit of microservices (independent deployment and schema evolution) while keeping the operational complexity. |
|
||||||
|
| **Modular monolith with enforced boundaries** | This is the recommended evolution (see ADR analysis). The current implementation has module structure but no boundary enforcement. Adding domain-layer interfaces (Protocol/ABC), a repository pattern, and import linting rules would achieve this without a microservices migration. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ADR-006: APScheduler In-Process over External Job System
|
||||||
|
|
||||||
|
**Date:** Project inception
|
||||||
|
**Status:** Accepted
|
||||||
|
|
||||||
|
### Context
|
||||||
|
|
||||||
|
Aegis requires periodic background tasks:
|
||||||
|
|
||||||
|
| Task | Frequency | Duration | Description |
|
||||||
|
|------|-----------|----------|-------------|
|
||||||
|
| MITRE ATT&CK sync | Every 24 hours | 30-120 seconds | Download STIX/TAXII feed, upsert ~700 techniques |
|
||||||
|
| Intel scan | Every 7 days | 10-60 seconds | Scan threat intelligence sources |
|
||||||
|
| Notification cleanup | Every 24 hours | < 5 seconds | Delete read notifications older than 90 days |
|
||||||
|
| Coverage snapshot | Weekly (Sunday 00:00) | 5-30 seconds | Capture point-in-time coverage state across all techniques |
|
||||||
|
| Recurring campaigns | Every 24 hours | < 10 seconds | Check and spawn due recurring test campaigns |
|
||||||
|
|
||||||
|
Requirements:
|
||||||
|
- Jobs must access the same database as the API.
|
||||||
|
- Jobs must not block API request handling.
|
||||||
|
- No additional infrastructure should be required beyond what Docker Compose already provides.
|
||||||
|
- Job failure should not crash the API server.
|
||||||
|
- Jobs do not need distributed execution (single-server deployment).
|
||||||
|
|
||||||
|
### Decision
|
||||||
|
|
||||||
|
We chose **APScheduler** (`BackgroundScheduler`) running as an in-process thread within the FastAPI application.
|
||||||
|
|
||||||
|
Implementation details:
|
||||||
|
- The scheduler is started during FastAPI's `lifespan` startup event and shut down on application exit.
|
||||||
|
- Each job function creates its own `SessionLocal()` instance, independent from request-scoped sessions.
|
||||||
|
- All jobs use try/except/finally to ensure sessions are closed even on failure.
|
||||||
|
- Jobs are registered with `replace_existing=True` to handle server restarts cleanly.
|
||||||
|
- The scheduler is a module-level singleton in `jobs/mitre_sync_job.py`.
|
||||||
|
|
||||||
|
### Consequences
|
||||||
|
|
||||||
|
**Positive:**
|
||||||
|
- Zero additional infrastructure — no message broker, no worker containers, no job database.
|
||||||
|
- Jobs share the same Python process, so they can import services directly (`sync_mitre`, `scan_intel`, `create_snapshot`, etc.) without serialization or RPC.
|
||||||
|
- Simple debugging — job logs appear in the same stdout as API logs.
|
||||||
|
- Session isolation per job prevents interference with request-scoped transactions.
|
||||||
|
- `replace_existing=True` prevents duplicate job registrations on hot reload.
|
||||||
|
|
||||||
|
**Negative:**
|
||||||
|
- **No persistence:** If the server crashes mid-job, the job state is lost. There is no retry mechanism — the job simply runs again at the next scheduled interval.
|
||||||
|
- **No distributed execution:** Cannot run jobs on a separate worker node. If the API is under heavy load, jobs compete for the same CPU and memory.
|
||||||
|
- **No dead letter queue:** Failed jobs are logged but not queued for retry. A failed MITRE sync silently waits 24 hours before trying again.
|
||||||
|
- **No job history:** There is no record of when jobs last ran, how long they took, or whether they succeeded — only log lines.
|
||||||
|
- **Single-instance constraint:** If multiple backend instances are running (horizontal scaling), each instance runs its own scheduler, causing duplicate job execution (double MITRE sync, double snapshots, etc.).
|
||||||
|
- **No manual trigger via scheduler:** Admin-triggered syncs go through the API endpoints (`/api/v1/system/*`), bypassing the scheduler entirely. There are effectively two paths to the same operations.
|
||||||
|
|
||||||
|
**Risks:**
|
||||||
|
- The single-instance constraint is the most significant risk. If Aegis scales horizontally, APScheduler must be replaced or augmented with a distributed lock (e.g., PostgreSQL advisory locks or Redis-based locking).
|
||||||
|
|
||||||
|
### Alternatives Considered
|
||||||
|
|
||||||
|
| Alternative | Reason Rejected |
|
||||||
|
|------------|-----------------|
|
||||||
|
| **Celery + Redis/RabbitMQ** | Requires an additional broker container (Redis or RabbitMQ), a separate worker process, and Celery configuration. Significant operational overhead for 5 periodic tasks that each run for < 2 minutes. Would be justified if job volume grows or horizontal scaling is needed. |
|
||||||
|
| **Dramatiq + Redis** | Similar to Celery but lighter. Still requires a Redis container and a separate worker process. Same operational overhead concern. |
|
||||||
|
| **Cron jobs (host-level)** | Would require the host to have cron configured and scripts that call API endpoints or run Python commands inside the container. Breaks the "single Docker Compose" deployment model. Not portable. |
|
||||||
|
| **PostgreSQL `pg_cron`** | Runs inside the database, limited to SQL operations. Cannot execute Python logic (downloading ZIPs, parsing YAML, upserting with business rules). Would require stored procedures or external triggers. |
|
||||||
|
| **Kubernetes CronJobs** | Requires Kubernetes. Not applicable to the Docker Compose deployment model (see ADR-004). |
|
||||||
|
| **APScheduler with JobStore (PostgreSQL)** | APScheduler supports persistent job stores that would solve the single-instance problem via database locking. This is a viable evolution path — same library, minimal code change, adds distributed-safe execution. **Recommended as the first upgrade when horizontal scaling is needed.** |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ADR Evolution Path
|
||||||
|
|
||||||
|
The following table summarizes when each decision should be revisited:
|
||||||
|
|
||||||
|
| ADR | Revisit When | Likely Evolution |
|
||||||
|
|-----|-------------|-----------------|
|
||||||
|
| ADR-001 (FastAPI) | Stable — no change needed | Add structured logging, OpenTelemetry tracing |
|
||||||
|
| ADR-002 (PostgreSQL + JSONB) | JSONB query performance degrades | Add GIN indexes on JSONB columns, evaluate moving high-query fields to dedicated columns |
|
||||||
|
| ADR-003 (MinIO) | Cloud deployment required | Swap boto3 endpoint to AWS S3 / GCS (zero code change) |
|
||||||
|
| ADR-004 (Docker Compose) | Multi-server deployment needed | Migrate to Kubernetes with Helm charts, or add Ansible playbooks |
|
||||||
|
| ADR-005 (Modular Monolith) | Team grows > 5 developers, or domains need independent scaling | Enforce boundaries first (Clean Architecture refactor), then extract high-traffic domains as services if needed |
|
||||||
|
| ADR-006 (APScheduler) | Horizontal scaling required, or jobs need retry/history | Add APScheduler PostgreSQL JobStore first; migrate to Celery if job complexity grows significantly |
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
# Aegis — C4 Container Diagram (Level 2)
|
||||||
|
|
||||||
|
> **Author:** Architecture review
|
||||||
|
> **Date:** February 11, 2026
|
||||||
|
> **Notation:** C4 Model — Level 2 (Container Diagram)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Diagram
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
C4Container
|
||||||
|
title Aegis — Container Diagram (C4 Level 2)
|
||||||
|
|
||||||
|
%% ─── Actors ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
Person(security_team, "Security Team", "Red/Blue Technicians, Red/Blue Leads, Viewers — interact with the platform via browser")
|
||||||
|
Person(admin, "Administrator", "Manages users, triggers data syncs, configures scoring weights, reviews audit logs")
|
||||||
|
|
||||||
|
%% ─── System Boundary: Aegis Platform ────────────────────────────
|
||||||
|
|
||||||
|
Container_Boundary(aegis, "Aegis Platform") {
|
||||||
|
|
||||||
|
Container(frontend, "Frontend SPA", "React 19, TypeScript, Vite, Tailwind CSS, Nginx", "Single-page application served by Nginx in production. Provides dashboards, ATT&CK heatmaps, test workflows, campaign management, compliance views, and report exports. Proxies /api/ requests to backend.")
|
||||||
|
|
||||||
|
Container(backend, "Backend API", "Python 3.11, FastAPI, Uvicorn, SQLAlchemy", "REST API serving 21 router modules under /api/v1. Handles authentication (JWT + HttpOnly cookies), RBAC authorization (6 roles), Red/Blue test workflows, scoring engine, heatmap generation, report building, and CRUD for all domain entities. Rate-limited with SlowAPI.")
|
||||||
|
|
||||||
|
Container(scheduler, "Background Scheduler", "APScheduler (in-process)", "Runs inside the backend process as a BackgroundScheduler thread. Executes 5 periodic jobs: MITRE ATT&CK sync (24h), intel scan (7d), notification cleanup (24h), weekly coverage snapshot (Sundays 00:00), recurring campaigns check (24h). Each job manages its own DB session.")
|
||||||
|
|
||||||
|
ContainerDb(postgres, "PostgreSQL 15", "PostgreSQL, Alpine", "Primary relational data store. Holds techniques, tests, users, campaigns, threat actors, detection rules, compliance mappings, audit logs, notifications, coverage snapshots, and scoring configuration. Schema managed by Alembic migrations (18 versions).")
|
||||||
|
|
||||||
|
ContainerDb(minio, "MinIO", "MinIO (S3-compatible), Alpine", "Object storage for Red/Blue team evidence files (screenshots, logs, PCAPs, documents). Stores files in the 'evidence' bucket. Backend generates presigned URLs for secure direct downloads.")
|
||||||
|
}
|
||||||
|
|
||||||
|
%% ─── External Systems ───────────────────────────────────────────
|
||||||
|
|
||||||
|
System_Ext(mitre_taxii, "MITRE ATT&CK TAXII Server", "STIX/TAXII 2.0 feed providing Enterprise ATT&CK techniques and tactics catalog")
|
||||||
|
System_Ext(mitre_cti, "MITRE CTI GitHub", "STIX 2.0 bundles: ATT&CK techniques (fallback), threat actors (intrusion-sets), actor-technique relationships")
|
||||||
|
System_Ext(d3fend, "MITRE D3FEND API", "REST API providing defensive techniques and ATT&CK-to-D3FEND countermeasure mappings")
|
||||||
|
System_Ext(atomic, "Atomic Red Team", "GitHub repository with 1500+ atomic test YAML files mapped to ATT&CK techniques")
|
||||||
|
System_Ext(sigma, "SigmaHQ", "GitHub repository with Sigma detection rules in YAML, tagged with ATT&CK technique IDs")
|
||||||
|
System_Ext(elastic, "Elastic Detection Rules", "GitHub repository with Elastic SIEM rules in TOML format with MITRE threat mappings")
|
||||||
|
System_Ext(caldera, "MITRE CALDERA", "GitHub repository with CALDERA abilities in YAML, organized by tactic")
|
||||||
|
System_Ext(lolbas, "LOLBAS / GTFOBins", "GitHub repositories for Living Off The Land binaries (Windows) and GTFOBins (Linux)")
|
||||||
|
|
||||||
|
%% ─── Planned Systems ────────────────────────────────────────────
|
||||||
|
|
||||||
|
System_Ext(github_actions, "GitHub Actions (Planned)", "Future CI/CD: lint, type check, pytest, Docker build, deploy")
|
||||||
|
System_Ext(artifactory, "Artifactory (Planned)", "Future artifact repository for Docker images and versioned build artifacts")
|
||||||
|
|
||||||
|
%% ─── Relationships: Users → Containers ──────────────────────────
|
||||||
|
|
||||||
|
Rel(security_team, frontend, "Uses", "HTTPS / Browser")
|
||||||
|
Rel(admin, frontend, "Uses", "HTTPS / Browser")
|
||||||
|
|
||||||
|
%% ─── Relationships: Frontend → Backend ──────────────────────────
|
||||||
|
|
||||||
|
Rel(frontend, backend, "Proxies API requests to", "HTTP (Nginx reverse proxy to backend:8000/api/)")
|
||||||
|
|
||||||
|
%% ─── Relationships: Backend → Data Stores ───────────────────────
|
||||||
|
|
||||||
|
Rel(backend, postgres, "Reads/writes domain data", "TCP/5432, SQLAlchemy ORM")
|
||||||
|
Rel(backend, minio, "Uploads/downloads evidence files", "HTTP/9000, boto3 S3 API")
|
||||||
|
|
||||||
|
%% ─── Relationships: Scheduler → Data Stores ─────────────────────
|
||||||
|
|
||||||
|
Rel(scheduler, postgres, "Reads/writes via own sessions", "TCP/5432, SQLAlchemy")
|
||||||
|
|
||||||
|
%% ─── Relationships: Backend/Scheduler → External Sources ────────
|
||||||
|
|
||||||
|
Rel(scheduler, mitre_taxii, "Syncs techniques every 24h", "TAXII 2.0 / HTTPS")
|
||||||
|
Rel(backend, mitre_cti, "Imports threat actors + fallback sync", "HTTPS, ZIP download")
|
||||||
|
Rel(backend, d3fend, "Imports D3FEND techniques and mappings", "REST API / HTTPS")
|
||||||
|
Rel(backend, atomic, "Imports atomic test templates", "HTTPS, ZIP ~40MB")
|
||||||
|
Rel(backend, sigma, "Imports Sigma detection rules", "HTTPS, ZIP download")
|
||||||
|
Rel(backend, elastic, "Imports Elastic detection rules", "HTTPS, ZIP download")
|
||||||
|
Rel(backend, caldera, "Imports CALDERA abilities", "HTTPS, ZIP download")
|
||||||
|
Rel(backend, lolbas, "Imports LOLBAS and GTFOBins", "HTTPS, ZIP download")
|
||||||
|
|
||||||
|
%% ─── Relationships: Planned ─────────────────────────────────────
|
||||||
|
|
||||||
|
Rel(github_actions, backend, "Builds, tests, deploys (planned)", "HTTPS")
|
||||||
|
Rel(github_actions, frontend, "Builds, deploys (planned)", "HTTPS")
|
||||||
|
Rel(github_actions, artifactory, "Pushes Docker images (planned)", "HTTPS")
|
||||||
|
|
||||||
|
UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="1")
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Container Responsibilities
|
||||||
|
|
||||||
|
### Frontend SPA
|
||||||
|
|
||||||
|
| Attribute | Detail |
|
||||||
|
|-----------|--------|
|
||||||
|
| **Technology** | React 19, TypeScript 5.9, Vite 7.3, Tailwind CSS 4, React Router 7 |
|
||||||
|
| **Runtime (Dev)** | Node 20 + Vite dev server on port 5173 |
|
||||||
|
| **Runtime (Prod)** | Nginx Alpine serving static build artifacts on port 80 |
|
||||||
|
| **State Management** | AuthContext (React Context) + TanStack React Query for server state |
|
||||||
|
| **API Communication** | Axios client with `withCredentials: true` (HttpOnly JWT cookie) |
|
||||||
|
| **Security** | CSP headers, X-Frame-Options: DENY, X-Content-Type-Options: nosniff, gzip compression |
|
||||||
|
| **Responsibilities** | Render UI (21 pages, 30+ components), route protection by role, lazy loading, API proxy via Nginx `/api/` → `backend:8000/api/` |
|
||||||
|
|
||||||
|
### Backend API
|
||||||
|
|
||||||
|
| Attribute | Detail |
|
||||||
|
|-----------|--------|
|
||||||
|
| **Technology** | Python 3.11, FastAPI, Uvicorn, SQLAlchemy, Alembic, Pydantic |
|
||||||
|
| **Runtime** | Uvicorn ASGI server on port 8000 (behind Nginx proxy) |
|
||||||
|
| **API Surface** | 21 routers, 80+ endpoints under `/api/v1` |
|
||||||
|
| **Auth** | JWT (HS256) in HttpOnly cookie, bcrypt passwords, in-memory token blacklist |
|
||||||
|
| **RBAC** | 6 roles: admin, red_tech, blue_tech, red_lead, blue_lead, viewer |
|
||||||
|
| **Rate Limiting** | SlowAPI (5 req/min on login) |
|
||||||
|
| **Error Handling** | Global handlers for ValidationError → 400, SQLAlchemyError → 500, Exception → 500 |
|
||||||
|
| **Responsibilities** | All business logic, test workflow state machine, scoring engine, heatmap generation, report building, CRUD, data import orchestration, audit logging |
|
||||||
|
|
||||||
|
### Background Scheduler
|
||||||
|
|
||||||
|
| Attribute | Detail |
|
||||||
|
|-----------|--------|
|
||||||
|
| **Technology** | APScheduler `BackgroundScheduler` (runs in-process within backend) |
|
||||||
|
| **Lifecycle** | Starts on FastAPI lifespan startup, shuts down on app shutdown |
|
||||||
|
| **Session Model** | Each job creates and closes its own `SessionLocal()` instance |
|
||||||
|
| **Registered Jobs** | See table below |
|
||||||
|
|
||||||
|
| Job | Trigger | Frequency | Action |
|
||||||
|
|-----|---------|-----------|--------|
|
||||||
|
| `mitre_sync` | Interval | Every 24 hours | Syncs ATT&CK techniques via TAXII 2.0 (fallback: GitHub ZIP) |
|
||||||
|
| `intel_scan` | Interval | Every 7 days | Scans threat intelligence sources for new indicators |
|
||||||
|
| `notification_cleanup` | Interval | Every 24 hours | Deletes read notifications older than 90 days |
|
||||||
|
| `weekly_snapshot` | Cron | Sundays at 00:00 | Creates coverage snapshot, cleans up old ones (keeps last 52) |
|
||||||
|
| `recurring_campaigns` | Interval | Every 24 hours | Checks and spawns due recurring test campaigns |
|
||||||
|
|
||||||
|
### PostgreSQL 15
|
||||||
|
|
||||||
|
| Attribute | Detail |
|
||||||
|
|-----------|--------|
|
||||||
|
| **Image** | `postgres:15-alpine` |
|
||||||
|
| **Database** | `attackdb` |
|
||||||
|
| **Schema Management** | Alembic with 18 migration versions |
|
||||||
|
| **Connection** | `postgresql://user:pass@postgres:5432/attackdb` via SQLAlchemy |
|
||||||
|
| **Volumes** | Named volume `aegis_postgres_data_prod` for persistence |
|
||||||
|
| **Health Check** | `pg_isready` every 5 seconds |
|
||||||
|
| **Data Stored** | Techniques (ATT&CK), tests (Red/Blue workflow), users, campaigns, threat actors, detection rules (Sigma/Elastic), D3FEND mappings, compliance frameworks, audit logs, notifications, coverage snapshots, scoring config, intel items, data sources, evidence metadata |
|
||||||
|
|
||||||
|
### MinIO (S3-compatible)
|
||||||
|
|
||||||
|
| Attribute | Detail |
|
||||||
|
|-----------|--------|
|
||||||
|
| **Image** | `minio/minio:latest` |
|
||||||
|
| **Ports** | 9000 (S3 API), 9001 (admin console) |
|
||||||
|
| **Bucket** | `evidence` (auto-created on backend startup) |
|
||||||
|
| **Access** | Via boto3 S3 API from backend |
|
||||||
|
| **Volumes** | Named volume `aegis_minio_data_prod` for persistence |
|
||||||
|
| **Responsibilities** | Store Red/Blue team evidence files (screenshots, logs, PCAPs). Backend generates time-limited presigned URLs for secure browser downloads. |
|
||||||
|
|
||||||
|
### GitHub Actions (Planned)
|
||||||
|
|
||||||
|
| Attribute | Detail |
|
||||||
|
|-----------|--------|
|
||||||
|
| **Status** | Not yet implemented — no `.github/workflows/` directory exists |
|
||||||
|
| **Planned Scope** | Lint (ruff/flake8), type check (mypy), unit/integration tests (pytest), Docker image build, deploy to staging/production |
|
||||||
|
| **Integration** | Would trigger on push/PR to main branch |
|
||||||
|
| **Artifact Flow** | Build Docker images → push to Artifactory → deploy via compose |
|
||||||
|
|
||||||
|
### Artifactory (Planned)
|
||||||
|
|
||||||
|
| Attribute | Detail |
|
||||||
|
|-----------|--------|
|
||||||
|
| **Status** | Not yet implemented — no integration code exists |
|
||||||
|
| **Planned Scope** | Docker image registry for versioned backend/frontend images |
|
||||||
|
| **Integration** | Receive images from GitHub Actions CI pipeline, serve to production deploy |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Network Topology
|
||||||
|
|
||||||
|
```
|
||||||
|
Internet
|
||||||
|
│
|
||||||
|
│ HTTPS (:80 / :443)
|
||||||
|
▼
|
||||||
|
┌─────────────────┐
|
||||||
|
│ Frontend │
|
||||||
|
│ Nginx + React │
|
||||||
|
│ :80 │
|
||||||
|
└────────┬────────┘
|
||||||
|
│
|
||||||
|
┌────────────┼────────────────────────────────┐
|
||||||
|
│ │ aegis-network (bridge) │
|
||||||
|
│ │ /api/ proxy │
|
||||||
|
│ ▼ │
|
||||||
|
│ ┌─────────────────┐ │
|
||||||
|
│ │ Backend API │◄── Scheduler │
|
||||||
|
│ │ FastAPI/Uvicorn │ (in-process thread) │
|
||||||
|
│ │ :8000 │ │
|
||||||
|
│ └───┬─────────┬──┘ │
|
||||||
|
│ │ │ │
|
||||||
|
│ │ │ │
|
||||||
|
│ ▼ ▼ │
|
||||||
|
│ ┌─────────┐ ┌───────┐ │
|
||||||
|
│ │PostgreSQL│ │ MinIO │ │
|
||||||
|
│ │ :5432 │ │ :9000 │ │
|
||||||
|
│ └─────────┘ └───────┘ │
|
||||||
|
│ │
|
||||||
|
└──────────────────────────────────────────────┘
|
||||||
|
│
|
||||||
|
│ HTTPS (outbound only)
|
||||||
|
▼
|
||||||
|
External Data Sources
|
||||||
|
(MITRE, SigmaHQ, Elastic, etc.)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Data Flow Summary
|
||||||
|
|
||||||
|
| Flow | Path | Protocol | Notes |
|
||||||
|
|------|------|----------|-------|
|
||||||
|
| User → UI | Browser → Nginx | HTTPS | Static SPA assets, gzip compressed, 1-year cache for static files |
|
||||||
|
| UI → API | Nginx → Uvicorn | HTTP (internal) | Reverse proxy with 300s timeout for long sync operations |
|
||||||
|
| API → DB | Uvicorn → PostgreSQL | TCP/5432 | SQLAlchemy ORM, request-scoped sessions via `get_db()` |
|
||||||
|
| API → Storage | Uvicorn → MinIO | HTTP/9000 | boto3 S3 API, presigned URLs for downloads |
|
||||||
|
| Scheduler → DB | APScheduler thread → PostgreSQL | TCP/5432 | Independent sessions per job, created/closed in try/finally |
|
||||||
|
| Scheduler → External | APScheduler thread → MITRE TAXII | HTTPS | Scheduled sync every 24h, fallback to GitHub ZIP |
|
||||||
|
| Admin → External | API on-demand → GitHub repos | HTTPS | ZIP download triggered by admin via `/api/v1/system/*` endpoints |
|
||||||
|
| Health Check | Docker → Backend `/health` | HTTP (internal) | Restricted to private IPs via Nginx `allow/deny` directives |
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
# Aegis — C4 Context Diagram (Level 1)
|
||||||
|
|
||||||
|
> **Author:** Architecture review
|
||||||
|
> **Date:** February 11, 2026
|
||||||
|
> **Notation:** C4 Model — Level 1 (System Context)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Diagram
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
C4Context
|
||||||
|
title Aegis — System Context Diagram (C4 Level 1)
|
||||||
|
|
||||||
|
%% ─── Actors (People) ────────────────────────────────────────────
|
||||||
|
|
||||||
|
Person(red_tech, "Red Team Technician", "Executes offensive tests, submits evidence, creates tests from templates")
|
||||||
|
Person(blue_tech, "Blue Team Technician", "Evaluates detection results, submits blue evidence, documents findings")
|
||||||
|
Person(red_lead, "Red Team Lead", "Validates red team results, manages campaigns, reviews test outcomes")
|
||||||
|
Person(blue_lead, "Blue Team Lead", "Validates blue team results, manages remediation, reviews detection gaps")
|
||||||
|
Person(admin, "Administrator", "Manages users, triggers data syncs, configures scoring, oversees platform")
|
||||||
|
Person(viewer, "Viewer", "Read-only access to dashboards, reports, heatmaps, and compliance status")
|
||||||
|
|
||||||
|
%% ─── Core System ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
System(aegis, "Aegis Platform", "MITRE ATT&CK coverage management platform. Orchestrates Red/Blue team validation workflows, tracks technique coverage, generates heatmaps, compliance reports, and organizational scoring.")
|
||||||
|
|
||||||
|
%% ─── Internal Infrastructure (Owned / Deployed) ─────────────────
|
||||||
|
|
||||||
|
SystemDb(postgres, "PostgreSQL 15", "Primary data store. Stores techniques, tests, users, campaigns, threat actors, compliance mappings, audit logs, scoring config, and snapshots.")
|
||||||
|
SystemDb(minio, "MinIO (S3-compatible)", "Object storage for Red/Blue team evidence files (screenshots, logs, PCAPs). Serves presigned download URLs.")
|
||||||
|
|
||||||
|
%% ─── External Data Sources (Consumed) ───────────────────────────
|
||||||
|
|
||||||
|
System_Ext(mitre_taxii, "MITRE ATT&CK TAXII Server", "STIX/TAXII 2.0 feed providing Enterprise ATT&CK techniques and tactics. Primary source for technique catalog sync.")
|
||||||
|
System_Ext(mitre_cti, "MITRE CTI GitHub Repository", "STIX 2.0 bundles for ATT&CK techniques (fallback), intrusion-sets (threat actors), and actor-technique relationships.")
|
||||||
|
System_Ext(d3fend, "MITRE D3FEND API", "Public REST API providing defensive techniques and ATT&CK-to-D3FEND mappings for countermeasure coverage.")
|
||||||
|
System_Ext(atomic, "Atomic Red Team (GitHub)", "Repository of atomic tests mapped to ATT&CK techniques. Downloaded as ZIP, parsed from YAML atomics.")
|
||||||
|
System_Ext(sigma, "SigmaHQ (GitHub)", "Repository of Sigma detection rules in YAML format. Parsed for ATT&CK tags and imported as detection rules.")
|
||||||
|
System_Ext(elastic, "Elastic Detection Rules (GitHub)", "Repository of Elastic SIEM rules in TOML format. Parsed for MITRE threat mappings and imported as detection rules.")
|
||||||
|
System_Ext(caldera, "MITRE CALDERA (GitHub)", "Repository of CALDERA abilities. YAML files parsed from data/abilities/ and imported as test templates.")
|
||||||
|
System_Ext(lolbas, "LOLBAS Project (GitHub)", "Living Off The Land Binaries and Scripts. YAML-based catalog imported as test templates mapped to ATT&CK techniques.")
|
||||||
|
System_Ext(gtfobins, "GTFOBins (GitHub)", "Unix binaries exploitation reference. Markdown with YAML front-matter parsed and mapped to ATT&CK techniques.")
|
||||||
|
|
||||||
|
%% ─── Planned Systems (Not Yet Integrated) ──────────────────────
|
||||||
|
|
||||||
|
System_Ext(github_ent, "GitHub Enterprise (Planned)", "Future CI/CD pipeline integration for automated linting, type checking, test execution, and deployment workflows.")
|
||||||
|
System_Ext(artifactory, "Artifactory (Planned)", "Future artifact repository for storing Docker images, build artifacts, and versioned releases.")
|
||||||
|
|
||||||
|
%% ─── Relationships: Users → Aegis ───────────────────────────────
|
||||||
|
|
||||||
|
Rel(red_tech, aegis, "Creates and executes tests, uploads red evidence, uses test catalog", "HTTPS")
|
||||||
|
Rel(blue_tech, aegis, "Evaluates detections, uploads blue evidence, reviews detection rules", "HTTPS")
|
||||||
|
Rel(red_lead, aegis, "Validates red results, manages campaigns, reviews threat actor coverage", "HTTPS")
|
||||||
|
Rel(blue_lead, aegis, "Validates blue results, tracks remediation, reviews compliance", "HTTPS")
|
||||||
|
Rel(admin, aegis, "Manages users, triggers syncs, configures scoring weights, views audit logs", "HTTPS")
|
||||||
|
Rel(viewer, aegis, "Views dashboards, heatmaps, reports, and compliance status", "HTTPS")
|
||||||
|
|
||||||
|
%% ─── Relationships: Aegis → Infrastructure ──────────────────────
|
||||||
|
|
||||||
|
Rel(aegis, postgres, "Reads/writes all domain data", "TCP/5432, SQLAlchemy")
|
||||||
|
Rel(aegis, minio, "Uploads/downloads evidence files, generates presigned URLs", "HTTP/9000, boto3 S3 API")
|
||||||
|
|
||||||
|
%% ─── Relationships: Aegis → External Sources ────────────────────
|
||||||
|
|
||||||
|
Rel(aegis, mitre_taxii, "Syncs ATT&CK techniques every 24h", "TAXII 2.0 / HTTPS")
|
||||||
|
Rel(aegis, mitre_cti, "Fallback technique sync + threat actor import", "HTTPS, ZIP download")
|
||||||
|
Rel(aegis, d3fend, "Imports defensive techniques and ATT&CK mappings", "REST API / HTTPS")
|
||||||
|
Rel(aegis, atomic, "Imports Atomic Red Team test templates", "HTTPS, ZIP download")
|
||||||
|
Rel(aegis, sigma, "Imports Sigma detection rules with ATT&CK tags", "HTTPS, ZIP download")
|
||||||
|
Rel(aegis, elastic, "Imports Elastic SIEM detection rules", "HTTPS, ZIP download")
|
||||||
|
Rel(aegis, caldera, "Imports CALDERA abilities as test templates", "HTTPS, ZIP download")
|
||||||
|
Rel(aegis, lolbas, "Imports LOLBAS binaries as test templates", "HTTPS, ZIP download")
|
||||||
|
Rel(aegis, gtfobins, "Imports GTFOBins as test templates", "HTTPS, ZIP download")
|
||||||
|
|
||||||
|
%% ─── Relationships: Aegis → Planned ─────────────────────────────
|
||||||
|
|
||||||
|
Rel(aegis, github_ent, "CI/CD pipelines (planned)", "HTTPS")
|
||||||
|
Rel(aegis, artifactory, "Artifact storage (planned)", "HTTPS")
|
||||||
|
|
||||||
|
UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="1")
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Diagram Notes
|
||||||
|
|
||||||
|
### Actor Roles
|
||||||
|
|
||||||
|
| Role | Access Level | Primary Actions |
|
||||||
|
|------|-------------|-----------------|
|
||||||
|
| **Red Team Technician** | Standard | Create tests, execute attacks, upload red evidence, use test catalog |
|
||||||
|
| **Blue Team Technician** | Standard | Evaluate detections, upload blue evidence, review detection rules |
|
||||||
|
| **Red Team Lead** | Elevated | Validate red results, manage campaigns, review threat actor coverage |
|
||||||
|
| **Blue Team Lead** | Elevated | Validate blue results, track remediation, review compliance |
|
||||||
|
| **Administrator** | Full | User management, trigger data syncs, scoring config, audit logs |
|
||||||
|
| **Viewer** | Read-only | View dashboards, heatmaps, reports, compliance status |
|
||||||
|
|
||||||
|
### External Data Source Details
|
||||||
|
|
||||||
|
| Source | Protocol | Frequency | Data Imported |
|
||||||
|
|--------|----------|-----------|---------------|
|
||||||
|
| MITRE ATT&CK TAXII | STIX/TAXII 2.0 | Every 24 hours (scheduled) | Enterprise techniques and tactics |
|
||||||
|
| MITRE CTI GitHub | HTTPS (ZIP) | Fallback + on-demand | Techniques, threat actors (intrusion-sets), actor-technique relationships |
|
||||||
|
| MITRE D3FEND | REST API | On-demand (admin trigger) | Defensive techniques, ATT&CK-to-D3FEND mappings |
|
||||||
|
| Atomic Red Team | HTTPS (ZIP ~40MB) | On-demand (admin trigger) | Test templates from `atomics/T*/T*.yaml` |
|
||||||
|
| SigmaHQ | HTTPS (ZIP) | On-demand (admin trigger) | Sigma detection rules with ATT&CK tags |
|
||||||
|
| Elastic Detection Rules | HTTPS (ZIP) | On-demand (admin trigger) | Elastic SIEM rules in TOML with MITRE mappings |
|
||||||
|
| MITRE CALDERA | HTTPS (ZIP) | On-demand (admin trigger) | Abilities from `data/abilities/{tactic}/*.yml` |
|
||||||
|
| LOLBAS Project | HTTPS (ZIP) | On-demand (admin trigger) | Living Off The Land binaries/scripts |
|
||||||
|
| GTFOBins | HTTPS (ZIP) | On-demand (admin trigger) | Unix binary exploitation references |
|
||||||
|
|
||||||
|
### Planned Integrations (Not Yet Implemented)
|
||||||
|
|
||||||
|
| System | Purpose | Status |
|
||||||
|
|--------|---------|--------|
|
||||||
|
| **GitHub Enterprise** | CI/CD pipelines for automated lint, type check, tests, and deployment | Planned — no `.github/workflows` exist yet |
|
||||||
|
| **Artifactory** | Docker image and build artifact repository | Planned — no integration code exists yet |
|
||||||
|
|
||||||
|
### Infrastructure Boundary
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────┐
|
||||||
|
│ Docker Compose Network │
|
||||||
|
│ │
|
||||||
|
│ ┌──────────┐ ┌──────────┐ ┌───────────┐ │
|
||||||
|
│ │ Frontend │ │ Backend │ │ PostgreSQL│ │
|
||||||
|
│ │ (Nginx) │ │ (Uvicorn)│ │ 15 │ │
|
||||||
|
│ │ :80 │ │ :8000 │ │ :5432 │ │
|
||||||
|
│ └──────────┘ └──────────┘ └───────────┘ │
|
||||||
|
│ ┌───────────┐ │
|
||||||
|
│ │ MinIO │ │
|
||||||
|
│ │ :9000/9001│ │
|
||||||
|
│ └───────────┘ │
|
||||||
|
└─────────────────────────────────────────────┘
|
||||||
|
▲ │
|
||||||
|
│ HTTPS │ HTTPS (outbound)
|
||||||
|
│ ▼
|
||||||
|
Users External Sources
|
||||||
|
```
|
||||||
@@ -0,0 +1,291 @@
|
|||||||
|
# Aegis — Technology Justification
|
||||||
|
|
||||||
|
> **Document type:** Architecture Board Submission
|
||||||
|
> **Author:** Platform Architecture Team
|
||||||
|
> **Date:** February 11, 2026
|
||||||
|
> **Classification:** Internal
|
||||||
|
> **Status:** Approved
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This document provides a formal justification for the technology selections made in the Aegis platform. Each technology choice is evaluated against the project's operational requirements, organizational constraints, security posture, and long-term sustainability. This document is intended for review by the Architecture Board and serves as the authoritative reference for technology governance.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Project Context
|
||||||
|
|
||||||
|
Aegis is an internal security operations platform that manages MITRE ATT&CK technique coverage through structured Red Team / Blue Team validation workflows. The platform integrates with 9 external threat intelligence and detection rule sources, enforces role-based access for 6 distinct user roles, and provides coverage analytics including heatmaps, scoring, compliance mapping, and executive reporting.
|
||||||
|
|
||||||
|
### Operational Requirements
|
||||||
|
|
||||||
|
| Requirement | Detail |
|
||||||
|
|------------|--------|
|
||||||
|
| **Deployment model** | On-premise, single-server, air-gap compatible |
|
||||||
|
| **User base** | 10–100 concurrent security analysts and leads |
|
||||||
|
| **Data model** | 18+ relational entities with many-to-many relationships and semi-structured metadata |
|
||||||
|
| **External integrations** | 9 data sources (MITRE TAXII 2.0, GitHub REST, D3FEND REST, Sigma YAML, Elastic TOML, CALDERA YAML, LOLBAS YAML, GTFOBins Markdown, STIX 2.0 JSON) |
|
||||||
|
| **File storage** | Binary evidence files (screenshots, logs, PCAPs) ranging from KB to hundreds of MB |
|
||||||
|
| **Scheduled operations** | 5 periodic background jobs (24h–7d cycles) |
|
||||||
|
| **Security** | RBAC, JWT authentication, audit logging, evidence chain of custody |
|
||||||
|
|
||||||
|
### Organizational Constraints
|
||||||
|
|
||||||
|
| Constraint | Detail |
|
||||||
|
|-----------|--------|
|
||||||
|
| **Team expertise** | Primary competency in Python and TypeScript |
|
||||||
|
| **Target operators** | Security engineers, not DevOps specialists |
|
||||||
|
| **Infrastructure** | Docker available; Kubernetes not guaranteed |
|
||||||
|
| **Network** | Outbound HTTPS required for data source sync; inbound limited to platform UI |
|
||||||
|
| **Budget** | Open-source preference; no commercial license dependencies for core platform |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Backend Framework: FastAPI
|
||||||
|
|
||||||
|
### Selection: FastAPI 0.x (latest stable) with Uvicorn ASGI server
|
||||||
|
|
||||||
|
### Justification
|
||||||
|
|
||||||
|
FastAPI was selected as the backend framework based on four primary evaluation criteria: API development velocity, ecosystem compatibility, runtime performance, and developer experience.
|
||||||
|
|
||||||
|
**API Development Velocity.** Aegis exposes 80+ REST endpoints across 21 domain modules. FastAPI's automatic OpenAPI specification generation from Python type annotations eliminates the need for separate API documentation tooling. Pydantic integration provides request and response validation at the framework level, reducing boilerplate code for schema enforcement. The `Depends()` dependency injection system enables composable middleware chains for authentication, authorization, and database session management without requiring a third-party DI container.
|
||||||
|
|
||||||
|
**Ecosystem Compatibility.** The platform's 9 external data source integrations depend on Python-specific libraries with no mature equivalents in other ecosystems:
|
||||||
|
|
||||||
|
| Library | Purpose | Ecosystem |
|
||||||
|
|---------|---------|-----------|
|
||||||
|
| `taxii2-client` | STIX/TAXII 2.0 protocol | Python only |
|
||||||
|
| `pySigma` | Sigma rule parsing and transformation | Python only |
|
||||||
|
| `PyYAML` | YAML parsing (Atomic Red Team, CALDERA, LOLBAS) | Python preferred |
|
||||||
|
| `toml` | TOML parsing (Elastic detection rules) | Python preferred |
|
||||||
|
| `boto3` | S3-compatible storage API (MinIO) | Python preferred |
|
||||||
|
| `defusedxml` | Secure XML processing | Python preferred |
|
||||||
|
|
||||||
|
Selecting a non-Python backend would require reimplementing or wrapping these libraries, introducing significant engineering risk.
|
||||||
|
|
||||||
|
**Runtime Performance.** FastAPI's ASGI foundation provides asynchronous request handling capability. While the current implementation uses synchronous route handlers (due to SQLAlchemy's synchronous session model), the framework does not impose a performance ceiling for the target user base (10–100 concurrent users). Benchmark data from independent testing consistently places FastAPI among the highest-performing Python web frameworks.
|
||||||
|
|
||||||
|
**Developer Experience.** Interactive Swagger UI (`/docs`) and ReDoc (`/redoc`) are available in non-production environments, accelerating API exploration and frontend integration. These documentation endpoints are automatically disabled in production to reduce attack surface.
|
||||||
|
|
||||||
|
### Alternatives Evaluated
|
||||||
|
|
||||||
|
| Framework | Evaluation Summary | Disposition |
|
||||||
|
|-----------|-------------------|-------------|
|
||||||
|
| Django + Django REST Framework | Mature and feature-rich, but introduces heavier ORM opinions, an unnecessary admin panel, and slower cold-start times. Django's ORM lacks SQLAlchemy's flexibility for JSONB column handling and complex join patterns required by the scoring engine. | Rejected |
|
||||||
|
| Flask + Flask-RESTful | Lightweight but lacks built-in request validation, automatic OpenAPI generation, and dependency injection. Would require additional libraries (marshmallow, flask-apispec) to achieve parity with FastAPI's built-in capabilities. | Rejected |
|
||||||
|
| Go (Gin / Echo) | Superior raw throughput, but the team's primary expertise is Python. The 9 data source integrations depend on Python libraries with no Go equivalents. The development velocity loss would outweigh performance gains for a 10–100 user internal platform. | Rejected |
|
||||||
|
| NestJS (Node.js / TypeScript) | Would unify frontend and backend language, but splits runtime expertise. No mature Node.js equivalents for STIX/TAXII and Sigma rule parsing. The Python data science and security tooling ecosystem is substantially deeper. | Rejected |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Primary Database: PostgreSQL 15
|
||||||
|
|
||||||
|
### Selection: PostgreSQL 15 (Alpine) with SQLAlchemy ORM and Alembic migrations
|
||||||
|
|
||||||
|
### Justification
|
||||||
|
|
||||||
|
PostgreSQL was selected as the primary relational data store based on three requirements: relational integrity for a complex domain model, semi-structured data support for external source metadata, and operational maturity for on-premise deployment.
|
||||||
|
|
||||||
|
**Relational Integrity.** The Aegis data model comprises 18+ entities with deep relational dependencies: techniques relate to tests, tests belong to campaigns, campaigns map to threat actors, threat actors link to techniques, compliance controls map to techniques, and detection rules associate with both techniques and test templates. This graph of many-to-many relationships demands foreign key enforcement, transactional consistency, and efficient join operations — core strengths of a relational database.
|
||||||
|
|
||||||
|
**Semi-Structured Data (JSONB).** Several entities carry metadata with variable structure imported from external sources (STIX 2.0, Sigma YAML, Elastic TOML). PostgreSQL's native JSONB column type stores this data in a binary-indexed format that supports containment queries and GIN indexing, eliminating the need for a separate document store. Current JSONB usage is contained to 12 columns across 6 tables:
|
||||||
|
|
||||||
|
| Entity | JSONB Fields | Content |
|
||||||
|
|--------|-------------|---------|
|
||||||
|
| Technique | `platforms` | OS platform array from ATT&CK |
|
||||||
|
| Threat Actor | `aliases`, `target_sectors`, `target_regions`, `references` | STIX 2.0 metadata |
|
||||||
|
| Detection Rule | `platforms`, `log_sources` | Rule targeting metadata |
|
||||||
|
| Data Source | `last_sync_stats`, `config` | Import statistics and source-specific configuration |
|
||||||
|
| Campaign | `tags` | User-defined classification |
|
||||||
|
| Audit Log | `details` | Action-specific metadata (variable per action type) |
|
||||||
|
|
||||||
|
**Operational Maturity.** PostgreSQL 15 provides built-in health checking (`pg_isready`), mature backup tooling (`pg_dump`/`pg_restore`), extensive monitoring capabilities, and a 25+ year track record of production reliability. The Alpine-based Docker image is approximately 80MB, suitable for on-premise deployments with limited resources.
|
||||||
|
|
||||||
|
**Schema Management.** Alembic provides version-controlled database migrations (18 versions to date), enabling reproducible schema evolution and rollback capability.
|
||||||
|
|
||||||
|
### Alternatives Evaluated
|
||||||
|
|
||||||
|
| Database | Evaluation Summary | Disposition |
|
||||||
|
|----------|-------------------|-------------|
|
||||||
|
| MongoDB | The core domain is deeply relational. Modeling technique-test-campaign-actor relationships in MongoDB would require denormalization or manual reference integrity, trading the JSONB advantage for relational integrity loss. | Rejected |
|
||||||
|
| MySQL 8 (JSON) | PostgreSQL's JSONB is binary-indexed and faster for containment queries than MySQL's text-based JSON type. PostgreSQL also provides native UUID support (vs. BINARY(16) in MySQL), which aligns with the platform's UUID-based primary keys. | Rejected |
|
||||||
|
| PostgreSQL + MongoDB (dual) | The operational complexity of maintaining two database systems is unjustified for 12 JSONB columns. A dual-database architecture would also complicate transactional consistency across relational and document data. | Rejected |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Object Storage: MinIO
|
||||||
|
|
||||||
|
### Selection: MinIO (S3-compatible) accessed via boto3 (AWS S3 SDK for Python)
|
||||||
|
|
||||||
|
### Justification
|
||||||
|
|
||||||
|
MinIO was selected as the evidence storage system based on three requirements: S3 API compatibility for portability, on-premise deployment capability, and separation of binary data from the relational database.
|
||||||
|
|
||||||
|
**S3 API Compatibility.** MinIO implements the Amazon S3 API specification, accessed via the industry-standard `boto3` client library. This provides a zero-code-change migration path to AWS S3, Google Cloud Storage (via S3-compatible mode), or any other S3-compatible storage service should the deployment model change from on-premise to cloud. The storage interface (`upload_file`, `get_presigned_url`, `ensure_bucket_exists`) is a thin abstraction layer that is storage-backend agnostic.
|
||||||
|
|
||||||
|
**On-Premise Deployment.** The platform is designed for deployment within organizational security environments where external cloud storage services may not be permitted due to data classification or regulatory requirements. MinIO runs as a single Docker container with persistent volume storage, requiring no external dependencies or network egress for storage operations.
|
||||||
|
|
||||||
|
**Binary Data Separation.** Evidence files (screenshots, packet captures, log extracts) range from kilobytes to hundreds of megabytes. Storing binary data in PostgreSQL (BYTEA columns) would degrade database backup performance, increase storage costs, and complicate streaming downloads. MinIO's presigned URL mechanism offloads download bandwidth from the application server — the browser fetches evidence files directly from MinIO without proxying through the backend.
|
||||||
|
|
||||||
|
**Administrative Visibility.** MinIO Console (port 9001) provides a web-based management interface for administrators to inspect, audit, and manage stored evidence files without requiring command-line access.
|
||||||
|
|
||||||
|
### Alternatives Evaluated
|
||||||
|
|
||||||
|
| Storage | Evaluation Summary | Disposition |
|
||||||
|
|---------|-------------------|-------------|
|
||||||
|
| PostgreSQL BYTEA | Stores binary files in the relational database. Bloats backups, degrades query performance on large tables, and requires the backend to proxy all file downloads. Not designed as a file store. | Rejected |
|
||||||
|
| Local filesystem | Not portable across container restarts without host volume mounts. No presigned URL support (backend must proxy all downloads). No replication, versioning, or management interface. | Rejected |
|
||||||
|
| AWS S3 | Requires a cloud account, internet connectivity for storage operations, and AWS credential management. Incompatible with air-gap or restricted-network deployment requirements. | Rejected (as primary; migration path preserved) |
|
||||||
|
| SeaweedFS | Smaller community and less mature S3-compatible API layer. boto3 compatibility is not fully guaranteed. Insufficient adoption for long-term support confidence. | Rejected |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Frontend: React 19 + TypeScript 5.9
|
||||||
|
|
||||||
|
### Selection: React 19, TypeScript 5.9, Vite 7.3, Tailwind CSS 4, TanStack React Query 5
|
||||||
|
|
||||||
|
### Justification
|
||||||
|
|
||||||
|
The frontend technology selection was driven by four criteria: component ecosystem maturity, type safety for a complex domain, build tooling performance, and developer productivity.
|
||||||
|
|
||||||
|
**Component Ecosystem Maturity.** Aegis presents a complex user interface comprising 21 pages, 30+ components, and specialized visualizations including ATT&CK Navigator-compatible heatmaps, campaign timelines, compliance gauges, and multi-role workflow views. React's component model and its ecosystem (Recharts for data visualization, Lucide for iconography, TanStack Virtual for list virtualization) provide production-ready solutions for each of these requirements.
|
||||||
|
|
||||||
|
**Type Safety.** TypeScript's static type system enforces correctness across the API communication layer (22 domain-specific API modules), shared type definitions (`types/models.ts`), and component props. With `strict: true` in `tsconfig.json`, the compiler catches null reference errors, incorrect property access, and type mismatches at build time rather than runtime. This is particularly valuable for the complex test workflow state machine, where state-dependent UI behavior must correctly reflect 6 possible test states and 6 user roles.
|
||||||
|
|
||||||
|
**Build Tooling.** Vite provides sub-second hot module replacement during development and optimized production builds via Rollup. The multi-stage Docker build produces a minimal Nginx image (~25MB) serving pre-compiled static assets, eliminating the need for a Node.js runtime in production.
|
||||||
|
|
||||||
|
**Server State Management.** TanStack React Query manages all server-side state (caching, refetching, mutation invalidation), eliminating the need for a client-side state management library (Redux, Zustand, MobX) for data fetching concerns. Authentication state is managed via React Context, and UI feedback via a Toast context — both lightweight patterns that avoid unnecessary library dependencies.
|
||||||
|
|
||||||
|
**Styling.** Tailwind CSS 4 provides utility-first styling with zero-runtime CSS generation. The design system is consistent across all 21 pages without maintaining a separate CSS architecture or component library.
|
||||||
|
|
||||||
|
### Alternatives Evaluated
|
||||||
|
|
||||||
|
| Framework | Evaluation Summary | Disposition |
|
||||||
|
|-----------|-------------------|-------------|
|
||||||
|
| Angular | Comprehensive framework with built-in DI, routing, and HTTP client. However, the heavier abstraction layer and steeper learning curve are unnecessary for a team with React experience. Angular's opinionated module system adds boilerplate for a project of this scale. | Rejected |
|
||||||
|
| Vue 3 + TypeScript | Viable alternative with good TypeScript support and a smaller learning curve. However, the React ecosystem offers deeper library coverage for specialized components (ATT&CK heatmaps, data grids, chart libraries). The team's existing React proficiency favors continuity. | Rejected |
|
||||||
|
| Svelte / SvelteKit | Excellent developer experience and smaller bundle sizes, but a significantly smaller ecosystem for complex data visualization. Library availability for heatmaps, virtual scrolling, and charting is limited compared to React. | Rejected |
|
||||||
|
| HTMX + server-rendered templates | Would reduce frontend complexity but cannot support the interactive heatmap, drag-and-drop campaign management, real-time notification updates, and complex multi-step workflow forms required by the platform. | Rejected |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Containerization and Deployment: Docker Compose
|
||||||
|
|
||||||
|
### Selection: Docker with Docker Compose (V2), multi-stage Dockerfiles
|
||||||
|
|
||||||
|
### Justification
|
||||||
|
|
||||||
|
Docker Compose was selected as the deployment orchestration tool based on three requirements: single-command deployment for non-DevOps operators, consistent development-to-production environments, and minimal infrastructure prerequisites.
|
||||||
|
|
||||||
|
**Operator Accessibility.** The platform is deployed by security engineers who may not have Kubernetes expertise or access to container orchestration infrastructure. Docker Compose provides single-command deployment (`docker compose up -d --build`) with an interactive installation script (`install.sh`) that generates secrets, prompts for configuration, and produces a `.env` file. This reduces deployment complexity to a level appropriate for the target operator profile.
|
||||||
|
|
||||||
|
**Environment Consistency.** Two compose files maintain parity between development and production:
|
||||||
|
|
||||||
|
| Aspect | Development | Production |
|
||||||
|
|--------|-------------|------------|
|
||||||
|
| Frontend | Vite dev server, hot reload | Nginx serving static build |
|
||||||
|
| Backend | Source volume-mounted, auto-reload | Multi-stage build, non-root user |
|
||||||
|
| Ports | All services exposed | Only frontend exposed |
|
||||||
|
| Secrets | Auto-generated ephemeral | Required via environment |
|
||||||
|
|
||||||
|
**Infrastructure Footprint.** The entire platform (4 services) runs on a single server with Docker as the only prerequisite. Named volumes provide data persistence across container rebuilds. Health checks and dependency ordering ensure correct startup sequencing.
|
||||||
|
|
||||||
|
**Security Hardening.** The backend Dockerfile follows container security best practices: non-root user (`appuser`, UID 1001), minimal base image (`python:3.11-slim`), and no unnecessary system packages beyond build dependencies.
|
||||||
|
|
||||||
|
### Alternatives Evaluated
|
||||||
|
|
||||||
|
| Platform | Evaluation Summary | Disposition |
|
||||||
|
|----------|-------------------|-------------|
|
||||||
|
| Kubernetes | Provides horizontal scaling, rolling deployments, and self-healing. However, it requires a cluster, kubectl expertise, Helm charts, ingress controllers, and persistent volume claims. This operational overhead is disproportionate for a 4-service application targeting single-server deployment. | Rejected (viable future evolution for multi-server) |
|
||||||
|
| Docker Swarm | Adds orchestration with lower complexity than Kubernetes but provides minimal benefit over Compose for < 5 services. Docker Swarm's development trajectory has stalled relative to Compose V2. | Rejected |
|
||||||
|
| Bare metal / systemd | Loses containerization benefits: isolation, reproducibility, and dependency management. Would require manual installation of Python, Node.js, PostgreSQL, and MinIO on each target system, increasing deployment failure risk. | Rejected |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. CI/CD and Artifact Management: GitHub Enterprise + Artifactory
|
||||||
|
|
||||||
|
### Selection: GitHub Enterprise for source control and CI/CD; JFrog Artifactory for artifact storage
|
||||||
|
|
||||||
|
### Status: Planned — not yet implemented
|
||||||
|
|
||||||
|
### Justification
|
||||||
|
|
||||||
|
GitHub Enterprise and Artifactory are designated as the CI/CD and artifact management platforms for Aegis based on organizational standardization, security requirements, and the artifact lifecycle.
|
||||||
|
|
||||||
|
**Organizational Standardization.** GitHub Enterprise is the organization's standard source control and CI/CD platform. Adopting it for Aegis ensures consistency with existing developer workflows, access control policies, and audit mechanisms. Security teams reviewing the Aegis codebase will use familiar tooling and processes.
|
||||||
|
|
||||||
|
**CI/CD Pipeline (Planned).** The following GitHub Actions workflow stages are planned:
|
||||||
|
|
||||||
|
| Stage | Tools | Trigger |
|
||||||
|
|-------|-------|---------|
|
||||||
|
| **Lint** | ruff (Python), ESLint (TypeScript) | Push to any branch |
|
||||||
|
| **Type check** | mypy (Python), tsc --noEmit (TypeScript) | Push to any branch |
|
||||||
|
| **Unit tests** | pytest (backend), vitest (frontend) | Push to any branch |
|
||||||
|
| **Integration tests** | pytest with PostgreSQL service container | Pull request to main |
|
||||||
|
| **Docker build** | Multi-stage Dockerfile verification | Pull request to main |
|
||||||
|
| **Image publish** | Docker build + push to Artifactory | Merge to main |
|
||||||
|
| **Deploy** | Docker Compose pull + restart | Manual trigger or tag |
|
||||||
|
|
||||||
|
**Artifact Repository.** Artifactory serves as the Docker image registry for versioned backend and frontend images. This provides:
|
||||||
|
|
||||||
|
- **Versioned releases:** Each merge to main produces a tagged image (`aegis-backend:1.2.3`, `aegis-frontend:1.2.3`).
|
||||||
|
- **Rollback capability:** Previous image versions remain available for rapid rollback.
|
||||||
|
- **Vulnerability scanning:** Artifactory's Xray integration enables automated CVE scanning of Docker image layers.
|
||||||
|
- **Access control:** Image pull/push permissions align with organizational RBAC policies.
|
||||||
|
|
||||||
|
**Air-Gap Deployment Support.** For restricted-network deployments, Docker images can be exported from Artifactory as tarballs (`docker save`), transferred via secure media, and loaded into the target environment (`docker load`) without requiring network connectivity to the registry.
|
||||||
|
|
||||||
|
### Implementation Timeline
|
||||||
|
|
||||||
|
| Phase | Scope | Estimated Effort |
|
||||||
|
|-------|-------|-----------------|
|
||||||
|
| Phase 1 | Basic CI: lint + type check + unit tests | 1–2 days |
|
||||||
|
| Phase 2 | Integration tests with PostgreSQL service container | 2–3 days |
|
||||||
|
| Phase 3 | Docker image build + Artifactory publish | 1–2 days |
|
||||||
|
| Phase 4 | Automated deployment trigger | 2–3 days |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Technology Stack Summary
|
||||||
|
|
||||||
|
| Layer | Technology | Version | License | Purpose |
|
||||||
|
|-------|-----------|---------|---------|---------|
|
||||||
|
| **Backend** | Python | 3.11 | PSF | Runtime |
|
||||||
|
| | FastAPI | latest | MIT | Web framework |
|
||||||
|
| | Uvicorn | latest | BSD | ASGI server |
|
||||||
|
| | SQLAlchemy | latest | MIT | ORM |
|
||||||
|
| | Alembic | latest | MIT | Migrations |
|
||||||
|
| | Pydantic | v2 | MIT | Validation |
|
||||||
|
| | APScheduler | latest | MIT | Background jobs |
|
||||||
|
| | boto3 | latest | Apache 2.0 | S3 storage client |
|
||||||
|
| **Frontend** | React | 19.2 | MIT | UI framework |
|
||||||
|
| | TypeScript | 5.9 | Apache 2.0 | Type safety |
|
||||||
|
| | Vite | 7.3 | MIT | Build tooling |
|
||||||
|
| | Tailwind CSS | 4.1 | MIT | Styling |
|
||||||
|
| | TanStack Query | 5.90 | MIT | Server state |
|
||||||
|
| | Recharts | 2.15 | MIT | Visualization |
|
||||||
|
| **Database** | PostgreSQL | 15 | PostgreSQL | Relational store |
|
||||||
|
| **Storage** | MinIO | latest | AGPL-3.0 | Object storage |
|
||||||
|
| **Infrastructure** | Docker | latest | Apache 2.0 | Containerization |
|
||||||
|
| | Docker Compose | V2 | Apache 2.0 | Orchestration |
|
||||||
|
| | Nginx | Alpine | BSD | Reverse proxy |
|
||||||
|
| **CI/CD** | GitHub Enterprise | — | Commercial | Source control + CI |
|
||||||
|
| **Artifacts** | Artifactory | — | Commercial | Image registry |
|
||||||
|
|
||||||
|
### License Compliance Note
|
||||||
|
|
||||||
|
All core platform dependencies use permissive open-source licenses (MIT, BSD, Apache 2.0, PSF, PostgreSQL License). The only copyleft dependency is MinIO (AGPL-3.0), which is used as a standalone service (not linked into application code) and therefore does not impose AGPL obligations on the Aegis codebase. GitHub Enterprise and Artifactory are covered under existing organizational commercial licenses.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Approval
|
||||||
|
|
||||||
|
| Role | Name | Date | Signature |
|
||||||
|
|------|------|------|-----------|
|
||||||
|
| Platform Architect | | | |
|
||||||
|
| Security Architect | | | |
|
||||||
|
| Infrastructure Lead | | | |
|
||||||
|
| Development Lead | | | |
|
||||||
|
| Architecture Board Chair | | | |
|
||||||
+30
-13
@@ -1,26 +1,43 @@
|
|||||||
# Aegis — Task Tracker
|
# Aegis — Architectural Refactoring Task Tracker
|
||||||
|
|
||||||
## In Progress
|
## Tier 1 — Quick Wins
|
||||||
|
|
||||||
- [ ] Clean Architecture foundation: domain enums, value objects, entities, repository ports + implementations
|
- [ ] QW-1: Wire existing repos into `techniques.py` router
|
||||||
|
- [ ] QW-2: Fix `audit_service` to follow UoW (no direct `db.commit()`)
|
||||||
|
- [ ] QW-3: Consolidate `status_service` with `TechniqueEntity.recalculate_status()`
|
||||||
|
- [ ] QW-4: Remove remaining `HTTPException` from services
|
||||||
|
|
||||||
## Completed
|
## Tier 2 — Service Extraction (fat routers → thin routers + services)
|
||||||
|
|
||||||
|
- [ ] SE-1: Extract reports service from `reports.py`
|
||||||
|
- [ ] SE-2: Extract metrics service from `metrics.py`
|
||||||
|
- [ ] SE-3: Extract compliance service from `compliance.py`
|
||||||
|
- [ ] SE-4: Extract detection_rules service from `detection_rules.py`
|
||||||
|
- [ ] SE-5: Extract threat_actors service from `threat_actors.py`
|
||||||
|
|
||||||
|
## Tier 3 — Architectural Fixes
|
||||||
|
|
||||||
|
- [ ] AF-1: Persist scoring weights in DB (replace mutable `settings`)
|
||||||
|
- [ ] AF-2: Slim `tests.py` router (CRUD to repo/service)
|
||||||
|
- [ ] AF-3: Slim `evidence.py` router (permissions to domain)
|
||||||
|
- [ ] AF-4: Slim `campaigns.py` router (CRUD to service)
|
||||||
|
|
||||||
|
## Tier 4 — Polish
|
||||||
|
|
||||||
|
- [ ] P-1: Structured JSON logging
|
||||||
|
- [ ] P-2: Create architecture skill file for future agents
|
||||||
|
|
||||||
|
## Completed (prior sessions)
|
||||||
|
|
||||||
- [x] Domain exceptions hierarchy (domain/errors.py)
|
- [x] Domain exceptions hierarchy (domain/errors.py)
|
||||||
- [x] TestEntity with state machine (domain/test_entity.py)
|
- [x] TestEntity with state machine (domain/test_entity.py)
|
||||||
|
- [x] TechniqueEntity (domain/entities/technique.py)
|
||||||
|
- [x] Value objects: MitreId, ScoringWeights
|
||||||
- [x] Unit of Work (domain/unit_of_work.py)
|
- [x] Unit of Work (domain/unit_of_work.py)
|
||||||
- [x] Error handler middleware (middleware/error_handler.py)
|
- [x] Error handler middleware (middleware/error_handler.py)
|
||||||
- [x] Redis-backed token blacklist (auth.py)
|
- [x] Redis-backed token blacklist (auth.py)
|
||||||
- [x] CI pipeline (.github/workflows/ci.yml)
|
- [x] CI pipeline (.github/workflows/ci.yml)
|
||||||
- [x] Heatmap service extracted (services/heatmap_service.py)
|
- [x] Heatmap service extracted (services/heatmap_service.py)
|
||||||
- [x] Scoring bulk queries (bulk_technique_scores)
|
- [x] Scoring bulk queries (bulk_technique_scores)
|
||||||
- [x] Architecture skill file (.cursor/rules/aegis-architecture.md)
|
- [x] Repository ports + implementations (Technique, Test)
|
||||||
- [x] Agent validation script (scripts/agent_validate_backend.sh)
|
- [x] Agent validation script (scripts/agent_validate_backend.sh)
|
||||||
|
|
||||||
## Backlog
|
|
||||||
|
|
||||||
- [ ] Application layer use cases
|
|
||||||
- [ ] Migrate fat routers to use repositories
|
|
||||||
- [ ] Scoring config persistence (DB instead of mutable settings)
|
|
||||||
- [ ] Structured JSON logging
|
|
||||||
- [ ] Frontend type generation from OpenAPI
|
|
||||||
|
|||||||
Reference in New Issue
Block a user