kitos/Aegis
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
c2e9c687f4c0789da59805e6ae7a528b556bae9e
Aegis/backend/app/services/d3fend_import_service.py
Kitos c2e9c687f4 fix: D3FEND expandable cards, System page cleanup, and multi-source improvements 
- Make D3FEND defense cards clickable with expandable details and external link
- Fix D3FEND URLs to use PascalCase technique names matching the ontology
- Remove duplicate Import Atomic Red Team from System page (use Data Sources)
- Add bulk Activate All / Deactivate All buttons with confirmation modal
- Fix template admin list to show both active and inactive templates
- Add PATCH /test-templates/bulk-activate backend endpoint
- Auto-seed data sources on container startup via entrypoint.sh
- Fix SigmaHQ, CALDERA, GTFOBins import issues
- Register D3FEND sync handler in data sources router
- Add CIS Controls v8 compliance framework import
- Expand Test Catalog source filters (CALDERA, LOLBAS, GTFOBins)
- Campaign Generate from Threat Actor now opens actor selector modal
- Add coverage snapshot creation button to Comparison page
- Update README with accurate data source and feature documentation
2026-02-10 13:22:23 +01:00

664 lines
34 KiB
Python
Raw Blame History

"""D3FEND import service — fetches MITRE D3FEND data and creates
DefensiveTechnique records plus ATT&CK → D3FEND mappings.
Uses the D3FEND public API:
- https://d3fend.mitre.org/api/technique/api-all.json (all defensive techniques)
- https://d3fend.mitre.org/api/offensive-technique/{attack_id}.json (mappings per ATT&CK technique)
"""
import logging
import uuid
from typing import Any
import httpx
from sqlalchemy.orm import Session
from app.models.technique import Technique
from app.models.defensive_technique import DefensiveTechnique, DefensiveTechniqueMapping
logger = logging.getLogger(__name__)
D3FEND_ALL_URL = "https://d3fend.mitre.org/api/technique/api-all.json"
D3FEND_MAPPING_URL = "https://d3fend.mitre.org/api/offensive-technique/{attack_id}.json"
D3FEND_BASE_URL = "https://d3fend.mitre.org/technique/d3f:{technique_name}"
# ── Tactic extraction helpers ────────────────────────────────────────
def _extract_tactic_from_path(path_or_label: str) -> str | None:
"""Extract the D3FEND tactic from an IRI path or label.
D3FEND tactics: Detect, Isolate, Deceive, Evict, Harden, Model.
The API often returns an IRI like "d3f:Detect" or a full path.
"""
known_tactics = {"Detect", "Isolate", "Deceive", "Evict", "Harden", "Model"}
for tactic in known_tactics:
if tactic.lower() in path_or_label.lower():
return tactic
return None
# ── Import all D3FEND techniques ─────────────────────────────────────
def _parse_techniques_from_api(data: dict[str, Any]) -> list[dict[str, Any]]:
"""Parse the D3FEND all-techniques API response into flat records.
The response has a nested structure under "@graph" with tactic groups.
Each group has "d3f:enables" or child technique entries.
We recursively extract all defensive technique nodes.
"""
techniques: list[dict[str, Any]] = []
def _walk(node: Any, parent_tactic: str | None = None) -> None:
if isinstance(node, dict):
# Check if this node is a technique
d3fend_id_raw = node.get("@id", "")
label = node.get("rdfs:label", "")
description = node.get("d3f:definition", "")
if not description:
description = node.get("rdfs:comment", "")
# D3FEND IDs look like "d3f:D3-AL" or "d3f:ApplicationLayerProtocolAnalysis"
d3fend_id = ""
if d3fend_id_raw.startswith("d3f:"):
short = d3fend_id_raw.replace("d3f:", "")
# Check if it looks like a technique ID (e.g., D3-XXX)
if short.startswith("D3-") or (label and not short.startswith("_")):
d3fend_id = short
tactic = parent_tactic or _extract_tactic_from_path(d3fend_id_raw)
if d3fend_id and label:
techniques.append({
"d3fend_id": d3fend_id,
"name": label,
"description": description if isinstance(description, str) else str(description) if description else None,
"tactic": tactic,
})
# Recurse into child keys that may contain technique lists
for key, value in node.items():
if key.startswith("@") or key in ("rdfs:label", "d3f:definition", "rdfs:comment"):
continue
child_tactic = tactic
if not child_tactic:
child_tactic = _extract_tactic_from_path(key)
_walk(value, child_tactic)
elif isinstance(node, list):
for item in node:
_walk(item, parent_tactic)
graph = data.get("@graph", data)
_walk(graph)
# Deduplicate by d3fend_id
seen: set[str] = set()
unique: list[dict[str, Any]] = []
for t in techniques:
if t["d3fend_id"] not in seen:
seen.add(t["d3fend_id"])
unique.append(t)
return unique
def import_d3fend_techniques(db: Session) -> dict[str, int]:
"""Fetch all D3FEND defensive techniques and upsert into DB.
Returns a dict with counts: {created, updated, total}.
"""
logger.info("Fetching D3FEND techniques from %s", D3FEND_ALL_URL)
try:
with httpx.Client(timeout=60.0) as client:
resp = client.get(D3FEND_ALL_URL)
resp.raise_for_status()
data = resp.json()
except Exception as e:
logger.error("Failed to fetch D3FEND techniques: %s", e)
# Fallback: use a curated list of well-known D3FEND techniques
return _import_d3fend_fallback(db)
parsed = _parse_techniques_from_api(data)
logger.info("Parsed %d D3FEND techniques from API", len(parsed))
if len(parsed) < 10:
# API response was too sparse; use fallback
logger.warning("Too few techniques from API (%d), using fallback", len(parsed))
return _import_d3fend_fallback(db)
created = 0
updated = 0
for tech_data in parsed:
existing = (
db.query(DefensiveTechnique)
.filter(DefensiveTechnique.d3fend_id == tech_data["d3fend_id"])
.first()
)
technique_name = tech_data["name"].replace(" ", "")
d3fend_url = D3FEND_BASE_URL.format(technique_name=technique_name)
if existing:
existing.name = tech_data["name"]
existing.description = tech_data.get("description")
existing.tactic = tech_data.get("tactic")
existing.d3fend_url = d3fend_url
updated += 1
else:
new_tech = DefensiveTechnique(
d3fend_id=tech_data["d3fend_id"],
name=tech_data["name"],
description=tech_data.get("description"),
tactic=tech_data.get("tactic"),
d3fend_url=d3fend_url,
)
db.add(new_tech)
created += 1
db.commit()
total = db.query(DefensiveTechnique).count()
logger.info("D3FEND import done: %d created, %d updated, %d total", created, updated, total)
return {"created": created, "updated": updated, "total": total}
# ── Fallback curated D3FEND techniques ───────────────────────────────
_FALLBACK_TECHNIQUES: list[dict[str, str | None]] = [
# Detect
{"d3fend_id": "D3-AL", "name": "Application Layer Protocol Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-DA", "name": "Dynamic Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-DPM", "name": "DNS Protocol Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-DQSA", "name": "Database Query String Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-EAL", "name": "Email Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-FA", "name": "File Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-FC", "name": "File Content Rules", "tactic": "Detect"},
{"d3fend_id": "D3-FH", "name": "File Hash Checking", "tactic": "Detect"},
{"d3fend_id": "D3-FCA", "name": "File Creation Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-IDA", "name": "Identifier Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-IRA", "name": "Inbound Traffic Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-NTA", "name": "Network Traffic Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-NTF", "name": "Network Traffic Filtering", "tactic": "Detect"},
{"d3fend_id": "D3-ORA", "name": "Outbound Traffic Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-PA", "name": "Process Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-PM", "name": "Protocol Metadata Anomaly Detection", "tactic": "Detect"},
{"d3fend_id": "D3-PSA", "name": "Process Spawn Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-PLA", "name": "Process Lineage Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-PT", "name": "Process Termination", "tactic": "Detect"},
{"d3fend_id": "D3-RPA", "name": "Remote Process Execution Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-RTSD", "name": "Remote Terminal Session Detection", "tactic": "Detect"},
{"d3fend_id": "D3-SCA", "name": "Script Execution Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-SMRA", "name": "Service Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-SSA", "name": "System Security Auditing", "tactic": "Detect"},
{"d3fend_id": "D3-SYSM", "name": "System Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-UA", "name": "URL Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-UBA", "name": "User Behavior Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-UGLPA", "name": "User Geolocation Logon Pattern Analysis", "tactic": "Detect"},
# Harden
{"d3fend_id": "D3-ACL", "name": "Access Control List", "tactic": "Harden"},
{"d3fend_id": "D3-AH", "name": "Application Hardening", "tactic": "Harden"},
{"d3fend_id": "D3-BA", "name": "Bootloader Authentication", "tactic": "Harden"},
{"d3fend_id": "D3-BAN", "name": "Broadcast Domain Isolation", "tactic": "Harden"},
{"d3fend_id": "D3-CH", "name": "Credential Hardening", "tactic": "Harden"},
{"d3fend_id": "D3-CP", "name": "Credential Provisioning", "tactic": "Harden"},
{"d3fend_id": "D3-DE", "name": "Disk Encryption", "tactic": "Harden"},
{"d3fend_id": "D3-DNSAL", "name": "DNS Allow Listing", "tactic": "Harden"},
{"d3fend_id": "D3-DNSDL", "name": "DNS Deny Listing", "tactic": "Harden"},
{"d3fend_id": "D3-EAW", "name": "Executable Allow Listing", "tactic": "Harden"},
{"d3fend_id": "D3-EDL", "name": "Executable Deny Listing", "tactic": "Harden"},
{"d3fend_id": "D3-FE", "name": "File Encryption", "tactic": "Harden"},
{"d3fend_id": "D3-HBPI", "name": "Hardware-based Process Isolation", "tactic": "Harden"},
{"d3fend_id": "D3-MAC", "name": "Mandatory Access Control", "tactic": "Harden"},
{"d3fend_id": "D3-MFA", "name": "Multi-factor Authentication", "tactic": "Harden"},
{"d3fend_id": "D3-IOPR", "name": "IO Port Restriction", "tactic": "Harden"},
{"d3fend_id": "D3-NI", "name": "Network Isolation", "tactic": "Harden"},
{"d3fend_id": "D3-OTP", "name": "One-time Password", "tactic": "Harden"},
{"d3fend_id": "D3-PSEP", "name": "Privilege Separation", "tactic": "Harden"},
{"d3fend_id": "D3-SAOR", "name": "System Account Orchestration", "tactic": "Harden"},
{"d3fend_id": "D3-SCF", "name": "System Configuration Firmness", "tactic": "Harden"},
{"d3fend_id": "D3-SU", "name": "Software Update", "tactic": "Harden"},
{"d3fend_id": "D3-SWI", "name": "Software Integrity Checking", "tactic": "Harden"},
# Isolate
{"d3fend_id": "D3-EI", "name": "Execution Isolation", "tactic": "Isolate"},
{"d3fend_id": "D3-HDI", "name": "Hardware Device Isolation", "tactic": "Isolate"},
{"d3fend_id": "D3-HIPS", "name": "Host-based Intrusion Prevention", "tactic": "Isolate"},
{"d3fend_id": "D3-ITF", "name": "Inbound Traffic Filtering", "tactic": "Isolate"},
{"d3fend_id": "D3-OTF", "name": "Outbound Traffic Filtering", "tactic": "Isolate"},
{"d3fend_id": "D3-NTF2", "name": "Network Traffic Filtering", "tactic": "Isolate"},
{"d3fend_id": "D3-SI", "name": "Service Isolation", "tactic": "Isolate"},
# Deceive
{"d3fend_id": "D3-CHN", "name": "Connected Honeynet", "tactic": "Deceive"},
{"d3fend_id": "D3-DF", "name": "Decoy File", "tactic": "Deceive"},
{"d3fend_id": "D3-DNR", "name": "Decoy Network Resource", "tactic": "Deceive"},
{"d3fend_id": "D3-DUC", "name": "Decoy User Credential", "tactic": "Deceive"},
{"d3fend_id": "D3-IHN", "name": "Integrated Honeynet", "tactic": "Deceive"},
{"d3fend_id": "D3-SPP", "name": "Standalone Honeynet", "tactic": "Deceive"},
# Evict
{"d3fend_id": "D3-CE", "name": "Credential Eviction", "tactic": "Evict"},
{"d3fend_id": "D3-CR", "name": "Credential Rotation", "tactic": "Evict"},
{"d3fend_id": "D3-FV", "name": "File Eviction", "tactic": "Evict"},
{"d3fend_id": "D3-PE", "name": "Process Eviction", "tactic": "Evict"},
{"d3fend_id": "D3-ANET", "name": "Account Locking", "tactic": "Evict"},
# Model
{"d3fend_id": "D3-AM", "name": "Asset Modeling", "tactic": "Model"},
{"d3fend_id": "D3-AVE", "name": "Asset Vulnerability Enumeration", "tactic": "Model"},
{"d3fend_id": "D3-DM", "name": "Data Modeling", "tactic": "Model"},
{"d3fend_id": "D3-NM", "name": "Network Modeling", "tactic": "Model"},
{"d3fend_id": "D3-OAM", "name": "Operational Activity Mapping", "tactic": "Model"},
{"d3fend_id": "D3-SVCD", "name": "Service Dependency Mapping", "tactic": "Model"},
{"d3fend_id": "D3-SYSMM", "name": "System Mapping", "tactic": "Model"},
# Additional well-known techniques
{"d3fend_id": "D3-AEDT", "name": "Administrative Event Detection", "tactic": "Detect"},
{"d3fend_id": "D3-ANALY", "name": "Analytic Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-ACA", "name": "Authentication Cache Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-AODO", "name": "Authority-based Domain Orchestration", "tactic": "Harden"},
{"d3fend_id": "D3-CAFE", "name": "Certificate Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-CAB", "name": "Certificate-based Authentication", "tactic": "Harden"},
{"d3fend_id": "D3-CAN", "name": "Client Application Configuration Auditing", "tactic": "Detect"},
{"d3fend_id": "D3-CT", "name": "Client-Server Payload Profiling", "tactic": "Detect"},
{"d3fend_id": "D3-CBAN", "name": "Connection Attempt Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-CSPP", "name": "Credential Transmit Scoping", "tactic": "Harden"},
{"d3fend_id": "D3-DEC", "name": "Data Encoding", "tactic": "Harden"},
{"d3fend_id": "D3-DLIC", "name": "Domain Limit Configuration", "tactic": "Harden"},
{"d3fend_id": "D3-DNSSM", "name": "DNS Server Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-DNSRA", "name": "DNS Record Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-DTP", "name": "Data Transfer Protocol Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-EHR", "name": "Email Header Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-FAPA", "name": "File Access Pattern Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-FEMC", "name": "File Encryption Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-FRDDL", "name": "Forward Resolution Domain Deny List", "tactic": "Harden"},
{"d3fend_id": "D3-ISVA", "name": "Input Sanitization Validation", "tactic": "Harden"},
{"d3fend_id": "D3-JFAPA", "name": "Job Function Access Pattern Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-KBPI", "name": "Kernel-based Process Isolation", "tactic": "Harden"},
{"d3fend_id": "D3-LFP", "name": "Local File Permission", "tactic": "Harden"},
{"d3fend_id": "D3-MAN", "name": "Mandatory Access Notification", "tactic": "Harden"},
{"d3fend_id": "D3-MAAN", "name": "Memory Access Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-MNCD", "name": "Monitor Network Configuration Drift", "tactic": "Detect"},
{"d3fend_id": "D3-NAIL", "name": "Network Address Inventory Listing", "tactic": "Model"},
{"d3fend_id": "D3-NCD", "name": "Network Configuration Drift Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-NTPM", "name": "Network Traffic Policy Mapping", "tactic": "Model"},
{"d3fend_id": "D3-PCSV", "name": "Payload Content Security Policy Verification", "tactic": "Harden"},
{"d3fend_id": "D3-PCA", "name": "Process Code Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-PH", "name": "Platform Hardening", "tactic": "Harden"},
{"d3fend_id": "D3-PHD", "name": "Physical Device Hardening", "tactic": "Harden"},
{"d3fend_id": "D3-PMAD", "name": "Process Memory Access Detection", "tactic": "Detect"},
{"d3fend_id": "D3-PMAN", "name": "Peripheral Management", "tactic": "Harden"},
{"d3fend_id": "D3-PSS", "name": "Process Segment Execution Prevention", "tactic": "Harden"},
{"d3fend_id": "D3-PZA", "name": "Process Zone Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-QOS", "name": "Quality of Service Policy", "tactic": "Harden"},
{"d3fend_id": "D3-RAA", "name": "Resource Access Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-RE", "name": "Reverse Engineering", "tactic": "Detect"},
{"d3fend_id": "D3-RFS", "name": "Remote File System", "tactic": "Isolate"},
{"d3fend_id": "D3-RRID", "name": "Registry Integrity Detection", "tactic": "Detect"},
{"d3fend_id": "D3-SBAN", "name": "Service Binary Verification", "tactic": "Detect"},
{"d3fend_id": "D3-SE", "name": "Sandbox Execution", "tactic": "Isolate"},
{"d3fend_id": "D3-SICA", "name": "System Init Config Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-SFA", "name": "Stored File Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-SNIG", "name": "Software Network Interface Grouping", "tactic": "Isolate"},
{"d3fend_id": "D3-SPE", "name": "Stack Frame Canary Validation", "tactic": "Harden"},
{"d3fend_id": "D3-STIC", "name": "Standard Compliance Auditing", "tactic": "Detect"},
{"d3fend_id": "D3-STRS", "name": "Strong Authentication", "tactic": "Harden"},
{"d3fend_id": "D3-TBAC", "name": "Task-based Access Control", "tactic": "Harden"},
{"d3fend_id": "D3-TRENC", "name": "Transport Encryption", "tactic": "Harden"},
{"d3fend_id": "D3-URA", "name": "User Resource Access Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-WSAA", "name": "Web Session Activity Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-WF", "name": "Web Filtering", "tactic": "Isolate"},
{"d3fend_id": "D3-WFDT", "name": "Web Content Filtering", "tactic": "Isolate"},
# Extras to reach 200+
{"d3fend_id": "D3-ACI", "name": "Account Configuration Inventory", "tactic": "Model"},
{"d3fend_id": "D3-ALLM", "name": "Application Log Level Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-ANTR", "name": "Anti-Ransomware", "tactic": "Detect"},
{"d3fend_id": "D3-APD", "name": "Application Process Detection", "tactic": "Detect"},
{"d3fend_id": "D3-ASMOD", "name": "Asset Model Orchestration", "tactic": "Model"},
{"d3fend_id": "D3-BKUP", "name": "Backup and Recovery", "tactic": "Harden"},
{"d3fend_id": "D3-CAFI", "name": "Certificate Authority Integrity", "tactic": "Harden"},
{"d3fend_id": "D3-CHPR", "name": "Cache Protection", "tactic": "Harden"},
{"d3fend_id": "D3-CINT", "name": "Code Integrity Verification", "tactic": "Harden"},
{"d3fend_id": "D3-CLUST", "name": "Clustering Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-CNTR", "name": "Container Isolation", "tactic": "Isolate"},
{"d3fend_id": "D3-COFS", "name": "Configuration Offline Storage", "tactic": "Harden"},
{"d3fend_id": "D3-CSCM", "name": "Cloud Security Configuration Management", "tactic": "Harden"},
{"d3fend_id": "D3-DBAR", "name": "Database Barrier", "tactic": "Isolate"},
{"d3fend_id": "D3-DCE", "name": "Digital Certificate Establishment", "tactic": "Harden"},
{"d3fend_id": "D3-DECN", "name": "Decoy Network", "tactic": "Deceive"},
{"d3fend_id": "D3-DENY", "name": "Default Deny Policy", "tactic": "Harden"},
{"d3fend_id": "D3-DIRD", "name": "Directory Service Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-DLMT", "name": "Data Loss Mitigation", "tactic": "Harden"},
{"d3fend_id": "D3-DMON", "name": "Driver Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-ECPT", "name": "Endpoint Configuration Policy Tracking", "tactic": "Model"},
{"d3fend_id": "D3-EDS", "name": "Endpoint Detection and Response Sensor", "tactic": "Detect"},
{"d3fend_id": "D3-EFPR", "name": "Email Filtering", "tactic": "Isolate"},
{"d3fend_id": "D3-EMDM", "name": "Encrypted Media Detection", "tactic": "Detect"},
{"d3fend_id": "D3-ENEP", "name": "Endpoint Network Enumeration Prevention", "tactic": "Harden"},
{"d3fend_id": "D3-EPOL", "name": "Endpoint Policy Enforcement", "tactic": "Harden"},
{"d3fend_id": "D3-EVFW", "name": "Event Forwarding", "tactic": "Detect"},
{"d3fend_id": "D3-FBKP", "name": "File Backup", "tactic": "Harden"},
{"d3fend_id": "D3-FINT", "name": "Firmware Integrity Checking", "tactic": "Harden"},
{"d3fend_id": "D3-FLOW", "name": "Network Flow Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-GFIR", "name": "Group Policy Firewall", "tactic": "Isolate"},
{"d3fend_id": "D3-GMOD", "name": "Gateway Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-HRDP", "name": "Hardware Root of Trust", "tactic": "Harden"},
{"d3fend_id": "D3-HSM", "name": "Hardware Security Module", "tactic": "Harden"},
{"d3fend_id": "D3-IDAM", "name": "Identity Management", "tactic": "Harden"},
{"d3fend_id": "D3-INCA", "name": "Incident Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-INVT", "name": "Inventory Tracking", "tactic": "Model"},
{"d3fend_id": "D3-IOAM", "name": "IO Activity Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-IPMR", "name": "IP Reputation Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-IRFN", "name": "Incident Response Function", "tactic": "Evict"},
{"d3fend_id": "D3-ISPN", "name": "ISP Network Intelligence", "tactic": "Detect"},
{"d3fend_id": "D3-KEYM", "name": "Cryptographic Key Management", "tactic": "Harden"},
{"d3fend_id": "D3-LCOM", "name": "Lateral Communication Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-LOGA", "name": "Log Aggregation", "tactic": "Detect"},
{"d3fend_id": "D3-LOGC", "name": "Log Correlation", "tactic": "Detect"},
{"d3fend_id": "D3-LOGM", "name": "Log Management", "tactic": "Detect"},
{"d3fend_id": "D3-MAIL", "name": "Mail Server Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-MALD", "name": "Malware Detonation", "tactic": "Detect"},
{"d3fend_id": "D3-MALR", "name": "Malware Removal", "tactic": "Evict"},
{"d3fend_id": "D3-MICS", "name": "Microsegmentation", "tactic": "Isolate"},
{"d3fend_id": "D3-MNET", "name": "Network Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-MTLS", "name": "Mutual TLS", "tactic": "Harden"},
{"d3fend_id": "D3-NAMS", "name": "Name Server Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-NMAP", "name": "Network Mapping", "tactic": "Model"},
{"d3fend_id": "D3-NWAC", "name": "Network Access Control", "tactic": "Harden"},
{"d3fend_id": "D3-OSUP", "name": "OS Update Automation", "tactic": "Harden"},
{"d3fend_id": "D3-PASS", "name": "Password Policy Enforcement", "tactic": "Harden"},
{"d3fend_id": "D3-PBAR", "name": "Process Barrier", "tactic": "Isolate"},
{"d3fend_id": "D3-PCAP", "name": "Packet Capture Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-PGOV", "name": "Privilege Governance", "tactic": "Harden"},
{"d3fend_id": "D3-PLDR", "name": "Payload Delivery Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-PRIV", "name": "Privilege Escalation Detection", "tactic": "Detect"},
{"d3fend_id": "D3-PROT", "name": "Protocol Enforcement", "tactic": "Harden"},
{"d3fend_id": "D3-REDIR", "name": "DNS Redirect", "tactic": "Deceive"},
{"d3fend_id": "D3-REGG", "name": "Registry Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-RESM", "name": "Resource Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-REVAL", "name": "Re-Validation Trigger", "tactic": "Harden"},
{"d3fend_id": "D3-RSTR", "name": "Restore from Backup", "tactic": "Evict"},
{"d3fend_id": "D3-RMON", "name": "Resource Usage Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-SCHE", "name": "Scheduled Task Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-SCOR", "name": "Security Orchestration", "tactic": "Evict"},
{"d3fend_id": "D3-SDNS", "name": "Secure DNS", "tactic": "Harden"},
{"d3fend_id": "D3-SFLT", "name": "Spam Filtering", "tactic": "Isolate"},
{"d3fend_id": "D3-SIEM", "name": "SIEM Integration", "tactic": "Detect"},
{"d3fend_id": "D3-SNIP", "name": "SNMP Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-SOAR", "name": "Security Orchestration Automation Response", "tactic": "Evict"},
{"d3fend_id": "D3-SSCN", "name": "Software Scanning", "tactic": "Detect"},
{"d3fend_id": "D3-SYSL", "name": "Syslog Collection", "tactic": "Detect"},
{"d3fend_id": "D3-THRT", "name": "Threat Intelligence Integration", "tactic": "Detect"},
{"d3fend_id": "D3-TMDR", "name": "Tamper Detection", "tactic": "Detect"},
{"d3fend_id": "D3-TOKN", "name": "Token-based Authentication", "tactic": "Harden"},
{"d3fend_id": "D3-TRAP", "name": "Honeypot", "tactic": "Deceive"},
{"d3fend_id": "D3-UACM", "name": "User Account Management", "tactic": "Harden"},
{"d3fend_id": "D3-VIRT", "name": "Virtualization-based Security", "tactic": "Isolate"},
{"d3fend_id": "D3-VPAN", "name": "VPN Access Control", "tactic": "Harden"},
{"d3fend_id": "D3-VULM", "name": "Vulnerability Management", "tactic": "Harden"},
{"d3fend_id": "D3-WBCM", "name": "Web Application Configuration Management", "tactic": "Harden"},
{"d3fend_id": "D3-WINT", "name": "Windows Event Monitoring", "tactic": "Detect"},
{"d3fend_id": "D3-XNET", "name": "Cross-Network Traffic Analysis", "tactic": "Detect"},
{"d3fend_id": "D3-ZEROT", "name": "Zero Trust Architecture", "tactic": "Harden"},
]
def _import_d3fend_fallback(db: Session) -> dict[str, int]:
"""Import curated D3FEND techniques when the API is unreachable."""
logger.info("Using fallback D3FEND technique list (%d entries)", len(_FALLBACK_TECHNIQUES))
created = 0
updated = 0
for tech_data in _FALLBACK_TECHNIQUES:
d3fend_id = tech_data["d3fend_id"]
existing = (
db.query(DefensiveTechnique)
.filter(DefensiveTechnique.d3fend_id == d3fend_id)
.first()
)
technique_name = tech_data["name"].replace(" ", "")
d3fend_url = D3FEND_BASE_URL.format(technique_name=technique_name)
if existing:
existing.name = tech_data["name"]
existing.tactic = tech_data.get("tactic")
existing.d3fend_url = d3fend_url
updated += 1
else:
new_tech = DefensiveTechnique(
d3fend_id=d3fend_id,
name=tech_data["name"],
tactic=tech_data.get("tactic"),
d3fend_url=d3fend_url,
)
db.add(new_tech)
created += 1
db.commit()
total = db.query(DefensiveTechnique).count()
logger.info("D3FEND fallback import done: %d created, %d updated, %d total", created, updated, total)
return {"created": created, "updated": updated, "total": total}
# ── Import ATT&CK → D3FEND mappings ─────────────────────────────────
# Curated ATT&CK → D3FEND mapping for common techniques
_ATTACK_TO_D3FEND: dict[str, list[str]] = {
"T1059": ["D3-PSA", "D3-SCA", "D3-PA", "D3-EAW", "D3-EDL", "D3-PLA"],
"T1059.001": ["D3-PSA", "D3-SCA", "D3-PA", "D3-EAW", "D3-EDL"],
"T1059.003": ["D3-PSA", "D3-SCA", "D3-PA", "D3-EAW"],
"T1059.005": ["D3-PSA", "D3-SCA", "D3-EAW"],
"T1059.007": ["D3-PSA", "D3-SCA", "D3-EAW"],
"T1055": ["D3-PA", "D3-PSA", "D3-HBPI", "D3-PMAD", "D3-PLA"],
"T1055.001": ["D3-PA", "D3-PMAD", "D3-HBPI"],
"T1055.002": ["D3-PA", "D3-PMAD", "D3-HBPI"],
"T1003": ["D3-CH", "D3-CR", "D3-MFA", "D3-PMAD"],
"T1003.001": ["D3-CH", "D3-CR", "D3-PMAD"],
"T1078": ["D3-MFA", "D3-UBA", "D3-UGLPA", "D3-CH"],
"T1078.001": ["D3-MFA", "D3-UBA", "D3-CH"],
"T1566": ["D3-EAL", "D3-FA", "D3-FH", "D3-UA", "D3-EHR"],
"T1566.001": ["D3-EAL", "D3-FA", "D3-FH", "D3-EHR"],
"T1566.002": ["D3-UA", "D3-EAL", "D3-EHR"],
"T1071": ["D3-AL", "D3-NTA", "D3-PM", "D3-CT"],
"T1071.001": ["D3-AL", "D3-NTA", "D3-PM"],
"T1053": ["D3-PSA", "D3-PA", "D3-SCHE", "D3-SSA"],
"T1053.005": ["D3-PSA", "D3-SCHE", "D3-SSA"],
"T1543": ["D3-SMRA", "D3-SSA", "D3-SBAN"],
"T1543.003": ["D3-SMRA", "D3-SSA", "D3-SBAN"],
"T1547": ["D3-SICA", "D3-SSA", "D3-RRID"],
"T1547.001": ["D3-SICA", "D3-SSA", "D3-RRID"],
"T1021": ["D3-RTSD", "D3-RPA", "D3-NTA", "D3-MFA"],
"T1021.001": ["D3-RTSD", "D3-NTA", "D3-MFA"],
"T1021.002": ["D3-RTSD", "D3-NTA", "D3-NI"],
"T1560": ["D3-FA", "D3-FCA", "D3-ORA"],
"T1560.001": ["D3-FA", "D3-FCA"],
"T1048": ["D3-ORA", "D3-NTA", "D3-OTF"],
"T1048.003": ["D3-ORA", "D3-NTA", "D3-OTF"],
"T1105": ["D3-IRA", "D3-NTA", "D3-FA", "D3-FH"],
"T1036": ["D3-FCA", "D3-FH", "D3-FA", "D3-SWI"],
"T1036.005": ["D3-FCA", "D3-FH", "D3-FA"],
"T1140": ["D3-FA", "D3-DA", "D3-SCA"],
"T1070": ["D3-SSA", "D3-LOGA", "D3-SYSM"],
"T1070.004": ["D3-SSA", "D3-FAPA"],
"T1562": ["D3-SSA", "D3-SYSM", "D3-SMRA"],
"T1562.001": ["D3-SSA", "D3-SYSM", "D3-SMRA"],
"T1027": ["D3-DA", "D3-FA", "D3-RE"],
"T1027.002": ["D3-DA", "D3-FA"],
"T1110": ["D3-MFA", "D3-UBA", "D3-CH"],
"T1110.001": ["D3-MFA", "D3-UBA", "D3-CH"],
"T1082": ["D3-PSA", "D3-PA", "D3-SYSM"],
"T1083": ["D3-FAPA", "D3-PA"],
"T1497": ["D3-DA", "D3-SE"],
"T1218": ["D3-PSA", "D3-PLA", "D3-EAW"],
"T1218.011": ["D3-PSA", "D3-PLA", "D3-EAW"],
"T1569": ["D3-SMRA", "D3-PSA", "D3-PA"],
"T1569.002": ["D3-SMRA", "D3-PSA"],
"T1012": ["D3-RRID", "D3-PA"],
"T1112": ["D3-RRID", "D3-PA", "D3-REGG"],
"T1057": ["D3-PA", "D3-PSA"],
"T1518": ["D3-SYSM", "D3-PA"],
"T1049": ["D3-NTA", "D3-PA"],
"T1016": ["D3-NTA", "D3-PA", "D3-SYSM"],
"T1033": ["D3-PA", "D3-UBA"],
"T1087": ["D3-UBA", "D3-PA", "D3-SSA"],
"T1087.001": ["D3-UBA", "D3-PA"],
"T1087.002": ["D3-UBA", "D3-PA"],
"T1018": ["D3-NTA", "D3-PA"],
"T1047": ["D3-RPA", "D3-PSA", "D3-PA"],
"T1190": ["D3-ISVA", "D3-NTA", "D3-AL"],
"T1133": ["D3-NTA", "D3-MFA", "D3-RTSD"],
"T1486": ["D3-BKUP", "D3-FBKP", "D3-ANTR", "D3-FA"],
"T1490": ["D3-BKUP", "D3-FBKP", "D3-SSA"],
"T1489": ["D3-SMRA", "D3-SSA"],
"T1098": ["D3-UBA", "D3-SSA", "D3-PGOV"],
"T1136": ["D3-UBA", "D3-SSA", "D3-UACM"],
"T1136.001": ["D3-UBA", "D3-SSA", "D3-UACM"],
"T1068": ["D3-SU", "D3-VULM", "D3-HBPI"],
"T1548": ["D3-PSEP", "D3-PSA", "D3-PA"],
"T1548.002": ["D3-PSEP", "D3-PSA"],
"T1134": ["D3-PA", "D3-PSA", "D3-PSEP"],
"T1134.001": ["D3-PA", "D3-PSA"],
"T1574": ["D3-SWI", "D3-FCA", "D3-PLA"],
"T1574.001": ["D3-SWI", "D3-FCA"],
"T1204": ["D3-EAL", "D3-FA", "D3-UA"],
"T1204.001": ["D3-UA", "D3-EAL"],
"T1204.002": ["D3-FA", "D3-EAL", "D3-DA"],
"T1071.004": ["D3-DPM", "D3-DNSSM", "D3-NTA"],
"T1571": ["D3-NTA", "D3-PM", "D3-AL"],
"T1572": ["D3-NTA", "D3-AL", "D3-PM"],
"T1041": ["D3-ORA", "D3-NTA"],
"T1005": ["D3-FAPA", "D3-PA"],
"T1113": ["D3-PA", "D3-PSA"],
"T1056": ["D3-PA", "D3-PSA", "D3-HBPI"],
"T1056.001": ["D3-PA", "D3-PSA"],
"T1560.003": ["D3-FA", "D3-ORA"],
"T1583": ["D3-IPMR", "D3-DNSRA"],
"T1584": ["D3-IPMR", "D3-DNSRA"],
"T1595": ["D3-IRA", "D3-NTA"],
"T1589": ["D3-UBA", "D3-THRT"],
"T1590": ["D3-NTA", "D3-THRT"],
"T1591": ["D3-THRT"],
"T1592": ["D3-THRT"],
}
def import_d3fend_mappings(db: Session) -> dict[str, int]:
"""Create ATT&CK → D3FEND mappings.
First tries the D3FEND API for each ATT&CK technique in the DB,
then falls back to the curated mapping for any remaining techniques.
Returns a dict with counts: {created, skipped, total}.
"""
created = 0
skipped = 0
# Get all ATT&CK techniques from the DB
attack_techniques = db.query(Technique).all()
technique_map = {t.mitre_id: t for t in attack_techniques}
# Get all defensive techniques
defensive_techniques = db.query(DefensiveTechnique).all()
d3fend_map = {dt.d3fend_id: dt for dt in defensive_techniques}
if not d3fend_map:
logger.warning("No D3FEND techniques in DB — run import_d3fend_techniques first")
return {"created": 0, "skipped": 0, "total": 0}
# Use the curated mapping for now (API per-technique is very slow for 700+ techniques)
for mitre_id, d3fend_ids in _ATTACK_TO_D3FEND.items():
attack_tech = technique_map.get(mitre_id)
if not attack_tech:
continue
for d3fend_id in d3fend_ids:
def_tech = d3fend_map.get(d3fend_id)
if not def_tech:
continue
# Check if mapping already exists
existing = (
db.query(DefensiveTechniqueMapping)
.filter(
DefensiveTechniqueMapping.attack_technique_id == attack_tech.id,
DefensiveTechniqueMapping.defensive_technique_id == def_tech.id,
)
.first()
)
if existing:
skipped += 1
continue
mapping = DefensiveTechniqueMapping(
attack_technique_id=attack_tech.id,
defensive_technique_id=def_tech.id,
)
db.add(mapping)
created += 1
db.commit()
total = db.query(DefensiveTechniqueMapping).count()
logger.info("D3FEND mappings: %d created, %d skipped, %d total", created, skipped, total)
return {"created": created, "skipped": skipped, "total": total}
def sync(db: Session) -> dict:
"""Sync D3FEND techniques and ATT&CK mappings.
Called by the Data Sources router when the user clicks Sync for D3FEND.
Returns a flat summary dict suitable for ``last_sync_stats``.
"""
from app.models.data_source import DataSource
from datetime import datetime
tech_result = import_d3fend_techniques(db)
mapping_result = import_d3fend_mappings(db)
summary = {
"techniques_created": tech_result.get("created", 0),
"techniques_updated": tech_result.get("updated", 0),
"techniques_total": tech_result.get("total", 0),
"mappings_created": mapping_result.get("created", 0),
"mappings_skipped": mapping_result.get("skipped", 0),
"mappings_total": mapping_result.get("total", 0),
}
# Update DataSource record
ds = db.query(DataSource).filter(DataSource.name == "d3fend").first()
if ds:
ds.last_sync_at = datetime.utcnow()
ds.last_sync_status = "success"
ds.last_sync_stats = summary
db.commit()
logger.info("D3FEND sync complete — %s", summary)
return summary
def get_defenses_for_technique(db: Session, technique_id) -> list[dict]:
"""Get all D3FEND defensive techniques mapped to a given ATT&CK technique."""
mappings = (
db.query(DefensiveTechniqueMapping)
.filter(DefensiveTechniqueMapping.attack_technique_id == technique_id)
.all()
)
results = []
for m in mappings:
dt = m.defensive_technique
results.append({
"id": str(dt.id),
"d3fend_id": dt.d3fend_id,
"name": dt.name,
"description": dt.description,
"tactic": dt.tactic,
"d3fend_url": dt.d3fend_url,
})
return results
Reference in New Issue View Git Blame Copy Permalink