kitos/Aegis
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor(pep8): enforce full PEP8 compliance across backend Python codebase

Browse Source 
- ruff.toml: select E/W/F/I/N rules, line-length=120, drop legacy ignores
- Auto-fix: sort 82 import blocks (isort), remove 29 unused imports,
  strip 6 trailing-whitespace blank lines in docstrings
- main.py: move setup_logging and settings imports to top (E402)
- errors.py: noqa N818 on DDD exception names (96 call sites, safe)
- intel_service.py: noqa N817 for universal ET alias
- atomic/elastic/sigma import services: move _MAX_UNCOMPRESSED_SIZE and
  _MAX_ENTRIES to module level (N806)
- compliance_import_service.py: move SAMPLE_CONTROLS / CIS_CONTROLS to
  module level; wrap long description strings (N806 + E501)
- snapshot_service.py: move STATUS_ORDER dict to module level (N806)
- sigma_import_service.py: remove dead dedup_key expression (F841)
- threat_actor_import_service.py: remove dead stix_to_actor expression (F841)
- data_source.py, seed_demo.py, campaign_scheduler_service.py,
  lolbas_import_service.py: wrap lines exceeding 120 chars (E501)
- d3fend_import_service.py: per-file E501 ignore (data file with long strings)

All 439 unit tests pass. ruff check app/ → All checks passed!
This commit is contained in:
kitos
2026-06-09 16:40:14 +02:00
parent 1249391ef0
commit 8f98bdd273
85 changed files with 712 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/compliance_import_service.py
+250 -107
View File
@@ -6,22 +6,256 @@ ComplianceControl, and ComplianceControlMapping records.
"""
import logging
import json
import re
from typing import Optional
import requests
from sqlalchemy.orm import Session
from app.models.compliance import (
ComplianceFramework,
ComplianceControl,
ComplianceControlMapping,
ComplianceFramework,
)
from app.models.technique import Technique
logger = logging.getLogger(__name__)
# ── Module-level control definitions (avoids N806 / uppercase-in-function) ────
_NIST_SAMPLE_CONTROLS = [
{
"control_id": "AC-2",
"title": "Account Management",
"category": "Access Control",
"techniques": ["T1078", "T1136", "T1098", "T1087", "T1069"],
},
{
"control_id": "AC-3",
"title": "Access Enforcement",
"category": "Access Control",
"techniques": ["T1078", "T1548", "T1134"],
},
{
"control_id": "AC-4",
"title": "Information Flow Enforcement",
"category": "Access Control",
"techniques": ["T1048", "T1041", "T1572"],
},
{
"control_id": "AC-6",
"title": "Least Privilege",
"category": "Access Control",
"techniques": ["T1078", "T1548", "T1134"],
},
{
"control_id": "AU-2",
"title": "Event Logging",
"category": "Audit and Accountability",
"techniques": ["T1562", "T1070"],
},
{
"control_id": "AU-6",
"title": "Audit Record Review",
"category": "Audit and Accountability",
"techniques": ["T1562", "T1070", "T1027"],
},
{
"control_id": "CA-7",
"title": "Continuous Monitoring",
"category": "Assessment, Authorization, and Monitoring",
"techniques": ["T1059", "T1053"],
},
{
"control_id": "CM-2",
"title": "Baseline Configuration",
"category": "Configuration Management",
"techniques": ["T1574", "T1546"],
},
{
"control_id": "CM-6",
"title": "Configuration Settings",
"category": "Configuration Management",
"techniques": ["T1574", "T1546", "T1112"],
},
{
"control_id": "CM-7",
"title": "Least Functionality",
"category": "Configuration Management",
"techniques": ["T1059", "T1218"],
},
{
"control_id": "IA-2",
"title": "Identification and Authentication",
"category": "Identification and Authentication",
"techniques": ["T1078", "T1110"],
},
{
"control_id": "IA-5",
"title": "Authenticator Management",
"category": "Identification and Authentication",
"techniques": ["T1078", "T1110", "T1003"],
},
{
"control_id": "IR-4",
"title": "Incident Handling",
"category": "Incident Response",
"techniques": ["T1059", "T1547"],
},
{
"control_id": "RA-5",
"title": "Vulnerability Monitoring and Scanning",
"category": "Risk Assessment",
"techniques": ["T1190", "T1203"],
},
{
"control_id": "SC-7",
"title": "Boundary Protection",
"category": "System and Communications Protection",
"techniques": ["T1048", "T1041", "T1071"],
},
{
"control_id": "SC-28",
"title": "Protection of Information at Rest",
"category": "System and Communications Protection",
"techniques": ["T1005", "T1114"],
},
{
"control_id": "SI-3",
"title": "Malicious Code Protection",
"category": "System and Information Integrity",
"techniques": ["T1059", "T1204", "T1566"],
},
{
"control_id": "SI-4",
"title": "System Monitoring",
"category": "System and Information Integrity",
"techniques": ["T1059", "T1053", "T1547"],
},
{
"control_id": "SI-7",
"title": "Software, Firmware, and Information Integrity",
"category": "System and Information Integrity",
"techniques": ["T1195", "T1553"],
},
{
"control_id": "PM-16",
"title": "Threat Awareness Program",
"category": "Program Management",
"techniques": ["T1566", "T1204"],
},
]
_CIS_CONTROLS = [
{
"control_id": "CIS-1",
"title": "Inventory and Control of Enterprise Assets",
"category": "IG1 — Basic",
"techniques": ["T1595", "T1590", "T1018", "T1082"],
},
{
"control_id": "CIS-2",
"title": "Inventory and Control of Software Assets",
"category": "IG1 — Basic",
"techniques": ["T1518", "T1072", "T1195"],
},
{
"control_id": "CIS-3",
"title": "Data Protection",
"category": "IG1 — Basic",
"techniques": ["T1005", "T1114", "T1560", "T1048", "T1041"],
},
{
"control_id": "CIS-4",
"title": "Secure Configuration of Enterprise Assets and Software",
"category": "IG1 — Basic",
"techniques": ["T1574", "T1546", "T1112", "T1543"],
},
{
"control_id": "CIS-5",
"title": "Account Management",
"category": "IG1 — Basic",
"techniques": ["T1078", "T1136", "T1098", "T1087"],
},
{
"control_id": "CIS-6",
"title": "Access Control Management",
"category": "IG1 — Basic",
"techniques": ["T1078", "T1548", "T1134", "T1021"],
},
{
"control_id": "CIS-7",
"title": "Continuous Vulnerability Management",
"category": "IG2 — Foundational",
"techniques": ["T1190", "T1203", "T1068", "T1210"],
},
{
"control_id": "CIS-8",
"title": "Audit Log Management",
"category": "IG2 — Foundational",
"techniques": ["T1562", "T1070", "T1059"],
},
{
"control_id": "CIS-9",
"title": "Email and Web Browser Protections",
"category": "IG2 — Foundational",
"techniques": ["T1566", "T1204", "T1189", "T1598"],
},
{
"control_id": "CIS-10",
"title": "Malware Defenses",
"category": "IG2 — Foundational",
"techniques": ["T1059", "T1204", "T1027", "T1140", "T1497"],
},
{
"control_id": "CIS-11",
"title": "Data Recovery",
"category": "IG1 — Basic",
"techniques": ["T1486", "T1490", "T1561"],
},
{
"control_id": "CIS-12",
"title": "Network Infrastructure Management",
"category": "IG2 — Foundational",
"techniques": ["T1557", "T1071", "T1572", "T1571"],
},
{
"control_id": "CIS-13",
"title": "Network Monitoring and Defense",
"category": "IG2 — Foundational",
"techniques": ["T1071", "T1048", "T1041", "T1105", "T1572"],
},
{
"control_id": "CIS-14",
"title": "Security Awareness and Skills Training",
"category": "IG1 — Basic",
"techniques": ["T1566", "T1204", "T1598"],
},
{
"control_id": "CIS-15",
"title": "Service Provider Management",
"category": "IG2 — Foundational",
"techniques": ["T1199", "T1195"],
},
{
"control_id": "CIS-16",
"title": "Application Software Security",
"category": "IG2 — Foundational",
"techniques": ["T1190", "T1059", "T1203"],
},
{
"control_id": "CIS-17",
"title": "Incident Response Management",
"category": "IG2 — Foundational",
"techniques": ["T1059", "T1547", "T1053"],
},
{
"control_id": "CIS-18",
"title": "Penetration Testing",
"category": "IG3 — Organizational",
"techniques": ["T1595", "T1046", "T1190", "T1059"],
},
]
# URL for the NIST 800-53 Rev 5 to ATT&CK mapping
# This is the JSON STIX bundle that contains the relationships
NIST_MAPPING_URL = (
@@ -53,7 +287,11 @@ def import_nist_800_53_mappings(db: Session) -> dict:
framework = ComplianceFramework(
name="NIST 800-53 Rev 5",
version="5",
description="National Institute of Standards and Technology Special Publication 800-53 Revision 5 — Security and Privacy Controls for Information Systems and Organizations",
description=(
"National Institute of Standards and Technology "
"Special Publication 800-53 Revision 5 — "
"Security and Privacy Controls for Information Systems and Organizations"
),
url="https://csrc.nist.gov/publications/detail/sp/800-53/rev-5/final",
is_active=True,
)
@@ -216,49 +454,6 @@ def _import_sample_nist_mappings(db: Session, framework: ComplianceFramework) ->
This ensures the feature works even without network access.
"""
SAMPLE_CONTROLS = [
{"control_id": "AC-2", "title": "Account Management", "category": "Access Control",
"techniques": ["T1078", "T1136", "T1098", "T1087", "T1069"]},
{"control_id": "AC-3", "title": "Access Enforcement", "category": "Access Control",
"techniques": ["T1078", "T1548", "T1134"]},
{"control_id": "AC-4", "title": "Information Flow Enforcement", "category": "Access Control",
"techniques": ["T1048", "T1041", "T1572"]},
{"control_id": "AC-6", "title": "Least Privilege", "category": "Access Control",
"techniques": ["T1078", "T1548", "T1134"]},
{"control_id": "AU-2", "title": "Event Logging", "category": "Audit and Accountability",
"techniques": ["T1562", "T1070"]},
{"control_id": "AU-6", "title": "Audit Record Review", "category": "Audit and Accountability",
"techniques": ["T1562", "T1070", "T1027"]},
{"control_id": "CA-7", "title": "Continuous Monitoring", "category": "Assessment, Authorization, and Monitoring",
"techniques": ["T1059", "T1053"]},
{"control_id": "CM-2", "title": "Baseline Configuration", "category": "Configuration Management",
"techniques": ["T1574", "T1546"]},
{"control_id": "CM-6", "title": "Configuration Settings", "category": "Configuration Management",
"techniques": ["T1574", "T1546", "T1112"]},
{"control_id": "CM-7", "title": "Least Functionality", "category": "Configuration Management",
"techniques": ["T1059", "T1218"]},
{"control_id": "IA-2", "title": "Identification and Authentication", "category": "Identification and Authentication",
"techniques": ["T1078", "T1110"]},
{"control_id": "IA-5", "title": "Authenticator Management", "category": "Identification and Authentication",
"techniques": ["T1078", "T1110", "T1003"]},
{"control_id": "IR-4", "title": "Incident Handling", "category": "Incident Response",
"techniques": ["T1059", "T1547"]},
{"control_id": "RA-5", "title": "Vulnerability Monitoring and Scanning", "category": "Risk Assessment",
"techniques": ["T1190", "T1203"]},
{"control_id": "SC-7", "title": "Boundary Protection", "category": "System and Communications Protection",
"techniques": ["T1048", "T1041", "T1071"]},
{"control_id": "SC-28", "title": "Protection of Information at Rest", "category": "System and Communications Protection",
"techniques": ["T1005", "T1114"]},
{"control_id": "SI-3", "title": "Malicious Code Protection", "category": "System and Information Integrity",
"techniques": ["T1059", "T1204", "T1566"]},
{"control_id": "SI-4", "title": "System Monitoring", "category": "System and Information Integrity",
"techniques": ["T1059", "T1053", "T1547"]},
{"control_id": "SI-7", "title": "Software, Firmware, and Information Integrity", "category": "System and Information Integrity",
"techniques": ["T1195", "T1553"]},
{"control_id": "PM-16", "title": "Threat Awareness Program", "category": "Program Management",
"techniques": ["T1566", "T1204"]},
]
# Build technique lookup
all_techniques = {t.mitre_id: t for t in db.query(Technique).all()}
@@ -276,7 +471,7 @@ def _import_sample_nist_mappings(db: Session, framework: ComplianceFramework) ->
controls_created = 0
mappings_created = 0
for sample in SAMPLE_CONTROLS:
for sample in _NIST_SAMPLE_CONTROLS:
# Create or get control
if sample["control_id"] in existing_controls:
control = existing_controls[sample["control_id"]]
@@ -348,8 +543,11 @@ def import_cis_controls_v8_mappings(db: Session) -> dict:
framework = ComplianceFramework(
name="CIS Controls v8",
version="8",
description="Center for Internet Security Critical Security Controls Version 8 — "
"a prioritized set of 18 security safeguards organized by Implementation Groups (IG1, IG2, IG3).",
description=(
"Center for Internet Security Critical Security Controls Version 8 — "
"a prioritized set of 18 security safeguards "
"organized by Implementation Groups (IG1, IG2, IG3)."
),
url="https://www.cisecurity.org/controls/v8",
is_active=True,
)
@@ -360,62 +558,7 @@ def import_cis_controls_v8_mappings(db: Session) -> dict:
logger.info("CIS Controls v8 framework already exists")
# ── 2. Control definitions with ATT&CK mappings ───────────────
CIS_CONTROLS = [
{"control_id": "CIS-1", "title": "Inventory and Control of Enterprise Assets",
"category": "IG1 — Basic",
"techniques": ["T1595", "T1590", "T1018", "T1082"]},
{"control_id": "CIS-2", "title": "Inventory and Control of Software Assets",
"category": "IG1 — Basic",
"techniques": ["T1518", "T1072", "T1195"]},
{"control_id": "CIS-3", "title": "Data Protection",
"category": "IG1 — Basic",
"techniques": ["T1005", "T1114", "T1560", "T1048", "T1041"]},
{"control_id": "CIS-4", "title": "Secure Configuration of Enterprise Assets and Software",
"category": "IG1 — Basic",
"techniques": ["T1574", "T1546", "T1112", "T1543"]},
{"control_id": "CIS-5", "title": "Account Management",
"category": "IG1 — Basic",
"techniques": ["T1078", "T1136", "T1098", "T1087"]},
{"control_id": "CIS-6", "title": "Access Control Management",
"category": "IG1 — Basic",
"techniques": ["T1078", "T1548", "T1134", "T1021"]},
{"control_id": "CIS-7", "title": "Continuous Vulnerability Management",
"category": "IG2 — Foundational",
"techniques": ["T1190", "T1203", "T1068", "T1210"]},
{"control_id": "CIS-8", "title": "Audit Log Management",
"category": "IG2 — Foundational",
"techniques": ["T1562", "T1070", "T1059"]},
{"control_id": "CIS-9", "title": "Email and Web Browser Protections",
"category": "IG2 — Foundational",
"techniques": ["T1566", "T1204", "T1189", "T1598"]},
{"control_id": "CIS-10", "title": "Malware Defenses",
"category": "IG2 — Foundational",
"techniques": ["T1059", "T1204", "T1027", "T1140", "T1497"]},
{"control_id": "CIS-11", "title": "Data Recovery",
"category": "IG1 — Basic",
"techniques": ["T1486", "T1490", "T1561"]},
{"control_id": "CIS-12", "title": "Network Infrastructure Management",
"category": "IG2 — Foundational",
"techniques": ["T1557", "T1071", "T1572", "T1571"]},
{"control_id": "CIS-13", "title": "Network Monitoring and Defense",
"category": "IG2 — Foundational",
"techniques": ["T1071", "T1048", "T1041", "T1105", "T1572"]},
{"control_id": "CIS-14", "title": "Security Awareness and Skills Training",
"category": "IG1 — Basic",
"techniques": ["T1566", "T1204", "T1598"]},
{"control_id": "CIS-15", "title": "Service Provider Management",
"category": "IG2 — Foundational",
"techniques": ["T1199", "T1195"]},
{"control_id": "CIS-16", "title": "Application Software Security",
"category": "IG2 — Foundational",
"techniques": ["T1190", "T1059", "T1203"]},
{"control_id": "CIS-17", "title": "Incident Response Management",
"category": "IG2 — Foundational",
"techniques": ["T1059", "T1547", "T1053"]},
{"control_id": "CIS-18", "title": "Penetration Testing",
"category": "IG3 — Organizational",
"techniques": ["T1595", "T1046", "T1190", "T1059"]},
]
# (defined at module level as _CIS_CONTROLS)
# Build technique lookup
all_techniques = {t.mitre_id: t for t in db.query(Technique).all()}
@@ -439,7 +582,7 @@ def import_cis_controls_v8_mappings(db: Session) -> dict:
controls_created = 0
mappings_created = 0
for item in CIS_CONTROLS:
for item in _CIS_CONTROLS:
if item["control_id"] in existing_controls:
control = existing_controls[item["control_id"]]
else: