feat(intel): major intel scan improvements + Review Queue integration
Some checks failed
Aegis CI / lint-and-test (push) Has been cancelled
Some checks failed
Aegis CI / lint-and-test (push) Has been cancelled
Backend:
- intel_service: remove 50-technique limit (scan all techniques), improve
pattern matching with word boundaries (\bT1059\b), raise min name length
to 8 chars to reduce false positives, skip entries with empty titles
- technique_query_service: add intel_items to get_technique_detail() so
the technique page now shows recent threat intel articles (last 20)
- New GET /intel/items endpoint with optional technique_id filter
Frontend:
- New api/intel.ts with listIntelItems()
- ReviewQueuePage: complete redesign
* Expandable rows — click a technique to see its intel articles inline
* IntelPanel component fetches articles per technique on expand
* 'Create Template from Intel' button opens pre-filled modal:
name (from article title), source_url (article link), technique_id
User reads the article and fills the attack procedure
* Updated explanation text: lists all 3 reasons a technique can be flagged
(MITRE update / intel scan / new template or detection rule)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -39,6 +39,7 @@ from app.routers import advanced_metrics as advanced_metrics_router
|
||||
from app.routers import osint as osint_router
|
||||
from app.routers import webhooks as webhooks_router
|
||||
from app.routers import detection_lifecycle as detection_lifecycle_router
|
||||
from app.routers import intel as intel_router
|
||||
from app.routers import ownership as ownership_router
|
||||
from app.routers import attack_paths as attack_paths_router
|
||||
from app.routers import knowledge as knowledge_router
|
||||
@@ -145,6 +146,7 @@ app.include_router(heatmap_router.router, prefix="/api/v1")
|
||||
app.include_router(scores_router.router, prefix="/api/v1")
|
||||
app.include_router(operational_metrics_router.router, prefix="/api/v1")
|
||||
app.include_router(compliance_router.router, prefix="/api/v1")
|
||||
app.include_router(intel_router.router, prefix="/api/v1")
|
||||
app.include_router(snapshots_router.router, prefix="/api/v1")
|
||||
app.include_router(jira_router.router, prefix="/api/v1")
|
||||
app.include_router(worklogs_router.router, prefix="/api/v1")
|
||||
|
||||
54
backend/app/routers/intel.py
Normal file
54
backend/app/routers/intel.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""Intel items endpoints — list and manage threat intelligence items."""
|
||||
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.dependencies.auth import get_current_user
|
||||
from app.models.intel import IntelItem
|
||||
from app.models.user import User
|
||||
|
||||
router = APIRouter(prefix="/intel", tags=["intel"])
|
||||
|
||||
|
||||
class IntelItemOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
technique_id: Optional[uuid.UUID] = None
|
||||
url: str
|
||||
title: Optional[str] = None
|
||||
source: Optional[str] = None
|
||||
detected_at: Optional[str] = None
|
||||
reviewed: bool
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
@router.get("/items", response_model=list[IntelItemOut])
|
||||
def list_intel_items(
|
||||
technique_id: Optional[uuid.UUID] = Query(None, description="Filter by technique"),
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
db: Session = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""List threat intelligence items, optionally filtered by technique."""
|
||||
query = db.query(IntelItem).order_by(IntelItem.detected_at.desc())
|
||||
if technique_id:
|
||||
query = query.filter(IntelItem.technique_id == technique_id)
|
||||
items = query.limit(limit).all()
|
||||
return [
|
||||
IntelItemOut(
|
||||
id=item.id,
|
||||
technique_id=item.technique_id,
|
||||
url=item.url,
|
||||
title=item.title,
|
||||
source=item.source,
|
||||
detected_at=item.detected_at.isoformat() if item.detected_at else None,
|
||||
reviewed=item.reviewed,
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
@@ -57,8 +57,9 @@ RSS_FEEDS: list[dict[str, str]] = [
|
||||
# Timeout for each feed request (seconds)
|
||||
_FEED_TIMEOUT = 15
|
||||
|
||||
# Maximum number of techniques to scan (to keep MVP fast)
|
||||
_MAX_TECHNIQUES = 50
|
||||
# Minimum technique name length for name-based matching
|
||||
# Short names ("Kill", "BITS") produce too many false positives
|
||||
_MIN_NAME_LEN = 8
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -118,19 +119,36 @@ def _fetch_feed(url: str) -> list[dict[str, str]]:
|
||||
return entries
|
||||
|
||||
|
||||
def _build_patterns(technique: Technique) -> list[re.Pattern]:
|
||||
"""Build regex patterns to search feed content for a given technique."""
|
||||
patterns: list[re.Pattern] = []
|
||||
def _build_patterns(technique: Technique) -> tuple[list[re.Pattern], list[re.Pattern]]:
|
||||
"""Build regex patterns for a technique.
|
||||
|
||||
mitre_id = re.escape(technique.mitre_id)
|
||||
patterns.append(re.compile(mitre_id, re.IGNORECASE))
|
||||
Returns two lists:
|
||||
- ``id_patterns``: MITRE ID patterns (high confidence, word-boundary matched)
|
||||
- ``name_patterns``: technique name patterns (lower confidence, long names only)
|
||||
"""
|
||||
id_patterns: list[re.Pattern] = []
|
||||
name_patterns: list[re.Pattern] = []
|
||||
|
||||
# Technique name — match if the full name appears
|
||||
if technique.name and len(technique.name) > 4:
|
||||
# MITRE ID with word boundaries so T1059 doesn't partially match T1059.001
|
||||
mitre_id_escaped = re.escape(technique.mitre_id)
|
||||
id_patterns.append(re.compile(rf"\b{mitre_id_escaped}\b", re.IGNORECASE))
|
||||
|
||||
# Technique name — only for distinctly long names to reduce false positives
|
||||
if technique.name and len(technique.name) >= _MIN_NAME_LEN:
|
||||
name_escaped = re.escape(technique.name)
|
||||
patterns.append(re.compile(name_escaped, re.IGNORECASE))
|
||||
name_patterns.append(re.compile(rf"\b{name_escaped}\b", re.IGNORECASE))
|
||||
|
||||
return patterns
|
||||
return id_patterns, name_patterns
|
||||
|
||||
|
||||
def _entry_matches(
|
||||
entry: dict[str, str],
|
||||
id_patterns: list[re.Pattern],
|
||||
name_patterns: list[re.Pattern],
|
||||
) -> bool:
|
||||
"""Return True if any pattern matches the entry's title or description."""
|
||||
text = f"{entry.get('title', '')} {entry.get('description', '')}"
|
||||
return any(p.search(text) for p in id_patterns + name_patterns)
|
||||
|
||||
|
||||
def _entry_matches(entry: dict[str, str], patterns: list[re.Pattern]) -> bool:
|
||||
@@ -160,11 +178,10 @@ def scan_intel(db: Session) -> dict:
|
||||
"""
|
||||
logger.info("Intel scan starting...")
|
||||
|
||||
# 1. Load techniques (limit for MVP speed)
|
||||
# 1. Load all active techniques
|
||||
techniques = (
|
||||
db.query(Technique)
|
||||
.order_by(Technique.mitre_id)
|
||||
.limit(_MAX_TECHNIQUES)
|
||||
.all()
|
||||
)
|
||||
logger.info("Scanning %d techniques against %d feeds", len(techniques), len(RSS_FEEDS))
|
||||
@@ -192,10 +209,14 @@ def scan_intel(db: Session) -> dict:
|
||||
techniques_flagged: set[str] = set()
|
||||
|
||||
for technique in techniques:
|
||||
patterns = _build_patterns(technique)
|
||||
id_patterns, name_patterns = _build_patterns(technique)
|
||||
|
||||
for feed_name, entry in all_entries:
|
||||
if not _entry_matches(entry, patterns):
|
||||
if not _entry_matches(entry, id_patterns, name_patterns):
|
||||
continue
|
||||
|
||||
# Skip entries with no title (low-quality)
|
||||
if not entry.get("title", "").strip():
|
||||
continue
|
||||
|
||||
url = entry.get("link", "").strip()
|
||||
|
||||
@@ -6,6 +6,7 @@ from sqlalchemy.orm import Session, joinedload
|
||||
from app.domain.errors import EntityNotFoundError
|
||||
from app.models.technique import Technique
|
||||
from app.models.detection_rule import DetectionRule
|
||||
from app.models.intel import IntelItem
|
||||
from app.services.d3fend_import_service import get_defenses_for_technique
|
||||
|
||||
# Severity sort order for detection rules (most critical first)
|
||||
@@ -25,6 +26,15 @@ def get_technique_detail(db: Session, mitre_id: str) -> dict:
|
||||
|
||||
defenses = get_defenses_for_technique(db, technique.id)
|
||||
|
||||
# Recent intel items for this technique (newest 20)
|
||||
intel_items = (
|
||||
db.query(IntelItem)
|
||||
.filter(IntelItem.technique_id == technique.id)
|
||||
.order_by(IntelItem.detected_at.desc())
|
||||
.limit(20)
|
||||
.all()
|
||||
)
|
||||
|
||||
detection_rules = (
|
||||
db.query(DetectionRule)
|
||||
.filter(
|
||||
@@ -79,4 +89,15 @@ def get_technique_detail(db: Session, mitre_id: str) -> dict:
|
||||
for r in detection_rules
|
||||
],
|
||||
"d3fend_defenses": defenses,
|
||||
"intel_items": [
|
||||
{
|
||||
"id": str(item.id),
|
||||
"url": item.url,
|
||||
"title": item.title,
|
||||
"source": item.source,
|
||||
"detected_at": item.detected_at.isoformat() if item.detected_at else None,
|
||||
"reviewed": item.reviewed,
|
||||
}
|
||||
for item in intel_items
|
||||
],
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user