From b39a4fec14cad621c02ea4c70a3c9f13e0dd593a Mon Sep 17 00:00:00 2001 From: kitos Date: Fri, 29 May 2026 16:04:30 +0200 Subject: [PATCH] feat(intel): major intel scan improvements + Review Queue integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend: - intel_service: remove 50-technique limit (scan all techniques), improve pattern matching with word boundaries (\bT1059\b), raise min name length to 8 chars to reduce false positives, skip entries with empty titles - technique_query_service: add intel_items to get_technique_detail() so the technique page now shows recent threat intel articles (last 20) - New GET /intel/items endpoint with optional technique_id filter Frontend: - New api/intel.ts with listIntelItems() - ReviewQueuePage: complete redesign * Expandable rows — click a technique to see its intel articles inline * IntelPanel component fetches articles per technique on expand * 'Create Template from Intel' button opens pre-filled modal: name (from article title), source_url (article link), technique_id User reads the article and fills the attack procedure * Updated explanation text: lists all 3 reasons a technique can be flagged (MITRE update / intel scan / new template or detection rule) Co-Authored-By: Claude Sonnet 4.6 --- backend/app/main.py | 2 + backend/app/routers/intel.py | 54 ++ backend/app/services/intel_service.py | 51 +- .../app/services/technique_query_service.py | 21 + frontend/src/api/intel.ts | 22 + frontend/src/pages/ReviewQueuePage.tsx | 463 +++++++++++++++--- 6 files changed, 519 insertions(+), 94 deletions(-) create mode 100644 backend/app/routers/intel.py create mode 100644 frontend/src/api/intel.ts diff --git a/backend/app/main.py b/backend/app/main.py index 00bf549..81db6e2 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -39,6 +39,7 @@ from app.routers import advanced_metrics as advanced_metrics_router from app.routers import osint as osint_router from app.routers import webhooks as webhooks_router from app.routers import detection_lifecycle as detection_lifecycle_router +from app.routers import intel as intel_router from app.routers import ownership as ownership_router from app.routers import attack_paths as attack_paths_router from app.routers import knowledge as knowledge_router @@ -145,6 +146,7 @@ app.include_router(heatmap_router.router, prefix="/api/v1") app.include_router(scores_router.router, prefix="/api/v1") app.include_router(operational_metrics_router.router, prefix="/api/v1") app.include_router(compliance_router.router, prefix="/api/v1") +app.include_router(intel_router.router, prefix="/api/v1") app.include_router(snapshots_router.router, prefix="/api/v1") app.include_router(jira_router.router, prefix="/api/v1") app.include_router(worklogs_router.router, prefix="/api/v1") diff --git a/backend/app/routers/intel.py b/backend/app/routers/intel.py new file mode 100644 index 0000000..0c95642 --- /dev/null +++ b/backend/app/routers/intel.py @@ -0,0 +1,54 @@ +"""Intel items endpoints — list and manage threat intelligence items.""" + +import uuid +from typing import Optional + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from app.database import get_db +from app.dependencies.auth import get_current_user +from app.models.intel import IntelItem +from app.models.user import User + +router = APIRouter(prefix="/intel", tags=["intel"]) + + +class IntelItemOut(BaseModel): + id: uuid.UUID + technique_id: Optional[uuid.UUID] = None + url: str + title: Optional[str] = None + source: Optional[str] = None + detected_at: Optional[str] = None + reviewed: bool + + class Config: + from_attributes = True + + +@router.get("/items", response_model=list[IntelItemOut]) +def list_intel_items( + technique_id: Optional[uuid.UUID] = Query(None, description="Filter by technique"), + limit: int = Query(50, ge=1, le=200), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """List threat intelligence items, optionally filtered by technique.""" + query = db.query(IntelItem).order_by(IntelItem.detected_at.desc()) + if technique_id: + query = query.filter(IntelItem.technique_id == technique_id) + items = query.limit(limit).all() + return [ + IntelItemOut( + id=item.id, + technique_id=item.technique_id, + url=item.url, + title=item.title, + source=item.source, + detected_at=item.detected_at.isoformat() if item.detected_at else None, + reviewed=item.reviewed, + ) + for item in items + ] diff --git a/backend/app/services/intel_service.py b/backend/app/services/intel_service.py index c456885..5ff6323 100644 --- a/backend/app/services/intel_service.py +++ b/backend/app/services/intel_service.py @@ -57,8 +57,9 @@ RSS_FEEDS: list[dict[str, str]] = [ # Timeout for each feed request (seconds) _FEED_TIMEOUT = 15 -# Maximum number of techniques to scan (to keep MVP fast) -_MAX_TECHNIQUES = 50 +# Minimum technique name length for name-based matching +# Short names ("Kill", "BITS") produce too many false positives +_MIN_NAME_LEN = 8 # --------------------------------------------------------------------------- @@ -118,19 +119,36 @@ def _fetch_feed(url: str) -> list[dict[str, str]]: return entries -def _build_patterns(technique: Technique) -> list[re.Pattern]: - """Build regex patterns to search feed content for a given technique.""" - patterns: list[re.Pattern] = [] +def _build_patterns(technique: Technique) -> tuple[list[re.Pattern], list[re.Pattern]]: + """Build regex patterns for a technique. - mitre_id = re.escape(technique.mitre_id) - patterns.append(re.compile(mitre_id, re.IGNORECASE)) + Returns two lists: + - ``id_patterns``: MITRE ID patterns (high confidence, word-boundary matched) + - ``name_patterns``: technique name patterns (lower confidence, long names only) + """ + id_patterns: list[re.Pattern] = [] + name_patterns: list[re.Pattern] = [] - # Technique name — match if the full name appears - if technique.name and len(technique.name) > 4: + # MITRE ID with word boundaries so T1059 doesn't partially match T1059.001 + mitre_id_escaped = re.escape(technique.mitre_id) + id_patterns.append(re.compile(rf"\b{mitre_id_escaped}\b", re.IGNORECASE)) + + # Technique name — only for distinctly long names to reduce false positives + if technique.name and len(technique.name) >= _MIN_NAME_LEN: name_escaped = re.escape(technique.name) - patterns.append(re.compile(name_escaped, re.IGNORECASE)) + name_patterns.append(re.compile(rf"\b{name_escaped}\b", re.IGNORECASE)) - return patterns + return id_patterns, name_patterns + + +def _entry_matches( + entry: dict[str, str], + id_patterns: list[re.Pattern], + name_patterns: list[re.Pattern], +) -> bool: + """Return True if any pattern matches the entry's title or description.""" + text = f"{entry.get('title', '')} {entry.get('description', '')}" + return any(p.search(text) for p in id_patterns + name_patterns) def _entry_matches(entry: dict[str, str], patterns: list[re.Pattern]) -> bool: @@ -160,11 +178,10 @@ def scan_intel(db: Session) -> dict: """ logger.info("Intel scan starting...") - # 1. Load techniques (limit for MVP speed) + # 1. Load all active techniques techniques = ( db.query(Technique) .order_by(Technique.mitre_id) - .limit(_MAX_TECHNIQUES) .all() ) logger.info("Scanning %d techniques against %d feeds", len(techniques), len(RSS_FEEDS)) @@ -192,10 +209,14 @@ def scan_intel(db: Session) -> dict: techniques_flagged: set[str] = set() for technique in techniques: - patterns = _build_patterns(technique) + id_patterns, name_patterns = _build_patterns(technique) for feed_name, entry in all_entries: - if not _entry_matches(entry, patterns): + if not _entry_matches(entry, id_patterns, name_patterns): + continue + + # Skip entries with no title (low-quality) + if not entry.get("title", "").strip(): continue url = entry.get("link", "").strip() diff --git a/backend/app/services/technique_query_service.py b/backend/app/services/technique_query_service.py index ea5cebf..43b7777 100644 --- a/backend/app/services/technique_query_service.py +++ b/backend/app/services/technique_query_service.py @@ -6,6 +6,7 @@ from sqlalchemy.orm import Session, joinedload from app.domain.errors import EntityNotFoundError from app.models.technique import Technique from app.models.detection_rule import DetectionRule +from app.models.intel import IntelItem from app.services.d3fend_import_service import get_defenses_for_technique # Severity sort order for detection rules (most critical first) @@ -25,6 +26,15 @@ def get_technique_detail(db: Session, mitre_id: str) -> dict: defenses = get_defenses_for_technique(db, technique.id) + # Recent intel items for this technique (newest 20) + intel_items = ( + db.query(IntelItem) + .filter(IntelItem.technique_id == technique.id) + .order_by(IntelItem.detected_at.desc()) + .limit(20) + .all() + ) + detection_rules = ( db.query(DetectionRule) .filter( @@ -79,4 +89,15 @@ def get_technique_detail(db: Session, mitre_id: str) -> dict: for r in detection_rules ], "d3fend_defenses": defenses, + "intel_items": [ + { + "id": str(item.id), + "url": item.url, + "title": item.title, + "source": item.source, + "detected_at": item.detected_at.isoformat() if item.detected_at else None, + "reviewed": item.reviewed, + } + for item in intel_items + ], } diff --git a/frontend/src/api/intel.ts b/frontend/src/api/intel.ts new file mode 100644 index 0000000..fa4449c --- /dev/null +++ b/frontend/src/api/intel.ts @@ -0,0 +1,22 @@ +import client from "./client"; + +export interface IntelItem { + id: string; + technique_id: string | null; + url: string; + title: string | null; + source: string | null; + detected_at: string | null; + reviewed: boolean; +} + +/** Fetch intel items, optionally filtered by technique UUID. */ +export async function listIntelItems( + techniqueId?: string, + limit = 50, +): Promise { + const params: Record = { limit }; + if (techniqueId) params.technique_id = techniqueId; + const { data } = await client.get("/intel/items", { params }); + return data; +} diff --git a/frontend/src/pages/ReviewQueuePage.tsx b/frontend/src/pages/ReviewQueuePage.tsx index 1c793a2..37b640f 100644 --- a/frontend/src/pages/ReviewQueuePage.tsx +++ b/frontend/src/pages/ReviewQueuePage.tsx @@ -1,4 +1,4 @@ -import { useMemo } from "react"; +import { useState, useMemo } from "react"; import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; import { useNavigate } from "react-router-dom"; import { @@ -8,18 +8,27 @@ import { CheckCircle, ExternalLink, RefreshCw, + ChevronDown, + ChevronUp, + Rss, + BookOpen, + X, + Globe, } from "lucide-react"; import { getTechniques, markTechniqueReviewed } from "../api/techniques"; +import { listIntelItems, type IntelItem } from "../api/intel"; +import { createTemplate } from "../api/test-templates"; import type { TechniqueSummary } from "../api/techniques"; import { useAuth } from "../context/AuthContext"; +/* ── helpers ─────────────────────────────────────────────────────── */ + const STATUS_COLORS: Record = { - validated: "bg-green-500/10 text-green-400 border-green-500/20", - partial: "bg-yellow-500/10 text-yellow-400 border-yellow-500/20", - in_progress: "bg-blue-500/10 text-blue-400 border-blue-500/20", - not_covered: "bg-red-500/10 text-red-400 border-red-500/20", - not_evaluated: "bg-gray-500/10 text-gray-400 border-gray-600/20", - review_required:"bg-orange-500/10 text-orange-400 border-orange-500/20", + validated: "bg-green-500/10 text-green-400 border-green-500/20", + partial: "bg-yellow-500/10 text-yellow-400 border-yellow-500/20", + in_progress: "bg-blue-500/10 text-blue-400 border-blue-500/20", + not_covered: "bg-red-500/10 text-red-400 border-red-500/20", + not_evaluated: "bg-gray-500/10 text-gray-400 border-gray-600/20", }; const STATUS_LABELS: Record = { @@ -38,6 +47,287 @@ function formatDate(dateStr: string | null | undefined) { }); } +/* ── Create-template-from-intel modal ─────────────────────────────── */ + +function IntelTemplateModal({ + intel, + techniqueMitreId, + techniqueName, + onClose, + onSaved, +}: { + intel: IntelItem; + techniqueMitreId: string; + techniqueName: string; + onClose: () => void; + onSaved: () => void; +}) { + const [name, setName] = useState(intel.title ?? `Test for ${techniqueMitreId}`); + const [description, setDescription] = useState( + intel.title ? `Based on: ${intel.title}` : "" + ); + const [platform, setPlatform] = useState(""); + const [attackProcedure, setAttackProcedure] = useState(""); + const [expectedDetection, setExpectedDetection] = useState(""); + const [severity, setSeverity] = useState("medium"); + + const saveMutation = useMutation({ + mutationFn: () => + createTemplate({ + mitre_technique_id: techniqueMitreId, + name: name.trim(), + description: description.trim() || undefined, + platform: platform.trim() || undefined, + attack_procedure: attackProcedure.trim() || undefined, + expected_detection: expectedDetection.trim() || undefined, + source_url: intel.url, + severity, + source: "custom", + }), + onSuccess: () => onSaved(), + }); + + return ( +
+
+ {/* Header */} +
+
+ +

Create Template from Intel

+
+ +
+ +
+ {/* Source article banner */} +
+ +
+

+ Source: {intel.source ?? "Unknown"} · {formatDate(intel.detected_at)} +

+

{intel.title}

+
+ + + Open article + +
+ + {/* Technique banner */} +
+ Technique: {techniqueMitreId} + {techniqueName} +
+ + {/* Name */} +
+ + setName(e.target.value)} + className="w-full rounded-lg border border-gray-700 bg-gray-800 px-3 py-2 text-sm text-gray-200 placeholder-gray-500 focus:border-cyan-500 focus:outline-none" + /> +
+ + {/* Platform + Severity */} +
+
+ + setPlatform(e.target.value)} + placeholder="windows, linux, macos…" + className="w-full rounded-lg border border-gray-700 bg-gray-800 px-3 py-2 text-sm text-gray-200 placeholder-gray-500 focus:border-cyan-500 focus:outline-none" + /> +
+
+ + +
+
+ + {/* Description */} +
+ +