Compare commits

..

5 Commits

Author SHA1 Message Date
kitos 4692a8b93e feat(tests): add Detect Procedure UI, template suggested-procedure editing, lead review panel
Aegis CI / lint-and-test (push) Has been cancelled
Snyk Security Scan / Python vulnerabilities (backend) (push) Has been cancelled
Snyk Security Scan / npm vulnerabilities (frontend) (push) Has been cancelled
Snyk Security Scan / Docker image vulnerabilities (backend) (push) Has been cancelled
Blue Team gets a Detect Procedure textarea in the test detail view,
mirroring Red's Attack Procedure field. Template create/edit forms
gain a Detect Suggested Procedure field for leads to curate directly.
Leads also get a Procedure Suggestions panel on the Tests page,
scoped to their own team by the backend, to approve or reject
suggestions extracted from submitted rounds.
2026-07-14 15:30:47 +02:00
kitos 7ec3c5bc8b feat(tests): auto-propose procedure suggestions from submitted rounds, add lead review workflow
When Red or Blue submits a round for a test created from a template,
any extractable command in their procedure text is proposed as an
update to the template's suggested procedure — never written
automatically. Adds GET/approve/reject endpoints scoped so a lead only
reviews their own team's suggestions; approving writes the suggested
text into the template, rejecting leaves it untouched.
2026-07-14 15:22:03 +02:00
kitos 8bdbe48fbe feat(tests): add Blue detect_procedure field mapped from template suggestion
Blue Team gets a detect_procedure field on Test (what they actually
did to detect the attack), Blue's counterpart to Red's procedure_text.
It's seeded from a new detect_suggested_procedure field on
TestTemplate at test-creation time, so a junior who later reuses the
same template starts with prior guidance instead of a blank field.

Also adds the procedure_suggestions review table and Test.source_template_id,
laying the groundwork for suggesting template improvements from filled-in
procedure fields (reviewed and approved by a lead, never auto-written).
detect_procedure is archived (not cleared) on Blue reopen, matching
blue_summary, and now appears in the Jira round-archived and blue_review
comments alongside the existing detection/containment fields.
2026-07-14 14:58:29 +02:00
kitos fd94e55799 feat(tests): add regex-based command extraction heuristic
Extracts the actual command(s)/query out of a free-text procedure
field, discarding narrative and pasted output, for use in building
procedure-improvement suggestions from operator submissions.
2026-07-14 13:36:25 +02:00
kitos f87959369b fix(tests): remove blind visibility, both teams always see each other's fields
Red and Blue could not see each other's real field data until both
reviews passed, showing a placeholder message instead. This kept
detection testing blind to what Red actually did; both sides now see
the full, live test data at all times.
2026-07-14 13:36:19 +02:00
28 changed files with 1053 additions and 103 deletions
@@ -0,0 +1,58 @@
"""Add detect_procedure fields and procedure_suggestions review table.
Blue Team gets a "Detect Procedure" field (what they actually did to
detect the attack) mirroring Red's procedure_text, plus a matching
"suggested" field on test_templates. Commands extracted from either
side's procedure text are proposed as template improvements via a new
procedure_suggestions table, reviewed and approved/rejected by a lead
rather than written automatically.
Revision ID: b062
Revises: b061
Create Date: 2026-07-14
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision = "b062"
down_revision = "b061"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("tests", sa.Column("detect_procedure", sa.Text(), nullable=True))
op.add_column(
"tests",
sa.Column("source_template_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("test_templates.id"), nullable=True),
)
op.add_column("test_templates", sa.Column("detect_suggested_procedure", sa.Text(), nullable=True))
op.add_column("test_round_history", sa.Column("detect_procedure", sa.Text(), nullable=True))
op.create_table(
"procedure_suggestions",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("template_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("test_templates.id"), nullable=False),
sa.Column("team", sa.String(10), nullable=False),
sa.Column("suggested_text", sa.Text(), nullable=False),
sa.Column("source_test_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("tests.id"), nullable=True),
sa.Column("submitted_by", postgresql.UUID(as_uuid=True), sa.ForeignKey("users.id"), nullable=True),
sa.Column("status", sa.String(10), nullable=False, server_default="pending"),
sa.Column("reviewed_by", postgresql.UUID(as_uuid=True), sa.ForeignKey("users.id"), nullable=True),
sa.Column("reviewed_at", sa.DateTime(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
)
op.create_index("ix_procedure_suggestions_template_id", "procedure_suggestions", ["template_id"])
op.create_index("ix_procedure_suggestions_status", "procedure_suggestions", ["status"])
def downgrade() -> None:
op.drop_index("ix_procedure_suggestions_status", table_name="procedure_suggestions")
op.drop_index("ix_procedure_suggestions_template_id", table_name="procedure_suggestions")
op.drop_table("procedure_suggestions")
op.drop_column("test_round_history", "detect_procedure")
op.drop_column("test_templates", "detect_suggested_procedure")
op.drop_column("tests", "source_template_id")
op.drop_column("tests", "detect_procedure")
+2
View File
@@ -43,6 +43,7 @@ from app.routers import techniques as techniques_router
from app.routers import tests as tests_router from app.routers import tests as tests_router
from app.routers import evidence as evidence_router from app.routers import evidence as evidence_router
from app.routers import test_templates as test_templates_router from app.routers import test_templates as test_templates_router
from app.routers import procedure_suggestions as procedure_suggestions_router
from app.routers import system as system_router from app.routers import system as system_router
from app.routers import metrics as metrics_router from app.routers import metrics as metrics_router
from app.routers import users as users_router from app.routers import users as users_router
@@ -213,6 +214,7 @@ app.include_router(tests_router.router, prefix="/api/v1")
app.include_router(evidence_router.router, prefix="/api/v1") app.include_router(evidence_router.router, prefix="/api/v1")
# Call app.include_router() # Call app.include_router()
app.include_router(test_templates_router.router, prefix="/api/v1") app.include_router(test_templates_router.router, prefix="/api/v1")
app.include_router(procedure_suggestions_router.router, prefix="/api/v1")
# Call app.include_router() # Call app.include_router()
app.include_router(system_router.router, prefix="/api/v1") app.include_router(system_router.router, prefix="/api/v1")
# Call app.include_router() # Call app.include_router()
+2
View File
@@ -45,6 +45,7 @@ from app.models.technique import Technique
from app.models.test import Test from app.models.test import Test
from app.models.test_round_history import TestRoundHistory from app.models.test_round_history import TestRoundHistory
from app.models.test_template import TestTemplate from app.models.test_template import TestTemplate
from app.models.procedure_suggestion import ProcedureSuggestion
from app.models.user import User from app.models.user import User
# Assign __all__ = [ # Assign __all__ = [
@@ -88,4 +89,5 @@ __all__ = [
"AlertRule", "AlertRule",
"AlertInstance", "AlertInstance",
"TestRoundHistory", "TestRoundHistory",
"ProcedureSuggestion",
] ]
@@ -0,0 +1,41 @@
"""SQLAlchemy model for procedure-improvement suggestions.
When an operator submits a round with a filled-in procedure field
(``procedure_text`` for Red, ``detect_procedure`` for Blue), the command(s)
in it are extracted heuristically and proposed as an update to the
originating template's suggested-procedure field. Nothing is written to
the template automatically — a lead reviews and approves or rejects each
suggestion, so a junior who later picks up the same template only ever
sees vetted guidance.
"""
import uuid
from sqlalchemy import Column, DateTime, ForeignKey, String, Text, func
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import relationship
from app.database import Base
class ProcedureSuggestion(Base):
"""A proposed update to a TestTemplate's suggested procedure, pending lead review."""
__tablename__ = "procedure_suggestions"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
template_id = Column(UUID(as_uuid=True), ForeignKey("test_templates.id"), nullable=False)
team = Column(String(10), nullable=False) # "red" or "blue"
suggested_text = Column(Text, nullable=False)
source_test_id = Column(UUID(as_uuid=True), ForeignKey("tests.id"), nullable=True)
submitted_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
status = Column(String(10), nullable=False, default="pending", server_default="pending") # pending/approved/rejected
reviewed_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
reviewed_at = Column(DateTime, nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
template = relationship("TestTemplate", foreign_keys=[template_id])
source_test = relationship("Test", foreign_keys=[source_test_id])
submitter = relationship("User", foreign_keys=[submitted_by])
reviewer = relationship("User", foreign_keys=[reviewed_by])
+9
View File
@@ -59,6 +59,12 @@ class Test(Base):
execution_date = Column(DateTime, nullable=True) execution_date = Column(DateTime, nullable=True)
# Assign created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) # Assign created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
# The template this test was instantiated from, if any (null for
# standalone/manually-created/RT-imported tests). Lets a procedure
# suggestion know exactly which template to propose improving —
# matching purely by technique would be ambiguous when a technique has
# multiple templates.
source_template_id = Column(UUID(as_uuid=True), ForeignKey("test_templates.id"), nullable=True)
# Assign result = Column(Enum(TestResult, name="testresult"), nullable=True) # Assign result = Column(Enum(TestResult, name="testresult"), nullable=True)
result = Column(Enum(TestResult, name="testresult"), nullable=True) result = Column(Enum(TestResult, name="testresult"), nullable=True)
# Assign state = Column(Enum(TestState, name="teststate"), default=TestState.draft) # Assign state = Column(Enum(TestState, name="teststate"), default=TestState.draft)
@@ -83,6 +89,9 @@ class Test(Base):
# ── Blue Team fields ──────────────────────────────────────────── # ── Blue Team fields ────────────────────────────────────────────
blue_summary = Column(Text, nullable=True) blue_summary = Column(Text, nullable=True)
# What Blue actually did to detect the attack — Blue's counterpart to
# procedure_text. Free text; may be parsed for a procedure suggestion.
detect_procedure = Column(Text, nullable=True)
# Assign detection_result = Column(Enum(TestResult, name="testresult"), nullable=True) # Assign detection_result = Column(Enum(TestResult, name="testresult"), nullable=True)
detection_result = Column(Enum(TestResult, name="testresult"), nullable=True) detection_result = Column(Enum(TestResult, name="testresult"), nullable=True)
containment_result = Column(Enum(ContainmentResult, name="containmentresult"), nullable=True) containment_result = Column(Enum(ContainmentResult, name="containmentresult"), nullable=True)
+1
View File
@@ -50,6 +50,7 @@ class TestRoundHistory(Base):
detection_time = Column(DateTime, nullable=True) detection_time = Column(DateTime, nullable=True)
containment_time = Column(DateTime, nullable=True) containment_time = Column(DateTime, nullable=True)
blue_summary = Column(Text, nullable=True) blue_summary = Column(Text, nullable=True)
detect_procedure = Column(Text, nullable=True)
# Why the round was closed # Why the round was closed
review_notes = Column(Text, nullable=True) review_notes = Column(Text, nullable=True)
+5
View File
@@ -43,6 +43,11 @@ class TestTemplate(Base):
attack_procedure = Column(Text, nullable=True) # Suggested attack procedure attack_procedure = Column(Text, nullable=True) # Suggested attack procedure
# Assign expected_detection = Column(Text, nullable=True) # What blue team should detect # Assign expected_detection = Column(Text, nullable=True) # What blue team should detect
expected_detection = Column(Text, nullable=True) # What blue team should detect expected_detection = Column(Text, nullable=True) # What blue team should detect
# Suggested detection procedure — Blue's counterpart to attack_procedure.
# Only ever filled in via an approved procedure suggestion or a lead
# editing the template directly; external syncs never touch it (those
# only insert brand-new rows, never update existing ones).
detect_suggested_procedure = Column(Text, nullable=True)
# Assign platform = Column(String, nullable=True) # windows / linux... # Assign platform = Column(String, nullable=True) # windows / linux...
platform = Column(String, nullable=True) # windows / linux / macos platform = Column(String, nullable=True) # windows / linux / macos
# Assign tool_suggested = Column(String, nullable=True) # Assign tool_suggested = Column(String, nullable=True)
@@ -0,0 +1,139 @@
"""Router for procedure-improvement suggestion review.
Endpoints
---------
GET /procedure-suggestions — list pending suggestions (lead/admin)
POST /procedure-suggestions/{id}/approve — write suggestion into template (lead/admin)
POST /procedure-suggestions/{id}/reject — dismiss suggestion (lead/admin)
A red_lead only ever sees/acts on "red" team suggestions, a blue_lead only
"blue" — admins can see and act on both.
"""
import uuid
from fastapi import APIRouter, Depends, HTTPException
from app.database import get_db
from app.dependencies.auth import require_any_role_strict
from app.domain.errors import EntityNotFoundError, InvalidOperationError
from app.domain.unit_of_work import UnitOfWork
from app.models.procedure_suggestion import ProcedureSuggestion
from app.models.test_template import TestTemplate
from app.models.user import User
from app.schemas.procedure_suggestion import ProcedureSuggestionOut
from app.services.audit_service import log_action
from app.services.procedure_suggestion_service import (
approve_suggestion as approve_suggestion_svc,
)
from app.services.procedure_suggestion_service import (
list_pending_suggestions,
)
from app.services.procedure_suggestion_service import (
reject_suggestion as reject_suggestion_svc,
)
from sqlalchemy.orm import Session
router = APIRouter(prefix="/procedure-suggestions", tags=["procedure-suggestions"])
def _team_for_role(user: User) -> str | None:
"""Return the single team a non-admin lead is scoped to, or None for admin (both)."""
if user.role == "red_lead":
return "red"
if user.role == "blue_lead":
return "blue"
return None
def _to_out(suggestion, template_by_id: dict) -> ProcedureSuggestionOut:
out = ProcedureSuggestionOut.model_validate(suggestion)
template = template_by_id.get(suggestion.template_id)
if template is not None:
out.template_name = template.name
out.template_current_text = (
template.attack_procedure if suggestion.team == "red" else template.detect_suggested_procedure
)
return out
@router.get("", response_model=list[ProcedureSuggestionOut])
def list_suggestions(
db: Session = Depends(get_db),
current_user: User = Depends(require_any_role_strict("red_lead", "blue_lead", "admin")),
) -> list[ProcedureSuggestionOut]:
"""List pending procedure suggestions, scoped to the caller's team (admins see both)."""
team = _team_for_role(current_user)
suggestions = list_pending_suggestions(db, team=team)
template_ids = {s.template_id for s in suggestions}
templates = (
db.query(TestTemplate).filter(TestTemplate.id.in_(template_ids)).all() if template_ids else []
)
template_by_id = {t.id: t for t in templates}
return [_to_out(s, template_by_id) for s in suggestions]
@router.post("/{suggestion_id}/approve", response_model=ProcedureSuggestionOut)
def approve_suggestion(
suggestion_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(require_any_role_strict("red_lead", "blue_lead", "admin")),
) -> ProcedureSuggestionOut:
"""Approve a suggestion, writing it into its template's suggested-procedure field."""
_check_team_permission(current_user, _team_or_404(db, suggestion_id))
try:
with UnitOfWork(db) as uow:
suggestion = approve_suggestion_svc(db, suggestion_id, current_user)
log_action(
db, user_id=current_user.id, action="approve_procedure_suggestion",
entity_type="procedure_suggestion", entity_id=suggestion.id,
details={"template_id": str(suggestion.template_id), "team": suggestion.team},
)
uow.commit()
except EntityNotFoundError as e:
raise HTTPException(status_code=404, detail=str(e)) from e
except InvalidOperationError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
db.refresh(suggestion)
template = db.query(TestTemplate).filter(TestTemplate.id == suggestion.template_id).first()
return _to_out(suggestion, {template.id: template} if template else {})
@router.post("/{suggestion_id}/reject", response_model=ProcedureSuggestionOut)
def reject_suggestion(
suggestion_id: uuid.UUID,
db: Session = Depends(get_db),
current_user: User = Depends(require_any_role_strict("red_lead", "blue_lead", "admin")),
) -> ProcedureSuggestionOut:
"""Reject a suggestion without touching the template."""
_check_team_permission(current_user, _team_or_404(db, suggestion_id))
try:
with UnitOfWork(db) as uow:
suggestion = reject_suggestion_svc(db, suggestion_id, current_user)
log_action(
db, user_id=current_user.id, action="reject_procedure_suggestion",
entity_type="procedure_suggestion", entity_id=suggestion.id,
details={"template_id": str(suggestion.template_id), "team": suggestion.team},
)
uow.commit()
except EntityNotFoundError as e:
raise HTTPException(status_code=404, detail=str(e)) from e
except InvalidOperationError as e:
raise HTTPException(status_code=400, detail=str(e)) from e
db.refresh(suggestion)
return _to_out(suggestion, {})
def _team_or_404(db: Session, suggestion_id: uuid.UUID) -> str:
"""Look up just the team of a suggestion, for a permission check before mutating it."""
suggestion = db.query(ProcedureSuggestion).filter(ProcedureSuggestion.id == suggestion_id).first()
if suggestion is None:
raise HTTPException(status_code=404, detail="Procedure suggestion not found")
return suggestion.team
def _check_team_permission(user: User, team: str) -> None:
"""Raise 403 if a non-admin lead tries to act on the other team's suggestion."""
scoped_team = _team_for_role(user)
if scoped_team is not None and scoped_team != team:
raise HTTPException(status_code=403, detail=f"Not authorized to review {team} team suggestions")
+3 -44
View File
@@ -163,46 +163,6 @@ from app.services.test_workflow_service import (
router = APIRouter(prefix="/tests", tags=["tests"]) router = APIRouter(prefix="/tests", tags=["tests"])
# ---------------------------------------------------------------------------
# Blind visibility — hide the other team's fields until both reviews pass
# ---------------------------------------------------------------------------
_RED_ONLY_FIELDS = [
"procedure_text", "tool_used", "attack_success",
"execution_start_time", "execution_end_time", "red_summary",
"red_validation_status", "red_validated_by", "red_validated_at", "red_validation_notes",
]
_BLUE_ONLY_FIELDS = [
"detection_result", "containment_result", "detection_time", "containment_time",
"blue_summary", "blue_validation_status", "blue_validated_by", "blue_validated_at",
"blue_validation_notes", "system_gaps",
]
_BLIND_STATES = {"draft", "red_executing", "red_review", "blue_evaluating", "blue_review"}
def _mask_for_team_blindness(test_out: TestOut, *, viewer_role: str) -> TestOut:
"""Null out the other team's fields while the test is still blind.
admin and viewer are never blinded. Once the test reaches in_review or
beyond, both sides see everything (existing cross-validation behavior).
"""
if viewer_role in ("admin", "viewer"):
return test_out
test_state = test_out.state.value if hasattr(test_out.state, "value") else str(test_out.state)
if test_state not in _BLIND_STATES:
return test_out
if viewer_role in ("blue_tech", "blue_lead"):
hide_fields = _RED_ONLY_FIELDS
elif viewer_role in ("red_tech", "red_lead"):
hide_fields = _BLUE_ONLY_FIELDS
else:
return test_out
return test_out.model_copy(update={f: None for f in hide_fields})
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# GET /tests — list with filters # GET /tests — list with filters
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -504,12 +464,11 @@ def get_test(
Returns: Returns:
TestOut: Full test detail including split red/blue evidence lists. TestOut: Full test detail including split red/blue evidence lists.
Fields belonging to the other team are nulled out while the Both teams see each other's fields regardless of state — Red
test is blind (see :func:`_mask_for_team_blindness`). and Blue are meant to see each other's work, not test blind.
""" """
test = crud_get_test_detail(db, test_id) test = crud_get_test_detail(db, test_id)
test_out = TestOut.model_validate(test) return TestOut.model_validate(test)
return _mask_for_team_blindness(test_out, viewer_role=current_user.role)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -0,0 +1,28 @@
"""Pydantic schemas for procedure-suggestion review endpoints."""
import uuid
from datetime import datetime
from pydantic import BaseModel, ConfigDict
class ProcedureSuggestionOut(BaseModel):
"""A proposed template procedure update, pending or reviewed."""
id: uuid.UUID
template_id: uuid.UUID
team: str
suggested_text: str
source_test_id: uuid.UUID | None = None
submitted_by: uuid.UUID | None = None
status: str
reviewed_by: uuid.UUID | None = None
reviewed_at: datetime | None = None
created_at: datetime | None = None
# Populated for display — the template name and its current suggested
# value, so a lead can compare without a second lookup.
template_name: str | None = None
template_current_text: str | None = None
model_config = ConfigDict(from_attributes=True)
+4
View File
@@ -115,6 +115,7 @@ class TestBlueUpdate(BaseModel):
containment_time: datetime | None = None containment_time: datetime | None = None
# Assign blue_summary = None # Assign blue_summary = None
blue_summary: str | None = None blue_summary: str | None = None
detect_procedure: str | None = None
_normalize_datetimes = field_validator("detection_time", "containment_time")(_to_naive_utc) _normalize_datetimes = field_validator("detection_time", "containment_time")(_to_naive_utc)
@@ -142,6 +143,7 @@ class TestRoundHistoryOut(BaseModel):
detection_time: datetime | None = None detection_time: datetime | None = None
containment_time: datetime | None = None containment_time: datetime | None = None
blue_summary: str | None = None blue_summary: str | None = None
detect_procedure: str | None = None
review_notes: str | None = None review_notes: str | None = None
reviewed_by: uuid.UUID | None = None reviewed_by: uuid.UUID | None = None
archived_at: datetime | None = None archived_at: datetime | None = None
@@ -269,6 +271,7 @@ class TestOut(BaseModel):
execution_date: datetime | None = None execution_date: datetime | None = None
# Assign created_by = None # Assign created_by = None
created_by: uuid.UUID | None = None created_by: uuid.UUID | None = None
source_template_id: uuid.UUID | None = None
# Assign result = None # Assign result = None
result: TestResult | None = None result: TestResult | None = None
# Assign state = TestState.draft # Assign state = TestState.draft
@@ -293,6 +296,7 @@ class TestOut(BaseModel):
# Blue Team fields # Blue Team fields
blue_summary: str | None = None blue_summary: str | None = None
detect_procedure: str | None = None
# Assign detection_result = None # Assign detection_result = None
detection_result: TestResult | None = None detection_result: TestResult | None = None
containment_result: ContainmentResult | None = None containment_result: ContainmentResult | None = None
+2
View File
@@ -31,6 +31,7 @@ class TestTemplateOut(BaseModel):
attack_procedure: str | None = None attack_procedure: str | None = None
# Assign expected_detection = None # Assign expected_detection = None
expected_detection: str | None = None expected_detection: str | None = None
detect_suggested_procedure: str | None = None
# Assign platform = None # Assign platform = None
platform: str | None = None platform: str | None = None
# Assign tool_suggested = None # Assign tool_suggested = None
@@ -70,6 +71,7 @@ class TestTemplateCreate(BaseModel):
attack_procedure: str | None = None attack_procedure: str | None = None
# Assign expected_detection = None # Assign expected_detection = None
expected_detection: str | None = None expected_detection: str | None = None
detect_suggested_procedure: str | None = None
# Assign platform = None # Assign platform = None
platform: str | None = None platform: str | None = None
# Assign tool_suggested = None # Assign tool_suggested = None
+2
View File
@@ -486,6 +486,7 @@ def _build_state_comment(
f"Blue Team has submitted evidence for Round {test.blue_round_number or 1} and the " f"Blue Team has submitted evidence for Round {test.blue_round_number or 1} and the "
"test is awaiting Blue Lead review before cross-validation.", "test is awaiting Blue Lead review before cross-validation.",
"", "",
f"*Detect Procedure:* {test.detect_procedure or 'N/A'}",
f"*Detection Result:* {_enum_value(test.detection_result) or 'N/A'}", f"*Detection Result:* {_enum_value(test.detection_result) or 'N/A'}",
f"*Containment Result:* {_enum_value(test.containment_result) or 'N/A'}", f"*Containment Result:* {_enum_value(test.containment_result) or 'N/A'}",
] ]
@@ -1129,6 +1130,7 @@ def push_round_archived(db: Session, test, actor, *, round_data) -> None:
lines.append(f"*Attack success:* {round_data.attack_success.value if round_data.attack_success else '-'}") lines.append(f"*Attack success:* {round_data.attack_success.value if round_data.attack_success else '-'}")
lines.append(f"*Summary:* {round_data.red_summary or '-'}") lines.append(f"*Summary:* {round_data.red_summary or '-'}")
else: else:
lines.append(f"*Detect procedure:* {round_data.detect_procedure or '_none recorded_'}")
lines.append(f"*Detection result:* {round_data.detection_result.value if round_data.detection_result else '-'}") lines.append(f"*Detection result:* {round_data.detection_result.value if round_data.detection_result else '-'}")
lines.append(f"*Containment result:* {round_data.containment_result.value if round_data.containment_result else '-'}") lines.append(f"*Containment result:* {round_data.containment_result.value if round_data.containment_result else '-'}")
lines.append(f"*Summary:* {round_data.blue_summary or '-'}") lines.append(f"*Summary:* {round_data.blue_summary or '-'}")
@@ -0,0 +1,84 @@
"""Heuristic extraction of the actual command(s)/query out of a free-text
procedure field (``procedure_text`` for Red, ``detect_procedure`` for
Blue), for proposing as a template's suggested procedure.
Operators write these fields as free text mixing narrative ("I ran this
against the DC and it worked"), the actual command, and sometimes pasted
output. We only want the command — not the story around it, not the
output. There's no reliable general way to do this with NLP-level
accuracy, so this uses a deterministic, conservative regex approach:
1. Fenced code blocks (```...``` or ~~~...~~~) are the strongest signal —
an operator who bothered to fence something almost always fenced the
command, not the narrative. If any exist, their contents are returned
verbatim (narrative outside the fence is ignored).
2. Otherwise, each line is checked against a curated set of patterns that
look like real commands or detection queries (shell prompts, known
binaries/cmdlets, SQL/KQL/SPL-style query syntax). Only matching lines
survive; narrative sentences and plain command output are discarded.
This is deliberately conservative — false negatives (missing a real
command written in an unrecognized style) are far less harmful than false
positives (proposing narrative or output as "the command"), especially
since a lead reviews every suggestion before it reaches a template.
"""
from __future__ import annotations
import re
_FENCE_RE = re.compile(r"```(?:\w*\n)?(.*?)```|~~~(?:\w*\n)?(.*?)~~~", re.DOTALL)
# Leading shell/PowerShell prompt markers — stripped, not treated as
# evidence by themselves (the remainder still has to look like a command).
_PROMPT_RE = re.compile(r"^\s*(?:PS(?:\s+\S+)?>|[$#>])\s+")
# Signatures that indicate "this line is a command or query", not prose.
_COMMAND_LINE_RE = re.compile(
r"""
^\s*(?:PS(?:\s+\S+)?>|[$#>])\s*\S # explicit shell/PowerShell prompt with something after it
|
(?-i:\b[A-Z][a-zA-Z]+-[A-Z][a-zA-Z]+\b) # PowerShell Verb-Noun cmdlet, e.g. Get-Process
# (case-sensitive even though the rest of this
# pattern is IGNORECASE — otherwise lowercase
# hyphenated prose like "well-known" false-matches)
|
^\s*(?:sudo|chmod|chown|curl|wget|nmap|ssh|scp|python3?|bash|sh|zsh|
powershell(?:\.exe)?|cmd(?:\.exe)?|reg(?:\.exe)?|net(?:\.exe)?|
netsh|wmic|schtasks|sc|rundll32|regsvr32|msiexec|certutil|
mimikatz\S*|cscript|wscript|osascript|openssl|systemctl|service|
reg\s+query|whoami|nslookup|dig|ping|tasklist|ps\s|kill|
Invoke-\S+|iex|dsquery|nltest)\b
|
\bSELECT\s+.+\s+FROM\b # SQL
|
\bsearch\s+index\s*= # SPL (explicit "search" keyword)
|
\bindex\s*=\S # SPL (bare "index=value")
|
\|\s*(?:where|project|summarize|extend|join|render)\b # KQL pipe syntax
""",
re.IGNORECASE | re.VERBOSE,
)
def extract_commands(text: str | None) -> str | None:
"""Return the command/query lines found in *text*, or None if none found."""
if not text or not text.strip():
return None
fence_matches = _FENCE_RE.findall(text)
if fence_matches:
blocks = [(a or b).strip() for a, b in fence_matches if (a or b).strip()]
if blocks:
return "\n\n".join(blocks)
kept: list[str] = []
for line in text.splitlines():
stripped = line.strip()
if not stripped:
continue
if _COMMAND_LINE_RE.search(stripped):
kept.append(_PROMPT_RE.sub("", stripped))
return "\n".join(kept) if kept else None
@@ -0,0 +1,134 @@
"""Procedure-improvement suggestion workflow.
When an operator submits a round with a filled-in procedure field, the
command(s) in it are extracted heuristically (see
``procedure_extraction_service``) and proposed as an update to the
originating template's suggested-procedure field. A suggestion is only
ever created when there's a real, novel improvement to propose — nothing
is written to the template until a lead reviews and approves it.
"""
from __future__ import annotations
import uuid
from datetime import datetime
from sqlalchemy.orm import Session
from app.domain.errors import EntityNotFoundError, InvalidOperationError
from app.models.procedure_suggestion import ProcedureSuggestion
from app.models.test import Test
from app.models.test_template import TestTemplate
from app.models.user import User
from app.services.notification_service import notify_role
from app.services.procedure_extraction_service import extract_commands
# Which Test field holds the operator's free-text procedure, and which
# TestTemplate field it's proposed as an improvement to, per team.
_SOURCE_FIELD = {"red": "procedure_text", "blue": "detect_procedure"}
_TEMPLATE_FIELD = {"red": "attack_procedure", "blue": "detect_suggested_procedure"}
_LEAD_ROLE = {"red": "red_lead", "blue": "blue_lead"}
def create_suggestion_from_submission(db: Session, test: Test, team: str) -> ProcedureSuggestion | None:
"""Propose a template procedure improvement from *test*'s just-submitted round.
Returns the created (unflushed-to-caller-commit) suggestion, or ``None``
when there's nothing worth proposing: the test wasn't created from a
template, no command could be extracted, the extraction matches what
the template already suggests, or an identical suggestion is already
pending review. Does not commit; caller uses UnitOfWork.
"""
if not test.source_template_id:
return None
procedure_text = getattr(test, _SOURCE_FIELD[team])
extracted = extract_commands(procedure_text)
if not extracted:
return None
template = db.query(TestTemplate).filter(TestTemplate.id == test.source_template_id).first()
if template is None:
return None
current = (getattr(template, _TEMPLATE_FIELD[team]) or "").strip()
if extracted == current:
return None
duplicate = (
db.query(ProcedureSuggestion)
.filter(
ProcedureSuggestion.template_id == template.id,
ProcedureSuggestion.team == team,
ProcedureSuggestion.suggested_text == extracted,
ProcedureSuggestion.status == "pending",
)
.first()
)
if duplicate is not None:
return None
submitter_id = test.red_tech_assignee if team == "red" else test.blue_tech_assignee
suggestion = ProcedureSuggestion(
template_id=template.id,
team=team,
suggested_text=extracted,
source_test_id=test.id,
submitted_by=submitter_id,
)
db.add(suggestion)
db.flush()
notify_role(
db,
role=_LEAD_ROLE[team],
type="procedure_suggestion",
title="New procedure suggestion for review",
message=f'A {team} procedure improvement was proposed for template "{template.name}".',
entity_type="procedure_suggestion",
entity_id=suggestion.id,
)
return suggestion
def list_pending_suggestions(db: Session, *, team: str | None = None) -> list[ProcedureSuggestion]:
"""List pending suggestions, optionally filtered to one team."""
query = db.query(ProcedureSuggestion).filter(ProcedureSuggestion.status == "pending")
if team is not None:
query = query.filter(ProcedureSuggestion.team == team)
return query.order_by(ProcedureSuggestion.created_at.asc()).all()
def _get_pending_suggestion_or_raise(db: Session, suggestion_id: uuid.UUID) -> ProcedureSuggestion:
suggestion = db.query(ProcedureSuggestion).filter(ProcedureSuggestion.id == suggestion_id).first()
if suggestion is None:
raise EntityNotFoundError("ProcedureSuggestion", str(suggestion_id))
if suggestion.status != "pending":
raise InvalidOperationError("This suggestion has already been reviewed.")
return suggestion
def approve_suggestion(db: Session, suggestion_id: uuid.UUID, user: User) -> ProcedureSuggestion:
"""Approve a suggestion: write it into the template and mark reviewed. Does not commit."""
suggestion = _get_pending_suggestion_or_raise(db, suggestion_id)
template = db.query(TestTemplate).filter(TestTemplate.id == suggestion.template_id).first()
if template is None:
raise EntityNotFoundError("TestTemplate", str(suggestion.template_id))
setattr(template, _TEMPLATE_FIELD[suggestion.team], suggestion.suggested_text)
suggestion.status = "approved"
suggestion.reviewed_by = user.id
suggestion.reviewed_at = datetime.utcnow()
db.flush()
return suggestion
def reject_suggestion(db: Session, suggestion_id: uuid.UUID, user: User) -> ProcedureSuggestion:
"""Reject a suggestion without touching the template. Does not commit."""
suggestion = _get_pending_suggestion_or_raise(db, suggestion_id)
suggestion.status = "rejected"
suggestion.reviewed_by = user.id
suggestion.reviewed_at = datetime.utcnow()
db.flush()
return suggestion
@@ -364,9 +364,11 @@ def create_test_from_template(
platform=platform_override if platform_override is not None else template.platform, platform=platform_override if platform_override is not None else template.platform,
procedure_text=procedure_text_override if procedure_text_override is not None else template.attack_procedure, procedure_text=procedure_text_override if procedure_text_override is not None else template.attack_procedure,
tool_used=tool_used_override if tool_used_override is not None else template.tool_suggested, tool_used=tool_used_override if tool_used_override is not None else template.tool_suggested,
detect_procedure=template.detect_suggested_procedure,
remediation_steps=template.suggested_remediation, remediation_steps=template.suggested_remediation,
# Keyword argument: created_by # Keyword argument: created_by
created_by=creator_id, created_by=creator_id,
source_template_id=template.id,
# Keyword argument: state # Keyword argument: state
state=TestState.draft, state=TestState.draft,
created_at=datetime.utcnow(), # explicit — DB column has no server default created_at=datetime.utcnow(), # explicit — DB column has no server default
+13 -1
View File
@@ -340,6 +340,12 @@ def submit_red_evidence(db: Session, test: Test, user: User) -> Test:
except Exception as e: except Exception as e:
logger.warning("Jira RT end date push failed for test %s: %s", test.id, e, exc_info=True) logger.warning("Jira RT end date push failed for test %s: %s", test.id, e, exc_info=True)
try:
from app.services.procedure_suggestion_service import create_suggestion_from_submission
create_suggestion_from_submission(db, test, "red")
except Exception as e:
logger.warning("Procedure suggestion extraction failed for test %s: %s", test.id, e, exc_info=True)
return test return test
@@ -577,6 +583,12 @@ def submit_blue_evidence(db: Session, test: Test, user: User) -> Test:
except Exception as e: except Exception as e:
logger.warning("Jira BT end date push failed for test %s: %s", test.id, e, exc_info=True) logger.warning("Jira BT end date push failed for test %s: %s", test.id, e, exc_info=True)
try:
from app.services.procedure_suggestion_service import create_suggestion_from_submission
create_suggestion_from_submission(db, test, "blue")
except Exception as e:
logger.warning("Procedure suggestion extraction failed for test %s: %s", test.id, e, exc_info=True)
return test return test
@@ -623,7 +635,7 @@ def reopen_blue_review(db: Session, test: Test, user: User, notes: str) -> Test:
paused_seconds=test.blue_paused_seconds or 0, paused_seconds=test.blue_paused_seconds or 0,
detection_result=test.detection_result, containment_result=test.containment_result, detection_result=test.detection_result, containment_result=test.containment_result,
detection_time=test.detection_time, containment_time=test.containment_time, detection_time=test.detection_time, containment_time=test.containment_time,
blue_summary=test.blue_summary, blue_summary=test.blue_summary, detect_procedure=test.detect_procedure,
review_notes=notes.strip(), reviewed_by=user.id, review_notes=notes.strip(), reviewed_by=user.id,
) )
db.add(archived_round) db.add(archived_round)
+18 -28
View File
@@ -1,9 +1,10 @@
"""Tests for red/blue blind visibility on GET /tests/{id}. """Regression: Red and Blue must see each other's fields at every stage.
Neither team should see the other team's data while a test is in Blind visibility (hiding the other team's fields until both reviews pass)
draft/red_executing/red_review/blue_evaluating/blue_review. Both sides was a deliberate design in an earlier phase, but the org decided both teams
see everything once the test reaches in_review (and beyond). admin and should see each other's work throughout — detection testing isn't meant to
viewer are never blinded. be blind here. This confirms neither the frontend nor backend hides
anything based on role/state anymore.
""" """
import uuid import uuid
@@ -46,11 +47,11 @@ def technique(api, auth_headers):
@pytest.fixture @pytest.fixture
def blind_test(client, db, api, red_lead_headers, red_lead_user, red_tech_headers, technique): def in_progress_test(client, db, api, red_lead_headers, red_lead_user, red_tech_headers, technique):
"""A test in red_executing with red-side content already filled in.""" """A test in red_executing with red-side content already filled in."""
resp = api( resp = api(
"post", "/api/v1/tests", red_lead_headers, "post", "/api/v1/tests", red_lead_headers,
json={"technique_id": technique, "name": "Blind visibility test"}, json={"technique_id": technique, "name": "Visibility test"},
) )
test_id = resp.json()["id"] test_id = resp.json()["id"]
@@ -62,32 +63,21 @@ def blind_test(client, db, api, red_lead_headers, red_lead_user, red_tech_header
return test_id return test_id
def test_blue_viewer_cannot_see_red_fields_during_red_executing( def test_blue_viewer_sees_red_fields_during_red_executing(
client, db, api, blind_test, blue_tech_headers client, db, api, in_progress_test, blue_tech_headers
): ):
resp = api("get", f"/api/v1/tests/{blind_test}", blue_tech_headers) resp = api("get", f"/api/v1/tests/{in_progress_test}", blue_tech_headers)
assert resp.status_code == 200 assert resp.status_code == 200
body = resp.json() body = resp.json()
for field in _RED_FIELDS:
assert body[field] is None, f"{field} should be hidden from blue viewer"
def test_red_viewer_cannot_see_blue_fields_during_red_executing(
client, db, api, blind_test, red_tech_headers
):
resp = api("get", f"/api/v1/tests/{blind_test}", red_tech_headers)
assert resp.status_code == 200
body = resp.json()
for field in _BLUE_FIELDS:
assert body[field] is None, f"{field} should be hidden from red viewer"
# Red's own fields ARE visible to red
assert body["procedure_text"] == "run mimikatz" assert body["procedure_text"] == "run mimikatz"
assert body["tool_used"] == "mimikatz"
assert body["red_summary"] == "dumped creds"
def test_admin_sees_everything_during_blind_states( def test_red_viewer_sees_own_fields_during_red_executing(
client, db, api, blind_test, auth_headers client, db, api, in_progress_test, red_tech_headers
): ):
resp = api("get", f"/api/v1/tests/{blind_test}", auth_headers) resp = api("get", f"/api/v1/tests/{in_progress_test}", red_tech_headers)
assert resp.status_code == 200 assert resp.status_code == 200
body = resp.json() body = resp.json()
assert body["procedure_text"] == "run mimikatz" assert body["procedure_text"] == "run mimikatz"
@@ -95,9 +85,9 @@ def test_admin_sees_everything_during_blind_states(
def test_both_sides_see_everything_once_in_review( def test_both_sides_see_everything_once_in_review(
client, db, api, auth_headers, red_tech_headers, red_lead_headers, client, db, api, auth_headers, red_tech_headers, red_lead_headers,
blue_tech_headers, blue_lead_headers, red_lead_user, blue_lead_user, blind_test, blue_tech_headers, blue_lead_headers, red_lead_user, blue_lead_user, in_progress_test,
): ):
test_id = blind_test test_id = in_progress_test
_add_evidence(db, test_id, TeamSide.red) _add_evidence(db, test_id, TeamSide.red)
submit_red = api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers) submit_red = api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
assert submit_red.status_code == 200, submit_red.text assert submit_red.status_code == 200, submit_red.text
@@ -0,0 +1,97 @@
"""Tests for the regex-based command/query extraction heuristic used to
build procedure-improvement suggestions from free-text procedure fields."""
import pytest
from app.services.procedure_extraction_service import extract_commands
@pytest.mark.parametrize("value", [None, "", " ", "\n\n"])
def test_returns_none_for_empty_input(value):
assert extract_commands(value) is None
def test_returns_none_for_pure_narrative():
text = (
"I logged into the domain controller and tried a well-known credential "
"dumping tool. It worked well and I documented everything in the summary."
)
assert extract_commands(text) is None
def test_fenced_code_block_takes_priority_over_surrounding_narrative():
text = (
"Ran the following from an elevated prompt:\n"
"```\n"
"Invoke-Mimikatz -DumpCreds\n"
"```\n"
"It successfully extracted plaintext passwords from LSASS memory."
)
assert extract_commands(text) == "Invoke-Mimikatz -DumpCreds"
def test_tilde_fence_supported():
text = "~~~\nGet-Process lsass\n~~~"
assert extract_commands(text) == "Get-Process lsass"
def test_multiple_fenced_blocks_are_joined():
text = "```\nwhoami /priv\n```\nthen\n```\nGet-Process lsass\n```"
assert extract_commands(text) == "whoami /priv\n\nGet-Process lsass"
def test_extracts_command_lines_from_mixed_narrative_no_fence():
text = (
"First I checked what privileges I had.\n"
"whoami /priv\n"
"Then I dumped credentials from LSASS.\n"
"Invoke-Mimikatz -DumpCreds\n"
"This successfully retrieved plaintext passwords, as shown below.\n"
"Handles NPM(K) PM(K) WS(K) CPU(s) Id SI ProcessName\n"
" 542 27 23012 41564 0.45 1234 1 explorer\n"
)
result = extract_commands(text)
assert result == "whoami /priv\nInvoke-Mimikatz -DumpCreds"
def test_strips_shell_prompt_markers():
text = (
"Ran this against the target host:\n"
"$ curl -X POST http://target/api/exfil -d @creds.txt\n"
"Got a 200 OK back.\n"
)
assert extract_commands(text) == "curl -X POST http://target/api/exfil -d @creds.txt"
def test_strips_powershell_prompt_marker():
text = "PS C:\\Users\\op> Get-Process lsass"
assert extract_commands(text) == "Get-Process lsass"
def test_extracts_kql_detection_query_mixed_with_narrative():
text = (
"I built a Sentinel query to catch this technique going forward.\n"
"DeviceProcessEvents\n"
"| where FileName == \"mimikatz.exe\"\n"
"That caught it within a minute of execution in testing.\n"
)
result = extract_commands(text)
assert 'where FileName == "mimikatz.exe"' in result
assert "That caught it" not in result
def test_extracts_splunk_query():
text = (
"Search used for detection:\n"
"index=main sourcetype=WinEventLog:Security EventCode=4688\n"
"This surfaced the process creation event immediately.\n"
)
result = extract_commands(text)
assert "index=main sourcetype=WinEventLog:Security EventCode=4688" in result
assert "immediately" not in result
def test_known_binary_at_start_of_line_is_recognized():
text = "Notes: used the following.\nsudo tcpdump -i eth0 -w capture.pcap\nCaptured 500 packets."
result = extract_commands(text)
assert result == "sudo tcpdump -i eth0 -w capture.pcap"
+232
View File
@@ -0,0 +1,232 @@
"""End-to-end coverage for the procedure-suggestion review workflow.
An operator submitting a round with a filled-in procedure field that
contains an extractable command should produce a pending
ProcedureSuggestion against the originating template; a lead can then
approve it (writing it into the template) or reject it (leaving the
template untouched). Nothing is ever written to a template automatically.
"""
import uuid
import pytest
from app.models.enums import TeamSide
from app.models.evidence import Evidence
def _add_evidence(db, test_id, team: TeamSide):
ev = Evidence(
test_id=uuid.UUID(test_id),
file_name="proof.txt",
file_path="s3://bucket/proof.txt",
sha256_hash="a" * 64,
team=team,
)
db.add(ev)
db.commit()
@pytest.fixture
def technique(api, auth_headers):
resp = api(
"post", "/api/v1/techniques", auth_headers,
json={"mitre_id": "T1059.200", "name": "Command Line Suggestions"},
)
assert resp.status_code == 201, resp.text
return resp.json()["id"]
@pytest.fixture
def template(api, red_lead_headers, technique):
resp = api(
"post", "/api/v1/test-templates", red_lead_headers,
json={
"mitre_technique_id": "T1059.200",
"name": "Suggestion source template",
"attack_procedure": "Run a discovery command on the target.",
"detect_suggested_procedure": "Check process creation logs.",
},
)
assert resp.status_code == 201, resp.text
return resp.json()
@pytest.fixture
def test_from_template(api, red_lead_headers, technique, template):
resp = api(
"post", "/api/v1/tests/from-template", red_lead_headers,
json={"template_id": template["id"], "technique_id": technique},
)
assert resp.status_code == 201, resp.text
return resp.json()["id"]
def test_red_submission_with_command_creates_pending_suggestion(
client, db, api, test_from_template, red_tech_headers, red_lead_headers,
):
test_id = test_from_template
api("post", f"/api/v1/tests/{test_id}/start-execution", red_tech_headers)
api(
"patch", f"/api/v1/tests/{test_id}/red", red_tech_headers,
json={"procedure_text": "Ran discovery.\nwhoami /priv\nGot back SeDebugPrivilege."},
)
_add_evidence(db, test_id, TeamSide.red)
resp = api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
assert resp.status_code == 200, resp.text
listed = api("get", "/api/v1/procedure-suggestions", red_lead_headers)
assert listed.status_code == 200, listed.text
suggestions = listed.json()
assert len(suggestions) == 1
assert suggestions[0]["team"] == "red"
assert suggestions[0]["suggested_text"] == "whoami /priv"
assert suggestions[0]["status"] == "pending"
def test_blue_submission_with_command_creates_pending_suggestion(
client, db, api, test_from_template, red_tech_headers, red_lead_headers,
blue_tech_headers, blue_lead_headers,
):
test_id = test_from_template
api("post", f"/api/v1/tests/{test_id}/start-execution", red_tech_headers)
_add_evidence(db, test_id, TeamSide.red)
api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
api("post", f"/api/v1/tests/{test_id}/review-red", red_lead_headers, json={"decision": "approve"})
api("post", f"/api/v1/tests/{test_id}/start-blue-work", blue_tech_headers)
api(
"patch", f"/api/v1/tests/{test_id}/blue", blue_tech_headers,
json={
"detection_result": "detected",
"detect_procedure": "Checked logs.\nGet-WinEvent -LogName Security | where Id -eq 4688\nFound it.",
},
)
_add_evidence(db, test_id, TeamSide.blue)
resp = api("post", f"/api/v1/tests/{test_id}/submit-blue", blue_tech_headers)
assert resp.status_code == 200, resp.text
listed = api("get", "/api/v1/procedure-suggestions", blue_lead_headers)
assert listed.status_code == 200, listed.text
suggestions = listed.json()
assert len(suggestions) == 1
assert suggestions[0]["team"] == "blue"
assert "Get-WinEvent" in suggestions[0]["suggested_text"]
def test_submission_without_extractable_command_creates_no_suggestion(
client, db, api, test_from_template, red_tech_headers, red_lead_headers,
):
test_id = test_from_template
api("post", f"/api/v1/tests/{test_id}/start-execution", red_tech_headers)
api(
"patch", f"/api/v1/tests/{test_id}/red", red_tech_headers,
json={"procedure_text": "Just talked to the target and it agreed to be compromised."},
)
_add_evidence(db, test_id, TeamSide.red)
resp = api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
assert resp.status_code == 200, resp.text
listed = api("get", "/api/v1/procedure-suggestions", red_lead_headers)
assert listed.json() == []
def test_approving_a_suggestion_writes_it_into_the_template(
client, db, api, test_from_template, template, red_tech_headers, red_lead_headers,
):
test_id = test_from_template
api("post", f"/api/v1/tests/{test_id}/start-execution", red_tech_headers)
api(
"patch", f"/api/v1/tests/{test_id}/red", red_tech_headers,
json={"procedure_text": "Ran discovery.\nwhoami /priv\nDone."},
)
_add_evidence(db, test_id, TeamSide.red)
api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
suggestion = api("get", "/api/v1/procedure-suggestions", red_lead_headers).json()[0]
approve = api(
"post", f"/api/v1/procedure-suggestions/{suggestion['id']}/approve", red_lead_headers,
)
assert approve.status_code == 200, approve.text
assert approve.json()["status"] == "approved"
reloaded = api("get", f"/api/v1/test-templates/{template['id']}", red_lead_headers)
assert reloaded.json()["attack_procedure"] == "whoami /priv"
listed = api("get", "/api/v1/procedure-suggestions", red_lead_headers)
assert listed.json() == []
def test_rejecting_a_suggestion_leaves_the_template_untouched(
client, db, api, test_from_template, template, red_tech_headers, red_lead_headers,
):
test_id = test_from_template
api("post", f"/api/v1/tests/{test_id}/start-execution", red_tech_headers)
api(
"patch", f"/api/v1/tests/{test_id}/red", red_tech_headers,
json={"procedure_text": "Ran discovery.\nwhoami /priv\nDone."},
)
_add_evidence(db, test_id, TeamSide.red)
api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
suggestion = api("get", "/api/v1/procedure-suggestions", red_lead_headers).json()[0]
reject = api(
"post", f"/api/v1/procedure-suggestions/{suggestion['id']}/reject", red_lead_headers,
)
assert reject.status_code == 200, reject.text
assert reject.json()["status"] == "rejected"
reloaded = api("get", f"/api/v1/test-templates/{template['id']}", red_lead_headers)
assert reloaded.json()["attack_procedure"] == "Run a discovery command on the target."
def test_blue_lead_cannot_review_a_red_suggestion(
client, db, api, test_from_template, red_tech_headers, red_lead_headers, blue_lead_headers,
):
test_id = test_from_template
api("post", f"/api/v1/tests/{test_id}/start-execution", red_tech_headers)
api(
"patch", f"/api/v1/tests/{test_id}/red", red_tech_headers,
json={"procedure_text": "Ran discovery.\nwhoami /priv\nDone."},
)
_add_evidence(db, test_id, TeamSide.red)
api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
suggestion = api("get", "/api/v1/procedure-suggestions", red_lead_headers).json()[0]
# A blue_lead sees no red suggestions in their own queue...
blue_view = api("get", "/api/v1/procedure-suggestions", blue_lead_headers)
assert blue_view.json() == []
# ...and is forbidden from acting on one directly by id.
forbidden = api(
"post", f"/api/v1/procedure-suggestions/{suggestion['id']}/approve", blue_lead_headers,
)
assert forbidden.status_code == 403
def test_duplicate_submission_does_not_create_a_second_pending_suggestion(
client, db, api, test_from_template, red_tech_headers, red_lead_headers,
):
test_id = test_from_template
api("post", f"/api/v1/tests/{test_id}/start-execution", red_tech_headers)
api(
"patch", f"/api/v1/tests/{test_id}/red", red_tech_headers,
json={"procedure_text": "Ran discovery.\nwhoami /priv\nDone."},
)
_add_evidence(db, test_id, TeamSide.red)
api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
api(
"post", f"/api/v1/tests/{test_id}/review-red", red_lead_headers,
json={"decision": "reopen", "notes": "please redo this round"},
)
api(
"patch", f"/api/v1/tests/{test_id}/red", red_tech_headers,
json={"procedure_text": "Ran discovery again.\nwhoami /priv\nStill works."},
)
_add_evidence(db, test_id, TeamSide.red)
api("post", f"/api/v1/tests/{test_id}/submit-red", red_tech_headers)
listed = api("get", "/api/v1/procedure-suggestions", red_lead_headers).json()
assert len(listed) == 1
+20
View File
@@ -0,0 +1,20 @@
import client from "./client";
import type { ProcedureSuggestion } from "../types/models";
/** List pending procedure suggestions, scoped to the caller's team (admins see both). */
export async function getProcedureSuggestions(): Promise<ProcedureSuggestion[]> {
const { data } = await client.get<ProcedureSuggestion[]>("/procedure-suggestions");
return data;
}
/** Approve a suggestion — writes it into the template's suggested-procedure field. */
export async function approveProcedureSuggestion(id: string): Promise<ProcedureSuggestion> {
const { data } = await client.post<ProcedureSuggestion>(`/procedure-suggestions/${id}/approve`);
return data;
}
/** Reject a suggestion — the template is left untouched. */
export async function rejectProcedureSuggestion(id: string): Promise<ProcedureSuggestion> {
const { data } = await client.post<ProcedureSuggestion>(`/procedure-suggestions/${id}/reject`);
return data;
}
+1
View File
@@ -24,6 +24,7 @@ export interface CreateTemplatePayload {
source_url?: string; source_url?: string;
attack_procedure?: string; attack_procedure?: string;
expected_detection?: string; expected_detection?: string;
detect_suggested_procedure?: string;
platform?: string; platform?: string;
tool_suggested?: string; tool_suggested?: string;
severity?: string; severity?: string;
+1
View File
@@ -46,6 +46,7 @@ export interface BlueUpdatePayload {
detection_time?: string; detection_time?: string;
containment_time?: string; containment_time?: string;
blue_summary?: string; blue_summary?: string;
detect_procedure?: string;
} }
export interface RedValidationPayload { export interface RedValidationPayload {
@@ -114,6 +114,7 @@ interface TeamTabsProps {
detection_time: string; detection_time: string;
containment_time: string; containment_time: string;
blue_summary: string; blue_summary: string;
detect_procedure: string;
}; };
// Evidence // Evidence
@@ -177,13 +178,6 @@ export default function TeamTabs({
(role === "blue_lead" || (role === "blue_lead" ||
(role === "blue_tech" && !!test.blue_work_started_at)); (role === "blue_tech" && !!test.blue_work_started_at));
// Blind visibility: neither side sees the other's data until both reviews
// pass (matches the backend's field-masking on GET /tests/{id}).
const BLIND_STATES = ["draft", "red_executing", "red_review", "blue_evaluating", "blue_review"];
const isBlind = BLIND_STATES.includes(test.state) && role !== "admin" && role !== "viewer";
const hideBlueFromMe = isBlind && (role === "red_tech" || role === "red_lead");
const hideRedFromMe = isBlind && (role === "blue_tech" || role === "blue_lead");
// Containment fields only visible when attack was detected (draft or saved value) // Containment fields only visible when attack was detected (draft or saved value)
const isDetected = canEditBlue const isDetected = canEditBlue
? blueDraft.detection_result === "detected" || blueDraft.detection_result === "partially_detected" ? blueDraft.detection_result === "detected" || blueDraft.detection_result === "partially_detected"
@@ -211,17 +205,6 @@ export default function TeamTabs({
// ── Red Team Tab ───────────────────────────────────────────────── // ── Red Team Tab ─────────────────────────────────────────────────
const renderRedTab = () => { const renderRedTab = () => {
if (hideRedFromMe) {
return (
<div className="py-12 text-center">
<Shield className="mx-auto h-10 w-10 text-gray-600" />
<p className="mt-2 text-sm text-gray-400">
Red Team work is hidden until both teams' reviews are complete — this keeps
detection testing blind.
</p>
</div>
);
}
return ( return (
<div className="space-y-6"> <div className="space-y-6">
{/* Locked hint for red_tech in draft state */} {/* Locked hint for red_tech in draft state */}
@@ -420,17 +403,6 @@ export default function TeamTabs({
// ── Blue Team Tab ──────────────────────────────────────────────── // ── Blue Team Tab ────────────────────────────────────────────────
const renderBlueTab = () => { const renderBlueTab = () => {
if (hideBlueFromMe) {
return (
<div className="py-12 text-center">
<ShieldCheck className="mx-auto h-10 w-10 text-gray-600" />
<p className="mt-2 text-sm text-gray-400">
Blue Team work is hidden until both teams' reviews are complete this keeps
detection testing blind.
</p>
</div>
);
}
return ( return (
<div className="space-y-6"> <div className="space-y-6">
{/* Locked hint for blue_tech before Start Evaluation */} {/* Locked hint for blue_tech before Start Evaluation */}
@@ -440,6 +412,26 @@ export default function TeamTabs({
{blueLockedHint} {blueLockedHint}
</div> </div>
)} )}
{/* Detect Procedure */}
<div>
<label className="mb-1.5 block text-sm font-medium text-gray-300">
Detect Procedure
</label>
{canEditBlue ? (
<textarea
value={blueDraft.detect_procedure}
onChange={(e) => onBlueFieldChange("detect_procedure", e.target.value)}
rows={6}
className="w-full rounded-lg border border-gray-700 bg-gray-800 px-3 py-2 text-sm text-gray-200 placeholder-gray-500 focus:border-indigo-500 focus:outline-none focus:ring-1 focus:ring-indigo-500 font-mono"
placeholder="Describe how you detected the attack..."
/>
) : (
<pre className="whitespace-pre-wrap rounded-lg bg-gray-800 p-4 font-mono text-sm text-gray-300">
{test.detect_procedure || "No detection procedure documented."}
</pre>
)}
</div>
{/* Detection Result */} {/* Detection Result */}
<div> <div>
<label className="mb-2 block text-sm font-medium text-gray-300"> <label className="mb-2 block text-sm font-medium text-gray-300">
+26
View File
@@ -1355,6 +1355,7 @@ function CreateTemplateForm({
source: "custom", source: "custom",
attack_procedure: "", attack_procedure: "",
expected_detection: "", expected_detection: "",
detect_suggested_procedure: "",
platform: "", platform: "",
tool_suggested: "", tool_suggested: "",
severity: "", severity: "",
@@ -1491,6 +1492,20 @@ function CreateTemplateForm({
/> />
</div> </div>
{/* Detect Suggested Procedure */}
<div>
<label className="block text-sm font-medium text-gray-300 mb-1">
Detect Suggested Procedure
</label>
<textarea
value={form.detect_suggested_procedure || ""}
onChange={(e) => setForm({ ...form, detect_suggested_procedure: e.target.value })}
placeholder="Steps for the blue team to detect this technique..."
rows={3}
className="w-full rounded-lg border border-gray-700 bg-gray-800 px-3 py-2 text-sm text-gray-300 placeholder-gray-500 focus:border-cyan-500 focus:outline-none"
/>
</div>
{/* Tool Suggested */} {/* Tool Suggested */}
<div> <div>
<label className="block text-sm font-medium text-gray-300 mb-1"> <label className="block text-sm font-medium text-gray-300 mb-1">
@@ -1565,6 +1580,7 @@ function TemplateDetailModal({
description: template.description ?? "", description: template.description ?? "",
attack_procedure: template.attack_procedure ?? "", attack_procedure: template.attack_procedure ?? "",
expected_detection: template.expected_detection ?? "", expected_detection: template.expected_detection ?? "",
detect_suggested_procedure: template.detect_suggested_procedure ?? "",
platform: template.platform ?? "", platform: template.platform ?? "",
tool_suggested: template.tool_suggested ?? "", tool_suggested: template.tool_suggested ?? "",
severity: template.severity ?? "", severity: template.severity ?? "",
@@ -1696,6 +1712,16 @@ function TemplateDetailModal({
/> />
</div> </div>
<div>
<label className="block text-sm font-medium text-gray-300 mb-1">Detect Suggested Procedure</label>
<textarea
value={form.detect_suggested_procedure ?? ""}
onChange={(e) => setForm({ ...form, detect_suggested_procedure: e.target.value })}
rows={3}
className="w-full rounded-lg border border-gray-700 bg-gray-800 px-3 py-2 text-sm text-gray-300 focus:border-cyan-500 focus:outline-none"
/>
</div>
<div> <div>
<label className="block text-sm font-medium text-gray-300 mb-1">Suggested Tool</label> <label className="block text-sm font-medium text-gray-300 mb-1">Suggested Tool</label>
<input <input
+4
View File
@@ -116,12 +116,14 @@ export default function TestDetailPage() {
detection_time: string; detection_time: string;
containment_time: string; containment_time: string;
blue_summary: string; blue_summary: string;
detect_procedure: string;
}>({ }>({
detection_result: "", detection_result: "",
containment_result: "", containment_result: "",
detection_time: "", detection_time: "",
containment_time: "", containment_time: "",
blue_summary: "", blue_summary: "",
detect_procedure: "",
}); });
const [toast, setToast] = useState<{ message: string; type: "success" | "error" } | null>(null); const [toast, setToast] = useState<{ message: string; type: "success" | "error" } | null>(null);
@@ -185,6 +187,7 @@ export default function TestDetailPage() {
detection_time: isoToDatetimeLocal(test.detection_time), detection_time: isoToDatetimeLocal(test.detection_time),
containment_time: isoToDatetimeLocal(test.containment_time), containment_time: isoToDatetimeLocal(test.containment_time),
blue_summary: test.blue_summary || "", blue_summary: test.blue_summary || "",
detect_procedure: test.detect_procedure || "",
}); });
} }
}, [test]); }, [test]);
@@ -242,6 +245,7 @@ export default function TestDetailPage() {
detection_time: datetimeLocalToIso(blueDraft.detection_time), detection_time: datetimeLocalToIso(blueDraft.detection_time),
containment_time: datetimeLocalToIso(blueDraft.containment_time), containment_time: datetimeLocalToIso(blueDraft.containment_time),
blue_summary: blueDraft.blue_summary || undefined, blue_summary: blueDraft.blue_summary || undefined,
detect_procedure: blueDraft.detect_procedure || undefined,
}), }),
onSuccess: () => { onSuccess: () => {
invalidateAll(); invalidateAll();
+83 -1
View File
@@ -1,5 +1,5 @@
import { useState, useMemo } from "react"; import { useState, useMemo } from "react";
import { useQuery } from "@tanstack/react-query"; import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
import { useNavigate } from "react-router-dom"; import { useNavigate } from "react-router-dom";
import { import {
Loader2, Loader2,
@@ -19,8 +19,14 @@ import {
ChevronsUpDown, ChevronsUpDown,
Timer, Timer,
AlertTriangle, AlertTriangle,
Lightbulb,
} from "lucide-react"; } from "lucide-react";
import { getTests, type TestListFilters } from "../api/tests"; import { getTests, type TestListFilters } from "../api/tests";
import {
getProcedureSuggestions,
approveProcedureSuggestion,
rejectProcedureSuggestion,
} from "../api/procedure-suggestions";
import type { Test, TestState } from "../types/models"; import type { Test, TestState } from "../types/models";
import { useAuth } from "../context/AuthContext"; import { useAuth } from "../context/AuthContext";
@@ -707,6 +713,12 @@ export default function TestsPage() {
</div> </div>
)} )}
{/* ── Procedure suggestions for leads ───────────────────────────
GET /procedure-suggestions is already scoped server-side to the
caller's own team (red_lead sees only red, blue_lead only blue),
so no client-side filtering is needed here. */}
{isReviewLead && <ProcedureSuggestionsPanel />}
{/* ── Team queue overview for leads ───────────────────────────── {/* ── Team queue overview for leads ─────────────────────────────
red_lead/blue_lead never hit the plain "All Tests" branch above — red_lead/blue_lead never hit the plain "All Tests" branch above —
they're always routed into the review two-queue view or their they're always routed into the review two-queue view or their
@@ -742,6 +754,76 @@ export default function TestsPage() {
); );
} }
/* ── Procedure suggestions panel (leads only) ─────────────────────── */
function ProcedureSuggestionsPanel() {
const queryClient = useQueryClient();
const { data: suggestions = [] } = useQuery({
queryKey: ["procedure-suggestions"],
queryFn: getProcedureSuggestions,
});
const invalidate = () => queryClient.invalidateQueries({ queryKey: ["procedure-suggestions"] });
const approveMutation = useMutation({ mutationFn: approveProcedureSuggestion, onSuccess: invalidate });
const rejectMutation = useMutation({ mutationFn: rejectProcedureSuggestion, onSuccess: invalidate });
const isBusy = approveMutation.isPending || rejectMutation.isPending;
if (suggestions.length === 0) return null;
return (
<div className="rounded-xl border border-gray-800 bg-gray-900 p-6">
<div className="mb-4 flex items-center gap-2">
<Lightbulb className="h-5 w-5 text-yellow-400" />
<h2 className="text-lg font-semibold text-white">Procedure Suggestions</h2>
<span className="rounded-full border border-gray-600 bg-gray-800 px-2 py-0.5 text-xs font-medium text-gray-300">
{suggestions.length}
</span>
<span className="text-xs text-gray-500">
Commands extracted from submitted rounds, proposed as template updates
</span>
</div>
<div className="space-y-3">
{suggestions.map((s) => (
<div key={s.id} className="rounded-lg border border-gray-700 bg-gray-800/50 p-4">
<p className="text-sm font-medium text-gray-200">{s.template_name || "Unknown template"}</p>
{s.template_current_text && (
<div className="mt-2">
<p className="text-xs font-medium uppercase text-gray-500">Current</p>
<pre className="mt-1 whitespace-pre-wrap rounded bg-gray-900 p-2 font-mono text-xs text-gray-400">
{s.template_current_text}
</pre>
</div>
)}
<div className="mt-2">
<p className="text-xs font-medium uppercase text-gray-500">Suggested</p>
<pre className="mt-1 whitespace-pre-wrap rounded bg-gray-900 p-2 font-mono text-xs text-green-400">
{s.suggested_text}
</pre>
</div>
<div className="mt-3 flex gap-2">
<button
onClick={() => approveMutation.mutate(s.id)}
disabled={isBusy}
className="flex items-center gap-1 rounded-lg border border-green-500/30 bg-green-900/20 px-3 py-1.5 text-xs font-medium text-green-400 hover:bg-green-900/40 disabled:opacity-50"
>
<CheckCircle className="h-3.5 w-3.5" /> Approve
</button>
<button
onClick={() => rejectMutation.mutate(s.id)}
disabled={isBusy}
className="flex items-center gap-1 rounded-lg border border-red-500/30 bg-red-900/20 px-3 py-1.5 text-xs font-medium text-red-400 hover:bg-red-900/40 disabled:opacity-50"
>
<XCircle className="h-3.5 w-3.5" /> Reject
</button>
</div>
</div>
))}
</div>
</div>
);
}
/* ── Shared test table component ────────────────────────────────────── */ /* ── Shared test table component ────────────────────────────────────── */
function TestTable({ function TestTable({
+21
View File
@@ -90,6 +90,7 @@ export interface Test {
tool_used: string | null; tool_used: string | null;
execution_date: string | null; execution_date: string | null;
created_by: string | null; created_by: string | null;
source_template_id: string | null;
result: TestResult | null; result: TestResult | null;
state: TestState; state: TestState;
created_at: string; created_at: string;
@@ -107,6 +108,7 @@ export interface Test {
// Blue Team fields // Blue Team fields
blue_summary: string | null; blue_summary: string | null;
detect_procedure: string | null;
detection_result: TestResult | null; detection_result: TestResult | null;
containment_result: ContainmentResult | null; containment_result: ContainmentResult | null;
detection_time: string | null; detection_time: string | null;
@@ -196,6 +198,7 @@ export interface TestRoundHistory {
detection_time: string | null; detection_time: string | null;
containment_time: string | null; containment_time: string | null;
blue_summary: string | null; blue_summary: string | null;
detect_procedure: string | null;
review_notes: string | null; review_notes: string | null;
reviewed_by: string | null; reviewed_by: string | null;
archived_at: string | null; archived_at: string | null;
@@ -227,6 +230,7 @@ export interface TestTemplate {
source_url: string | null; source_url: string | null;
attack_procedure: string | null; attack_procedure: string | null;
expected_detection: string | null; expected_detection: string | null;
detect_suggested_procedure: string | null;
platform: string | null; platform: string | null;
tool_suggested: string | null; tool_suggested: string | null;
severity: string | null; severity: string | null;
@@ -236,6 +240,23 @@ export interface TestTemplate {
created_at: string; created_at: string;
} }
// ── Procedure suggestions (review queue) ────────────────────────────
export interface ProcedureSuggestion {
id: string;
template_id: string;
team: TeamSide;
suggested_text: string;
source_test_id: string | null;
submitted_by: string | null;
status: "pending" | "approved" | "rejected";
reviewed_by: string | null;
reviewed_at: string | null;
created_at: string | null;
template_name: string | null;
template_current_text: string | null;
}
export interface TestTemplateSummary { export interface TestTemplateSummary {
id: string; id: string;
mitre_technique_id: string; mitre_technique_id: string;