feat(tests): add Blue detect_procedure field mapped from template suggestion

Blue Team gets a detect_procedure field on Test (what they actually
did to detect the attack), Blue's counterpart to Red's procedure_text.
It's seeded from a new detect_suggested_procedure field on
TestTemplate at test-creation time, so a junior who later reuses the
same template starts with prior guidance instead of a blank field.

Also adds the procedure_suggestions review table and Test.source_template_id,
laying the groundwork for suggesting template improvements from filled-in
procedure fields (reviewed and approved by a lead, never auto-written).
detect_procedure is archived (not cleared) on Blue reopen, matching
blue_summary, and now appears in the Jira round-archived and blue_review
comments alongside the existing detection/containment fields.
This commit is contained in:
kitos
2026-07-14 14:58:29 +02:00
parent fd94e55799
commit 8bdbe48fbe
11 changed files with 127 additions and 1 deletions
@@ -0,0 +1,58 @@
"""Add detect_procedure fields and procedure_suggestions review table.
Blue Team gets a "Detect Procedure" field (what they actually did to
detect the attack) mirroring Red's procedure_text, plus a matching
"suggested" field on test_templates. Commands extracted from either
side's procedure text are proposed as template improvements via a new
procedure_suggestions table, reviewed and approved/rejected by a lead
rather than written automatically.
Revision ID: b062
Revises: b061
Create Date: 2026-07-14
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision = "b062"
down_revision = "b061"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("tests", sa.Column("detect_procedure", sa.Text(), nullable=True))
op.add_column(
"tests",
sa.Column("source_template_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("test_templates.id"), nullable=True),
)
op.add_column("test_templates", sa.Column("detect_suggested_procedure", sa.Text(), nullable=True))
op.add_column("test_round_history", sa.Column("detect_procedure", sa.Text(), nullable=True))
op.create_table(
"procedure_suggestions",
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
sa.Column("template_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("test_templates.id"), nullable=False),
sa.Column("team", sa.String(10), nullable=False),
sa.Column("suggested_text", sa.Text(), nullable=False),
sa.Column("source_test_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("tests.id"), nullable=True),
sa.Column("submitted_by", postgresql.UUID(as_uuid=True), sa.ForeignKey("users.id"), nullable=True),
sa.Column("status", sa.String(10), nullable=False, server_default="pending"),
sa.Column("reviewed_by", postgresql.UUID(as_uuid=True), sa.ForeignKey("users.id"), nullable=True),
sa.Column("reviewed_at", sa.DateTime(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
)
op.create_index("ix_procedure_suggestions_template_id", "procedure_suggestions", ["template_id"])
op.create_index("ix_procedure_suggestions_status", "procedure_suggestions", ["status"])
def downgrade() -> None:
op.drop_index("ix_procedure_suggestions_status", table_name="procedure_suggestions")
op.drop_index("ix_procedure_suggestions_template_id", table_name="procedure_suggestions")
op.drop_table("procedure_suggestions")
op.drop_column("test_round_history", "detect_procedure")
op.drop_column("test_templates", "detect_suggested_procedure")
op.drop_column("tests", "source_template_id")
op.drop_column("tests", "detect_procedure")
+2
View File
@@ -45,6 +45,7 @@ from app.models.technique import Technique
from app.models.test import Test from app.models.test import Test
from app.models.test_round_history import TestRoundHistory from app.models.test_round_history import TestRoundHistory
from app.models.test_template import TestTemplate from app.models.test_template import TestTemplate
from app.models.procedure_suggestion import ProcedureSuggestion
from app.models.user import User from app.models.user import User
# Assign __all__ = [ # Assign __all__ = [
@@ -88,4 +89,5 @@ __all__ = [
"AlertRule", "AlertRule",
"AlertInstance", "AlertInstance",
"TestRoundHistory", "TestRoundHistory",
"ProcedureSuggestion",
] ]
@@ -0,0 +1,41 @@
"""SQLAlchemy model for procedure-improvement suggestions.
When an operator submits a round with a filled-in procedure field
(``procedure_text`` for Red, ``detect_procedure`` for Blue), the command(s)
in it are extracted heuristically and proposed as an update to the
originating template's suggested-procedure field. Nothing is written to
the template automatically — a lead reviews and approves or rejects each
suggestion, so a junior who later picks up the same template only ever
sees vetted guidance.
"""
import uuid
from sqlalchemy import Column, DateTime, ForeignKey, String, Text, func
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import relationship
from app.database import Base
class ProcedureSuggestion(Base):
"""A proposed update to a TestTemplate's suggested procedure, pending lead review."""
__tablename__ = "procedure_suggestions"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
template_id = Column(UUID(as_uuid=True), ForeignKey("test_templates.id"), nullable=False)
team = Column(String(10), nullable=False) # "red" or "blue"
suggested_text = Column(Text, nullable=False)
source_test_id = Column(UUID(as_uuid=True), ForeignKey("tests.id"), nullable=True)
submitted_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
status = Column(String(10), nullable=False, default="pending", server_default="pending") # pending/approved/rejected
reviewed_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
reviewed_at = Column(DateTime, nullable=True)
created_at = Column(DateTime(timezone=True), server_default=func.now())
template = relationship("TestTemplate", foreign_keys=[template_id])
source_test = relationship("Test", foreign_keys=[source_test_id])
submitter = relationship("User", foreign_keys=[submitted_by])
reviewer = relationship("User", foreign_keys=[reviewed_by])
+9
View File
@@ -59,6 +59,12 @@ class Test(Base):
execution_date = Column(DateTime, nullable=True) execution_date = Column(DateTime, nullable=True)
# Assign created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) # Assign created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True)
# The template this test was instantiated from, if any (null for
# standalone/manually-created/RT-imported tests). Lets a procedure
# suggestion know exactly which template to propose improving —
# matching purely by technique would be ambiguous when a technique has
# multiple templates.
source_template_id = Column(UUID(as_uuid=True), ForeignKey("test_templates.id"), nullable=True)
# Assign result = Column(Enum(TestResult, name="testresult"), nullable=True) # Assign result = Column(Enum(TestResult, name="testresult"), nullable=True)
result = Column(Enum(TestResult, name="testresult"), nullable=True) result = Column(Enum(TestResult, name="testresult"), nullable=True)
# Assign state = Column(Enum(TestState, name="teststate"), default=TestState.draft) # Assign state = Column(Enum(TestState, name="teststate"), default=TestState.draft)
@@ -83,6 +89,9 @@ class Test(Base):
# ── Blue Team fields ──────────────────────────────────────────── # ── Blue Team fields ────────────────────────────────────────────
blue_summary = Column(Text, nullable=True) blue_summary = Column(Text, nullable=True)
# What Blue actually did to detect the attack — Blue's counterpart to
# procedure_text. Free text; may be parsed for a procedure suggestion.
detect_procedure = Column(Text, nullable=True)
# Assign detection_result = Column(Enum(TestResult, name="testresult"), nullable=True) # Assign detection_result = Column(Enum(TestResult, name="testresult"), nullable=True)
detection_result = Column(Enum(TestResult, name="testresult"), nullable=True) detection_result = Column(Enum(TestResult, name="testresult"), nullable=True)
containment_result = Column(Enum(ContainmentResult, name="containmentresult"), nullable=True) containment_result = Column(Enum(ContainmentResult, name="containmentresult"), nullable=True)
+1
View File
@@ -50,6 +50,7 @@ class TestRoundHistory(Base):
detection_time = Column(DateTime, nullable=True) detection_time = Column(DateTime, nullable=True)
containment_time = Column(DateTime, nullable=True) containment_time = Column(DateTime, nullable=True)
blue_summary = Column(Text, nullable=True) blue_summary = Column(Text, nullable=True)
detect_procedure = Column(Text, nullable=True)
# Why the round was closed # Why the round was closed
review_notes = Column(Text, nullable=True) review_notes = Column(Text, nullable=True)
+5
View File
@@ -43,6 +43,11 @@ class TestTemplate(Base):
attack_procedure = Column(Text, nullable=True) # Suggested attack procedure attack_procedure = Column(Text, nullable=True) # Suggested attack procedure
# Assign expected_detection = Column(Text, nullable=True) # What blue team should detect # Assign expected_detection = Column(Text, nullable=True) # What blue team should detect
expected_detection = Column(Text, nullable=True) # What blue team should detect expected_detection = Column(Text, nullable=True) # What blue team should detect
# Suggested detection procedure — Blue's counterpart to attack_procedure.
# Only ever filled in via an approved procedure suggestion or a lead
# editing the template directly; external syncs never touch it (those
# only insert brand-new rows, never update existing ones).
detect_suggested_procedure = Column(Text, nullable=True)
# Assign platform = Column(String, nullable=True) # windows / linux... # Assign platform = Column(String, nullable=True) # windows / linux...
platform = Column(String, nullable=True) # windows / linux / macos platform = Column(String, nullable=True) # windows / linux / macos
# Assign tool_suggested = Column(String, nullable=True) # Assign tool_suggested = Column(String, nullable=True)
+4
View File
@@ -115,6 +115,7 @@ class TestBlueUpdate(BaseModel):
containment_time: datetime | None = None containment_time: datetime | None = None
# Assign blue_summary = None # Assign blue_summary = None
blue_summary: str | None = None blue_summary: str | None = None
detect_procedure: str | None = None
_normalize_datetimes = field_validator("detection_time", "containment_time")(_to_naive_utc) _normalize_datetimes = field_validator("detection_time", "containment_time")(_to_naive_utc)
@@ -142,6 +143,7 @@ class TestRoundHistoryOut(BaseModel):
detection_time: datetime | None = None detection_time: datetime | None = None
containment_time: datetime | None = None containment_time: datetime | None = None
blue_summary: str | None = None blue_summary: str | None = None
detect_procedure: str | None = None
review_notes: str | None = None review_notes: str | None = None
reviewed_by: uuid.UUID | None = None reviewed_by: uuid.UUID | None = None
archived_at: datetime | None = None archived_at: datetime | None = None
@@ -269,6 +271,7 @@ class TestOut(BaseModel):
execution_date: datetime | None = None execution_date: datetime | None = None
# Assign created_by = None # Assign created_by = None
created_by: uuid.UUID | None = None created_by: uuid.UUID | None = None
source_template_id: uuid.UUID | None = None
# Assign result = None # Assign result = None
result: TestResult | None = None result: TestResult | None = None
# Assign state = TestState.draft # Assign state = TestState.draft
@@ -293,6 +296,7 @@ class TestOut(BaseModel):
# Blue Team fields # Blue Team fields
blue_summary: str | None = None blue_summary: str | None = None
detect_procedure: str | None = None
# Assign detection_result = None # Assign detection_result = None
detection_result: TestResult | None = None detection_result: TestResult | None = None
containment_result: ContainmentResult | None = None containment_result: ContainmentResult | None = None
+2
View File
@@ -31,6 +31,7 @@ class TestTemplateOut(BaseModel):
attack_procedure: str | None = None attack_procedure: str | None = None
# Assign expected_detection = None # Assign expected_detection = None
expected_detection: str | None = None expected_detection: str | None = None
detect_suggested_procedure: str | None = None
# Assign platform = None # Assign platform = None
platform: str | None = None platform: str | None = None
# Assign tool_suggested = None # Assign tool_suggested = None
@@ -70,6 +71,7 @@ class TestTemplateCreate(BaseModel):
attack_procedure: str | None = None attack_procedure: str | None = None
# Assign expected_detection = None # Assign expected_detection = None
expected_detection: str | None = None expected_detection: str | None = None
detect_suggested_procedure: str | None = None
# Assign platform = None # Assign platform = None
platform: str | None = None platform: str | None = None
# Assign tool_suggested = None # Assign tool_suggested = None
+2
View File
@@ -486,6 +486,7 @@ def _build_state_comment(
f"Blue Team has submitted evidence for Round {test.blue_round_number or 1} and the " f"Blue Team has submitted evidence for Round {test.blue_round_number or 1} and the "
"test is awaiting Blue Lead review before cross-validation.", "test is awaiting Blue Lead review before cross-validation.",
"", "",
f"*Detect Procedure:* {test.detect_procedure or 'N/A'}",
f"*Detection Result:* {_enum_value(test.detection_result) or 'N/A'}", f"*Detection Result:* {_enum_value(test.detection_result) or 'N/A'}",
f"*Containment Result:* {_enum_value(test.containment_result) or 'N/A'}", f"*Containment Result:* {_enum_value(test.containment_result) or 'N/A'}",
] ]
@@ -1129,6 +1130,7 @@ def push_round_archived(db: Session, test, actor, *, round_data) -> None:
lines.append(f"*Attack success:* {round_data.attack_success.value if round_data.attack_success else '-'}") lines.append(f"*Attack success:* {round_data.attack_success.value if round_data.attack_success else '-'}")
lines.append(f"*Summary:* {round_data.red_summary or '-'}") lines.append(f"*Summary:* {round_data.red_summary or '-'}")
else: else:
lines.append(f"*Detect procedure:* {round_data.detect_procedure or '_none recorded_'}")
lines.append(f"*Detection result:* {round_data.detection_result.value if round_data.detection_result else '-'}") lines.append(f"*Detection result:* {round_data.detection_result.value if round_data.detection_result else '-'}")
lines.append(f"*Containment result:* {round_data.containment_result.value if round_data.containment_result else '-'}") lines.append(f"*Containment result:* {round_data.containment_result.value if round_data.containment_result else '-'}")
lines.append(f"*Summary:* {round_data.blue_summary or '-'}") lines.append(f"*Summary:* {round_data.blue_summary or '-'}")
@@ -364,9 +364,11 @@ def create_test_from_template(
platform=platform_override if platform_override is not None else template.platform, platform=platform_override if platform_override is not None else template.platform,
procedure_text=procedure_text_override if procedure_text_override is not None else template.attack_procedure, procedure_text=procedure_text_override if procedure_text_override is not None else template.attack_procedure,
tool_used=tool_used_override if tool_used_override is not None else template.tool_suggested, tool_used=tool_used_override if tool_used_override is not None else template.tool_suggested,
detect_procedure=template.detect_suggested_procedure,
remediation_steps=template.suggested_remediation, remediation_steps=template.suggested_remediation,
# Keyword argument: created_by # Keyword argument: created_by
created_by=creator_id, created_by=creator_id,
source_template_id=template.id,
# Keyword argument: state # Keyword argument: state
state=TestState.draft, state=TestState.draft,
created_at=datetime.utcnow(), # explicit — DB column has no server default created_at=datetime.utcnow(), # explicit — DB column has no server default
@@ -623,7 +623,7 @@ def reopen_blue_review(db: Session, test: Test, user: User, notes: str) -> Test:
paused_seconds=test.blue_paused_seconds or 0, paused_seconds=test.blue_paused_seconds or 0,
detection_result=test.detection_result, containment_result=test.containment_result, detection_result=test.detection_result, containment_result=test.containment_result,
detection_time=test.detection_time, containment_time=test.containment_time, detection_time=test.detection_time, containment_time=test.containment_time,
blue_summary=test.blue_summary, blue_summary=test.blue_summary, detect_procedure=test.detect_procedure,
review_notes=notes.strip(), reviewed_by=user.id, review_notes=notes.strip(), reviewed_by=user.id,
) )
db.add(archived_round) db.add(archived_round)