feat(tests): load-balanced reviewer selection and Jira reviewer sync

This commit is contained in:
kitos
2026-07-06 10:56:40 +02:00
parent c41876b62f
commit 22be620665
3 changed files with 456 additions and 12 deletions
+30
View File
@@ -307,7 +307,9 @@ _SEVERITY_TO_PRIORITY: dict[str, str] = {
_STATE_EMOJI: dict[str, str] = {
"draft": "📝 Draft",
"red_executing": "🔴 Red Team Executing",
"red_review": "🔎 Red Lead Review",
"blue_evaluating": "🔵 Blue Team Evaluating",
"blue_review": "🔎 Blue Lead Review",
"in_review": "📋 In Review",
"validated": "✅ Validated",
"rejected": "❌ Rejected",
@@ -370,6 +372,18 @@ def _build_state_comment(
"Red Team has started the attack execution.",
]
elif new_state == "red_review":
lines += [
"Red Team has submitted evidence and the test is awaiting Red Lead review "
"before it queues for Blue Team.",
]
elif new_state == "blue_review":
lines += [
"Blue Team has submitted evidence and the test is awaiting Blue Lead review "
"before cross-validation.",
]
elif new_state == "blue_evaluating":
lines += [
"Red Team has finished execution and submitted evidence for Blue Team evaluation.",
@@ -667,6 +681,7 @@ def push_test_event(
new_state: str,
*,
extra: dict | None = None,
assignee: User | None = None,
) -> None:
"""Post a lifecycle comment to the Jira issue linked to *test*.
@@ -720,6 +735,21 @@ def push_test_event(
link.jira_issue_key, jira_account_id, exc_a,
)
if new_state in ("red_review", "blue_review") and assignee:
jira_account_id = getattr(assignee, "jira_account_id", None)
if jira_account_id:
try:
jira.assign_issue(link.jira_issue_key, account_id=jira_account_id)
logger.info(
"Assigned Jira ticket %s to reviewer account %s",
link.jira_issue_key, jira_account_id,
)
except Exception as exc_a:
logger.warning(
"Could not assign %s to reviewer %s: %s",
link.jira_issue_key, jira_account_id, exc_a,
)
link.last_synced_at = datetime.utcnow()
db.flush()
logger.info(
+243 -7
View File
@@ -26,7 +26,7 @@ from sqlalchemy.orm import Session
# Import settings from app.config
from app.config import settings
from app.domain.exceptions import InvalidOperationError
from app.domain.exceptions import BusinessRuleViolation, InvalidOperationError
from app.domain.test_entity import TestEntity
from app.models.enums import TestState, TeamSide
from app.models.evidence import Evidence
@@ -263,7 +263,7 @@ def submit_red_evidence(db: Session, test: Test, user: User) -> Test:
# Assign test = transition_state(
test = transition_state(
db, test, TestState.blue_evaluating, user,
db, test, TestState.red_review, user,
# Keyword argument: action_name
action_name="submit_red_evidence",
)
@@ -287,10 +287,52 @@ def submit_red_evidence(db: Session, test: Test, user: User) -> Test:
description=f"Red Team execution: {test.name}",
)
# Start Blue Team timer
test.blue_started_at = now
# Assign test.blue_paused_seconds = 0
test.blue_paused_seconds = 0
reviewer = select_reviewer(
db, role="red_lead",
exclude_user_id=user.id if user.role == "red_lead" else None,
)
test.red_reviewer_assignee = reviewer.id
db.flush()
try:
create_notification(
db, user_id=reviewer.id, type="review_assigned",
title="Test awaiting your review",
message=f'Test "{test.name}" is waiting for your red-team review.',
entity_type="test", entity_id=test.id,
)
except Exception as e:
logger.warning("Notification failed for test %s: %s", test.id, e, exc_info=True)
try:
from app.services.jira_service import push_test_event
push_test_event(db, test, user, "red_review", assignee=reviewer)
except Exception as e:
logger.warning("Jira push failed for test %s: %s", test.id, e, exc_info=True)
return test
def approve_red_review(db: Session, test: Test, user: User, notes: str | None = None) -> Test:
"""Red Lead approves the operator's work — moves red_review to blue_evaluating."""
entity = TestEntity.from_orm(test)
entity.approve_red_review()
entity.apply_to(test)
test.red_review_by = user.id
test.red_review_at = datetime.utcnow()
test.red_review_notes = notes
db.flush()
log_action(
db, user_id=user.id, action="approve_red_review",
entity_type="test", entity_id=test.id,
details={"notes": notes, "test_name": test.name},
)
try:
notify_test_state_change(db, test, "blue_evaluating")
except Exception as e:
logger.warning("Notification failed for test %s: %s", test.id, e, exc_info=True)
try:
from app.services.jira_service import push_test_event
@@ -301,6 +343,39 @@ def submit_red_evidence(db: Session, test: Test, user: User) -> Test:
return test
def reopen_red_review(db: Session, test: Test, user: User, notes: str) -> Test:
"""Red Lead sends the operator's work back for rework — moves red_review to red_executing."""
if not notes or not notes.strip():
raise InvalidOperationError("A comment is required when reopening a test for rework")
entity = TestEntity.from_orm(test)
entity.reopen_red_review()
entity.apply_to(test)
test.red_review_by = user.id
test.red_review_at = datetime.utcnow()
test.red_review_notes = notes.strip()
db.flush()
log_action(
db, user_id=user.id, action="reopen_red_review",
entity_type="test", entity_id=test.id,
details={"notes": notes, "test_name": test.name},
)
if test.red_tech_assignee:
try:
create_notification(
db, user_id=test.red_tech_assignee, type="test_reopened",
title="Test sent back for rework",
message=f'Test "{test.name}" was sent back by your Red Lead: {notes.strip()[:200]}',
entity_type="test", entity_id=test.id,
)
except Exception as e:
logger.warning("Notification failed for test %s: %s", test.id, e, exc_info=True)
return test
def start_blue_work(db: Session, test: Test, user: User) -> Test:
"""Mark that a blue tech has picked up this test to start evaluating.
@@ -368,7 +443,7 @@ def submit_blue_evidence(db: Session, test: Test, user: User) -> Test:
# Assign test = transition_state(
test = transition_state(
db, test, TestState.in_review, user,
db, test, TestState.blue_review, user,
# Keyword argument: action_name
action_name="submit_blue_evidence",
)
@@ -392,6 +467,121 @@ def submit_blue_evidence(db: Session, test: Test, user: User) -> Test:
description=f"Blue Team evaluation: {test.name}",
)
reviewer = select_reviewer(
db, role="blue_lead",
exclude_user_id=user.id if user.role == "blue_lead" else None,
)
test.blue_reviewer_assignee = reviewer.id
db.flush()
try:
create_notification(
db, user_id=reviewer.id, type="review_assigned",
title="Test awaiting your review",
message=f'Test "{test.name}" is waiting for your blue-team review.',
entity_type="test", entity_id=test.id,
)
except Exception as e:
logger.warning("Notification failed for test %s: %s", test.id, e, exc_info=True)
try:
from app.services.jira_service import push_test_event
push_test_event(db, test, user, "blue_review", assignee=reviewer)
except Exception as e:
logger.warning("Jira push failed for test %s: %s", test.id, e, exc_info=True)
return test
def approve_blue_review(db: Session, test: Test, user: User, notes: str | None = None) -> Test:
"""Blue Lead approves the operator's work — moves blue_review to in_review."""
entity = TestEntity.from_orm(test)
entity.approve_blue_review()
entity.apply_to(test)
test.blue_review_by = user.id
test.blue_review_at = datetime.utcnow()
test.blue_review_notes = notes
db.flush()
log_action(
db, user_id=user.id, action="approve_blue_review",
entity_type="test", entity_id=test.id,
details={"notes": notes, "test_name": test.name},
)
try:
notify_test_state_change(db, test, "in_review")
except Exception as e:
logger.warning("Notification failed for test %s: %s", test.id, e, exc_info=True)
try:
from app.services.jira_service import push_test_event
push_test_event(db, test, user, "in_review")
except Exception as e:
logger.warning("Jira push failed for test %s: %s", test.id, e, exc_info=True)
return test
def reopen_blue_review(db: Session, test: Test, user: User, notes: str) -> Test:
"""Blue Lead sends the operator's work back for rework — moves blue_review to blue_evaluating."""
if not notes or not notes.strip():
raise InvalidOperationError("A comment is required when reopening a test for rework")
entity = TestEntity.from_orm(test)
entity.reopen_blue_review()
entity.apply_to(test)
test.blue_work_started_at = None # split responsibility: entity doesn't own this field
test.blue_review_by = user.id
test.blue_review_at = datetime.utcnow()
test.blue_review_notes = notes.strip()
db.flush()
log_action(
db, user_id=user.id, action="reopen_blue_review",
entity_type="test", entity_id=test.id,
details={"notes": notes, "test_name": test.name},
)
if test.blue_tech_assignee:
try:
create_notification(
db, user_id=test.blue_tech_assignee, type="test_reopened",
title="Test sent back for rework",
message=f'Test "{test.name}" was sent back by your Blue Lead: {notes.strip()[:200]}',
entity_type="test", entity_id=test.id,
)
except Exception as e:
logger.warning("Notification failed for test %s: %s", test.id, e, exc_info=True)
return test
def flag_blue_review_gap(db: Session, test: Test, user: User, system_gaps: str, notes: str | None = None) -> Test:
"""Blue Lead flags a capability gap — proceeds to in_review anyway (a retry can't fix a missing tool)."""
if not system_gaps or not system_gaps.strip():
raise InvalidOperationError("system_gaps description is required when flagging a capability gap")
entity = TestEntity.from_orm(test)
entity.flag_blue_review_gap()
entity.apply_to(test)
test.blue_review_by = user.id
test.blue_review_at = datetime.utcnow()
test.blue_review_notes = notes
test.system_gaps = system_gaps.strip()
db.flush()
log_action(
db, user_id=user.id, action="flag_blue_review_gap",
entity_type="test", entity_id=test.id,
details={"system_gaps": system_gaps, "notes": notes, "test_name": test.name},
)
try:
notify_test_state_change(db, test, "in_review")
except Exception as e:
logger.warning("Notification failed for test %s: %s", test.id, e, exc_info=True)
try:
from app.services.jira_service import push_test_event
push_test_event(db, test, user, "in_review")
@@ -502,6 +692,52 @@ def resume_timer(db: Session, test: Test, user: User) -> Test:
return test
def select_reviewer(
db: Session,
*,
role: str,
exclude_user_id: uuid.UUID | None = None,
) -> User:
"""Pick the least-loaded active user with *role* to review a test.
Load is measured as the count of tests currently sitting in the
matching review state (``red_review`` for role ``red_lead``,
``blue_review`` for role ``blue_lead``) with that user set as the
reviewer. Ties broken by username for determinism.
Raises BusinessRuleViolation if no eligible reviewer exists (e.g. the
only lead is the person who executed the test, or there are no leads
with this role at all).
"""
review_state = TestState.red_review if role == "red_lead" else TestState.blue_review
reviewer_field = Test.red_reviewer_assignee if role == "red_lead" else Test.blue_reviewer_assignee
candidates_query = db.query(User).filter(User.role == role, User.is_active == True) # noqa: E712
if exclude_user_id is not None:
candidates_query = candidates_query.filter(User.id != exclude_user_id)
candidates = candidates_query.order_by(User.username).all()
if not candidates:
raise BusinessRuleViolation(
f"No available {role} to review this test (cannot self-review, "
f"and no other {role} is active)"
)
best_user = None
best_count = None
for candidate in candidates:
count = (
db.query(Test)
.filter(Test.state == review_state, reviewer_field == candidate.id)
.count()
)
if best_count is None or count < best_count:
best_user = candidate
best_count = count
return best_user
# Define function _create_phase_worklog
def _create_phase_worklog(
# Entry: db
+183 -5
View File
@@ -10,6 +10,7 @@ without requiring a running database.
import sys
import os
import uuid
import pytest
from unittest.mock import MagicMock, patch
from types import ModuleType
from datetime import datetime
@@ -100,13 +101,16 @@ for _mod in [
from fastapi import HTTPException
from app.domain.exceptions import InvalidOperationError, InvalidTransitionError
from app.models.enums import TestState, TestResult
from app.models.test import Test
from app.services.test_workflow_service import (
VALID_TRANSITIONS,
can_transition,
transition_state,
start_execution,
submit_red_evidence,
approve_red_review,
submit_blue_evidence,
approve_blue_review,
validate_as_red_lead,
validate_as_blue_lead,
check_dual_validation,
@@ -158,27 +162,33 @@ def _make_db() -> MagicMock:
# ===========================================================================
@patch("app.services.test_workflow_service.select_reviewer")
@patch("app.services.test_workflow_service.log_action")
def test_full_happy_path(mock_log):
"""draft -> red_executing -> blue_evaluating -> in_review -> validated"""
def test_full_happy_path(mock_log, mock_select_reviewer):
"""draft -> red_executing -> red_review -> blue_evaluating -> blue_review -> in_review -> validated"""
test = _make_test(TestState.draft)
red_tech = _make_user("red_tech")
blue_tech = _make_user("blue_tech")
red_lead = _make_user("red_lead")
blue_lead = _make_user("blue_lead")
db = _make_db()
mock_select_reviewer.side_effect = [red_lead, blue_lead]
# Step 1: draft -> red_executing
result = start_execution(db, test, red_tech)
assert result.state == TestState.red_executing
assert result.execution_date is not None
# Step 2: red_executing -> blue_evaluating
# Step 2: red_executing -> red_review -> (Red Lead approves) -> blue_evaluating
result = submit_red_evidence(db, result, red_tech)
assert result.state == TestState.red_review
result = approve_red_review(db, result, red_lead)
assert result.state == TestState.blue_evaluating
# Step 3: blue_evaluating -> in_review
# Step 3: blue_evaluating -> blue_review -> (Blue Lead approves) -> in_review
result = submit_blue_evidence(db, result, blue_tech)
assert result.state == TestState.blue_review
result = approve_blue_review(db, result, blue_lead)
assert result.state == TestState.in_review
# Step 4: Red Lead approves
@@ -205,19 +215,24 @@ def test_full_happy_path(mock_log):
# ===========================================================================
@patch("app.services.test_workflow_service.select_reviewer")
@patch("app.services.test_workflow_service.log_action")
def test_rejection_and_reopen(mock_log):
def test_rejection_and_reopen(mock_log, mock_select_reviewer):
"""in_review -> rejected -> draft -> red_executing -> ..."""
test = _make_test(TestState.draft)
red_tech = _make_user("red_tech")
blue_tech = _make_user("blue_tech")
red_lead = _make_user("red_lead")
blue_lead = _make_user("blue_lead")
db = _make_db()
mock_select_reviewer.side_effect = [red_lead, blue_lead]
# Advance to in_review
start_execution(db, test, red_tech)
submit_red_evidence(db, test, red_tech)
approve_red_review(db, test, red_lead)
submit_blue_evidence(db, test, blue_tech)
approve_blue_review(db, test, blue_lead)
assert test.state == TestState.in_review
# Red Lead rejects -> rejected
@@ -577,6 +592,169 @@ def test_cannot_reopen_non_rejected_test(mock_log):
# Run all
# ---------------------------------------------------------------------------
# ===========================================================================
# 12b. Review-decision functions (approve/reopen/gap)
# ===========================================================================
class TestReviewDecisions:
@patch("app.services.test_workflow_service.log_action")
def test_approve_red_review_moves_to_blue_evaluating(self, mock_log):
test = _make_test(TestState.red_review)
reviewer = _make_user("red_lead")
db = _make_db()
from app.services.test_workflow_service import approve_red_review
result = approve_red_review(db, test, reviewer, notes="looks good")
assert result.state == TestState.blue_evaluating
assert result.red_review_by == reviewer.id
assert result.red_review_notes == "looks good"
assert result.blue_started_at is not None
def test_reopen_red_review_requires_notes(self):
test = _make_test(TestState.red_review)
reviewer = _make_user("red_lead")
db = _make_db()
from app.services.test_workflow_service import reopen_red_review
with pytest.raises(InvalidOperationError):
reopen_red_review(db, test, reviewer, notes="")
@patch("app.services.test_workflow_service.log_action")
def test_reopen_red_review_moves_to_red_executing_with_notes(self, mock_log):
test = _make_test(TestState.red_review, red_tech_assignee=uuid.uuid4())
reviewer = _make_user("red_lead")
db = _make_db()
from app.services.test_workflow_service import reopen_red_review
result = reopen_red_review(db, test, reviewer, notes="add more detail")
assert result.state == TestState.red_executing
assert result.red_review_notes == "add more detail"
@patch("app.services.test_workflow_service.log_action")
def test_approve_blue_review_moves_to_in_review(self, mock_log):
test = _make_test(TestState.blue_review)
reviewer = _make_user("blue_lead")
db = _make_db()
from app.services.test_workflow_service import approve_blue_review
result = approve_blue_review(db, test, reviewer)
assert result.state == TestState.in_review
def test_reopen_blue_review_requires_notes(self):
test = _make_test(TestState.blue_review)
reviewer = _make_user("blue_lead")
db = _make_db()
from app.services.test_workflow_service import reopen_blue_review
with pytest.raises(InvalidOperationError):
reopen_blue_review(db, test, reviewer, notes=None)
@patch("app.services.test_workflow_service.log_action")
def test_reopen_blue_review_moves_to_blue_evaluating(self, mock_log):
test = _make_test(TestState.blue_review, blue_tech_assignee=uuid.uuid4())
reviewer = _make_user("blue_lead")
db = _make_db()
from app.services.test_workflow_service import reopen_blue_review
result = reopen_blue_review(db, test, reviewer, notes="redo it")
assert result.state == TestState.blue_evaluating
assert result.blue_work_started_at is None
def test_flag_blue_review_gap_requires_system_gaps_text(self):
test = _make_test(TestState.blue_review)
reviewer = _make_user("blue_lead")
db = _make_db()
from app.services.test_workflow_service import flag_blue_review_gap
with pytest.raises(InvalidOperationError):
flag_blue_review_gap(db, test, reviewer, system_gaps="")
@patch("app.services.test_workflow_service.log_action")
def test_flag_blue_review_gap_moves_to_in_review(self, mock_log):
test = _make_test(TestState.blue_review)
reviewer = _make_user("blue_lead")
db = _make_db()
from app.services.test_workflow_service import flag_blue_review_gap
result = flag_blue_review_gap(db, test, reviewer, system_gaps="Missing EDR agent on host X")
assert result.state == TestState.in_review
assert result.system_gaps == "Missing EDR agent on host X"
# ===========================================================================
# 13. select_reviewer — load-balanced reviewer assignment
# ===========================================================================
class TestReviewerSelection:
"""Uses the real sqlite `db` fixture from conftest.py (not MagicMock),
since load-balancing needs real COUNT() queries."""
def _make_lead(self, db, username, role="red_lead"):
from app.models.user import User
u = User(username=username, role=role, hashed_password="x", is_active=True)
db.add(u)
db.flush()
return u
def _make_technique(self, db, mitre_id="T1059"):
from app.models.technique import Technique
t = Technique(mitre_id=mitre_id, name="Command Line", tactic="execution")
db.add(t)
db.flush()
return t
def test_picks_lead_with_fewest_active_reviews(self, db):
from app.services.test_workflow_service import select_reviewer
lead_a = self._make_lead(db, "reda_selrev")
lead_b = self._make_lead(db, "redb_selrev")
tech = self._make_technique(db, "T1059.selrev1")
busy_test = Test(
technique_id=tech.id, name="Busy",
state=TestState.red_review, red_reviewer_assignee=lead_a.id,
)
db.add(busy_test)
db.commit()
chosen = select_reviewer(db, role="red_lead")
assert chosen.id == lead_b.id
def test_excludes_the_submitter_if_they_are_a_lead(self, db):
from app.services.test_workflow_service import select_reviewer
from app.domain.exceptions import BusinessRuleViolation
lead_a = self._make_lead(db, "reda_excl")
db.commit()
with pytest.raises(BusinessRuleViolation, match="No available"):
select_reviewer(db, role="red_lead", exclude_user_id=lead_a.id)
def test_no_candidates_raises_clear_error(self, db):
from app.services.test_workflow_service import select_reviewer
from app.domain.exceptions import BusinessRuleViolation
with pytest.raises(BusinessRuleViolation, match="No available"):
select_reviewer(db, role="red_lead")
def test_ties_broken_by_username(self, db):
from app.services.test_workflow_service import select_reviewer
self._make_lead(db, "zzz_tie", role="blue_lead")
self._make_lead(db, "aaa_tie", role="blue_lead")
db.commit()
chosen = select_reviewer(db, role="blue_lead")
assert chosen.username == "aaa_tie"
if __name__ == "__main__":
print("T-125 Validation: Workflow Tests")
print("=" * 55)