fix(evaluations): correct fallback rounds + friendlier error messages
Some checks failed
Aegis CI / lint-and-test (push) Has been cancelled

- Fallback names now use hyphens matching live API (carbanak-fin7, wizard-spider-sandworm)
- Add APT3 (R1) and Enterprise 2025/er7 (R7) to fallback - verified from live API
- Remove OilRig (R6) from fallback - CrowdStrike did not participate in Round 6
- Orange fallback banner only shows when NO rounds are available at all
- Soft gray note when rounds are loaded but API had transient error
- Check-new and import errors: detect 502/Cloudflare messages and show user-friendly text
  instead of raw Cloudflare HTML error messages

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
kitos
2026-06-05 16:24:06 +02:00
parent a4cdc06534
commit cbaa0deedd
2 changed files with 56 additions and 21 deletions

View File

@@ -64,10 +64,20 @@ _HEADERS = {
# ---------------------------------------------------------------------------
# Fallback: hardcoded public CrowdStrike ENTERPRISE rounds
# Used when evals.mitre.org API is unreachable (Cloudflare, outage, etc.)
# These are well-known, publicly documented rounds — safe to hardcode.
# Used when evals.mitre.org API is unreachable (Cloudflare 502, outage, etc.)
#
# Names use the EXACT slugs the live API returns (hyphens, not underscores).
# Verified from live API response on 2025-06-05.
# CrowdStrike did NOT participate in Round 6 (OilRig) — not included.
# ---------------------------------------------------------------------------
_FALLBACK_ROUNDS: list[dict[str, Any]] = [
{
"name": "apt3",
"display_name": "APT3",
"eval_round": 1,
"domain": "ENTERPRISE",
"status": "PUBLIC",
},
{
"name": "apt29",
"display_name": "APT29",
@@ -76,14 +86,14 @@ _FALLBACK_ROUNDS: list[dict[str, Any]] = [
"status": "PUBLIC",
},
{
"name": "carbanak_fin7",
"name": "carbanak-fin7",
"display_name": "Carbanak+FIN7",
"eval_round": 3,
"domain": "ENTERPRISE",
"status": "PUBLIC",
},
{
"name": "wizard_spider_sandworm",
"name": "wizard-spider-sandworm",
"display_name": "Wizard Spider + Sandworm",
"eval_round": 4,
"domain": "ENTERPRISE",
@@ -97,9 +107,9 @@ _FALLBACK_ROUNDS: list[dict[str, Any]] = [
"status": "PUBLIC",
},
{
"name": "oilrig",
"display_name": "OilRig",
"eval_round": 6,
"name": "er7",
"display_name": "Enterprise 2025",
"eval_round": 7,
"domain": "ENTERPRISE",
"status": "PUBLIC",
},

View File

@@ -438,18 +438,24 @@ export default function SystemPage() {
</div>
</div>
{/* API fallback warning */}
{!evalApiReachable && evalApiError && (
{/* API fallback warning — only when no rounds loaded at all */}
{!evalApiReachable && evalApiError && (!evalRounds || evalRounds.length === 0) && (
<div className="mb-4 flex items-start gap-3 rounded-lg border border-orange-500/30 bg-orange-900/10 p-3">
<AlertCircle className="h-4 w-4 text-orange-400 flex-shrink-0 mt-0.5" />
<div className="text-xs text-orange-300">
<span className="font-medium">Live API unavailable</span>
{" — "}showing known public rounds (hardcoded). Importing will attempt to fetch
live results and may also fail if the API remains unreachable.
<span className="font-medium">MITRE Evaluations API temporarily unavailable</span>
{" — "}could not load round list. The server may be overloaded. Try again later.
<span className="block mt-1 text-orange-400/70 font-mono truncate">{evalApiError}</span>
</div>
</div>
)}
{/* Soft note when using fallback data but rounds are still visible */}
{!evalApiReachable && evalRounds && evalRounds.length > 0 && (
<div className="mb-4 flex items-center gap-2 rounded-lg border border-gray-700 bg-gray-800/40 px-3 py-2 text-xs text-gray-400">
<AlertCircle className="h-3.5 w-3.5 text-gray-500 flex-shrink-0" />
Showing cached round list live API had a transient error. Import may still work; retry if it fails.
</div>
)}
{/* New round check result */}
{evalCheckResult && (
@@ -461,9 +467,15 @@ export default function SystemPage() {
: "border-gray-700 bg-gray-800/50"
}`}>
{evalCheckResult.error ? (
<div className="flex items-center gap-2 text-sm text-red-400">
<XCircle className="h-4 w-4 flex-shrink-0" />
<span>Check failed: {evalCheckResult.error}</span>
<div className="flex items-start gap-2 text-sm">
<XCircle className="h-4 w-4 flex-shrink-0 text-red-400 mt-0.5" />
<div>
<span className="text-red-400">
{evalCheckResult.error.includes("502") || evalCheckResult.error.includes("Cloudflare") || evalCheckResult.error.includes("origin")
? "MITRE Evaluations API temporarily unavailable (server overload). Try again in a few minutes."
: `Check failed: ${evalCheckResult.error}`}
</span>
</div>
</div>
) : evalCheckResult.new_round_available ? (
<div className="flex items-center gap-2 text-sm">
@@ -514,12 +526,25 @@ export default function SystemPage() {
{/* Import error */}
{(importLatestMutation.isError || importRoundMutation.isError) && (
<div className="mb-4 rounded-lg border border-red-500/30 bg-red-900/20 p-3">
<div className="flex items-center gap-2 text-sm text-red-400">
<XCircle className="h-4 w-4 flex-shrink-0" />
<span>
{((importLatestMutation.error || importRoundMutation.error) as Error)?.message ??
"Import failed. This round may already be imported."}
<div className="flex items-start gap-2 text-sm">
<XCircle className="h-4 w-4 flex-shrink-0 text-red-400 mt-0.5" />
<div>
{(() => {
const msg = ((importLatestMutation.error || importRoundMutation.error) as Error)?.message ?? "";
if (msg.includes("502") || msg.includes("Cloudflare") || msg.includes("origin") || msg.includes("Connection")) {
return (
<span className="text-red-400">
MITRE Evaluations API is temporarily unavailable (server overload or maintenance).
The import fetches live results from evals.mitre.org please try again in a few minutes.
</span>
);
}
if (msg.includes("already imported") || msg.includes("409")) {
return <span className="text-yellow-400">This round has already been imported.</span>;
}
return <span className="text-red-400">{msg || "Import failed. This round may already be imported."}</span>;
})()}
</div>
</div>
</div>
)}