fix(qa): 5 bug fixes — audit dates, CSP, template modal, MITRE sync timeout, data source auto-sync
Some checks failed
Aegis CI / lint-and-test (push) Has been cancelled
Some checks failed
Aegis CI / lint-and-test (push) Has been cancelled
- audit_service: set timestamp=datetime.now(utc) explicitly so DB never stores NULL - AuditLogPage: formatDate handles null/undefined timestamps (was showing Jan 1 1970) - nginx.conf: add CSP script-src hash for inline script (sha256-31OgE8E9...) - system.py: MITRE sync now runs in BackgroundTasks — returns immediately, no more 120s timeout - mitre_sync_job.py: add _run_data_sources_sync job (every 6h) that checks sync_frequency and auto-syncs overdue enabled data sources - SystemPage: MITRE sync result shows "started" vs "complete" message - test-templates.ts: add updateTemplate() API function - SystemPage: template name cell is now clickable — opens TemplateDetailModal with full edit form (name, description, procedure, detection, platform, severity, tool) and Save / Activate / Deactivate / Close buttons Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@ sessions.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
@@ -124,6 +125,61 @@ def _run_osint_enrichment() -> None:
|
||||
db.close()
|
||||
|
||||
|
||||
_FREQUENCY_INTERVALS: dict[str, timedelta] = {
|
||||
"daily": timedelta(days=1),
|
||||
"weekly": timedelta(weeks=1),
|
||||
"monthly": timedelta(days=30),
|
||||
}
|
||||
|
||||
|
||||
def _run_data_sources_sync() -> None:
|
||||
"""Check all enabled data sources and sync those that are overdue."""
|
||||
logger.info("Scheduled data sources sync check starting...")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
from app.models.data_source import DataSource
|
||||
from app.services.data_source_service import sync_source
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
sources = (
|
||||
db.query(DataSource)
|
||||
.filter(DataSource.is_enabled == True) # noqa: E712
|
||||
.all()
|
||||
)
|
||||
synced = 0
|
||||
for ds in sources:
|
||||
freq = ds.sync_frequency
|
||||
if not freq or freq == "manual":
|
||||
continue
|
||||
interval = _FREQUENCY_INTERVALS.get(freq)
|
||||
if interval is None:
|
||||
continue
|
||||
last = ds.last_sync_at
|
||||
if last is None:
|
||||
# Never synced — run it now
|
||||
overdue = True
|
||||
else:
|
||||
# Make last timezone-aware if needed
|
||||
if last.tzinfo is None:
|
||||
last = last.replace(tzinfo=timezone.utc)
|
||||
overdue = now - last >= interval
|
||||
if overdue:
|
||||
logger.info(
|
||||
"Data source '%s' is overdue (freq=%s, last=%s) — syncing",
|
||||
ds.name, freq, last,
|
||||
)
|
||||
try:
|
||||
sync_source(db, str(ds.id))
|
||||
synced += 1
|
||||
except Exception:
|
||||
logger.exception("Failed to sync data source '%s'", ds.name)
|
||||
logger.info("Data sources sync check finished — %d source(s) synced", synced)
|
||||
except Exception:
|
||||
logger.exception("Data sources sync check failed")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _run_stale_detection() -> None:
|
||||
"""Execute daily stale coverage detection inside its own DB session."""
|
||||
logger.info("Scheduled stale coverage detection starting...")
|
||||
@@ -226,11 +282,19 @@ def start_scheduler() -> None:
|
||||
name="Data retention policies (daily)",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
_run_data_sources_sync,
|
||||
trigger="interval",
|
||||
hours=6,
|
||||
id="data_sources_sync",
|
||||
name="Data sources auto-sync (every 6h)",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.start()
|
||||
logger.info(
|
||||
"Background scheduler started — mitre_sync (24h), intel_scan (7d), "
|
||||
"notification_cleanup (24h), weekly_snapshot (Sundays 00:00), "
|
||||
"recurring_campaigns (daily), jira_sync (1h), "
|
||||
"osint_enrichment (weekly), stale_detection (daily), "
|
||||
"retention_policies (daily)"
|
||||
"retention_policies (daily), data_sources_sync (6h)"
|
||||
)
|
||||
|
||||
@@ -7,10 +7,10 @@ scheduler health introspection.
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.database import SessionLocal, get_db
|
||||
from app.dependencies.auth import require_role
|
||||
from app.models.user import User
|
||||
from app.services.mitre_sync_service import sync_mitre
|
||||
@@ -24,25 +24,39 @@ logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/system", tags=["system"])
|
||||
|
||||
|
||||
def _bg_mitre_sync() -> None:
|
||||
"""Run MITRE sync in a background task with its own DB session."""
|
||||
logger.info("Background MITRE sync task starting...")
|
||||
db = SessionLocal()
|
||||
try:
|
||||
summary = sync_mitre(db)
|
||||
logger.info("Background MITRE sync task finished — %s", summary)
|
||||
except Exception:
|
||||
logger.exception("Background MITRE sync task failed")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@router.post("/sync-mitre")
|
||||
@limiter.limit("2/hour")
|
||||
def trigger_mitre_sync(
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
background_tasks: BackgroundTasks,
|
||||
current_user: User = Depends(require_role("admin")),
|
||||
):
|
||||
"""Manually trigger a MITRE ATT&CK synchronisation.
|
||||
"""Manually trigger a MITRE ATT&CK synchronisation in the background.
|
||||
|
||||
**Requires** the ``admin`` role.
|
||||
|
||||
Returns a JSON object with the sync summary including the count of
|
||||
new and updated techniques.
|
||||
Returns immediately — the sync runs asynchronously. Poll
|
||||
``/system/scheduler-status`` for progress, or check server logs.
|
||||
"""
|
||||
summary = sync_mitre(db)
|
||||
background_tasks.add_task(_bg_mitre_sync)
|
||||
return {
|
||||
"message": "MITRE sync completed",
|
||||
"new": summary["created"],
|
||||
"updated": summary["updated"],
|
||||
"message": "MITRE sync started in background",
|
||||
"status": "started",
|
||||
"new": 0,
|
||||
"updated": 0,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -58,6 +58,7 @@ def log_action(
|
||||
ip_address=ip or None,
|
||||
user_agent=ua or None,
|
||||
session_id=session_id,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
)
|
||||
db.add(entry)
|
||||
db.flush()
|
||||
|
||||
Reference in New Issue
Block a user