kitos
fd94e55799
Extracts the actual command(s)/query out of a free-text procedure field, discarding narrative and pasted output, for use in building procedure-improvement suggestions from operator submissions.
85 lines
3.7 KiB
Python
85 lines
3.7 KiB
Python
"""Heuristic extraction of the actual command(s)/query out of a free-text
|
|
procedure field (``procedure_text`` for Red, ``detect_procedure`` for
|
|
Blue), for proposing as a template's suggested procedure.
|
|
|
|
Operators write these fields as free text mixing narrative ("I ran this
|
|
against the DC and it worked"), the actual command, and sometimes pasted
|
|
output. We only want the command — not the story around it, not the
|
|
output. There's no reliable general way to do this with NLP-level
|
|
accuracy, so this uses a deterministic, conservative regex approach:
|
|
|
|
1. Fenced code blocks (```...``` or ~~~...~~~) are the strongest signal —
|
|
an operator who bothered to fence something almost always fenced the
|
|
command, not the narrative. If any exist, their contents are returned
|
|
verbatim (narrative outside the fence is ignored).
|
|
2. Otherwise, each line is checked against a curated set of patterns that
|
|
look like real commands or detection queries (shell prompts, known
|
|
binaries/cmdlets, SQL/KQL/SPL-style query syntax). Only matching lines
|
|
survive; narrative sentences and plain command output are discarded.
|
|
|
|
This is deliberately conservative — false negatives (missing a real
|
|
command written in an unrecognized style) are far less harmful than false
|
|
positives (proposing narrative or output as "the command"), especially
|
|
since a lead reviews every suggestion before it reaches a template.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
_FENCE_RE = re.compile(r"```(?:\w*\n)?(.*?)```|~~~(?:\w*\n)?(.*?)~~~", re.DOTALL)
|
|
|
|
# Leading shell/PowerShell prompt markers — stripped, not treated as
|
|
# evidence by themselves (the remainder still has to look like a command).
|
|
_PROMPT_RE = re.compile(r"^\s*(?:PS(?:\s+\S+)?>|[$#>])\s+")
|
|
|
|
# Signatures that indicate "this line is a command or query", not prose.
|
|
_COMMAND_LINE_RE = re.compile(
|
|
r"""
|
|
^\s*(?:PS(?:\s+\S+)?>|[$#>])\s*\S # explicit shell/PowerShell prompt with something after it
|
|
|
|
|
(?-i:\b[A-Z][a-zA-Z]+-[A-Z][a-zA-Z]+\b) # PowerShell Verb-Noun cmdlet, e.g. Get-Process
|
|
# (case-sensitive even though the rest of this
|
|
# pattern is IGNORECASE — otherwise lowercase
|
|
# hyphenated prose like "well-known" false-matches)
|
|
|
|
|
^\s*(?:sudo|chmod|chown|curl|wget|nmap|ssh|scp|python3?|bash|sh|zsh|
|
|
powershell(?:\.exe)?|cmd(?:\.exe)?|reg(?:\.exe)?|net(?:\.exe)?|
|
|
netsh|wmic|schtasks|sc|rundll32|regsvr32|msiexec|certutil|
|
|
mimikatz\S*|cscript|wscript|osascript|openssl|systemctl|service|
|
|
reg\s+query|whoami|nslookup|dig|ping|tasklist|ps\s|kill|
|
|
Invoke-\S+|iex|dsquery|nltest)\b
|
|
|
|
|
\bSELECT\s+.+\s+FROM\b # SQL
|
|
|
|
|
\bsearch\s+index\s*= # SPL (explicit "search" keyword)
|
|
|
|
|
\bindex\s*=\S # SPL (bare "index=value")
|
|
|
|
|
\|\s*(?:where|project|summarize|extend|join|render)\b # KQL pipe syntax
|
|
""",
|
|
re.IGNORECASE | re.VERBOSE,
|
|
)
|
|
|
|
|
|
def extract_commands(text: str | None) -> str | None:
|
|
"""Return the command/query lines found in *text*, or None if none found."""
|
|
if not text or not text.strip():
|
|
return None
|
|
|
|
fence_matches = _FENCE_RE.findall(text)
|
|
if fence_matches:
|
|
blocks = [(a or b).strip() for a, b in fence_matches if (a or b).strip()]
|
|
if blocks:
|
|
return "\n\n".join(blocks)
|
|
|
|
kept: list[str] = []
|
|
for line in text.splitlines():
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
continue
|
|
if _COMMAND_LINE_RE.search(stripped):
|
|
kept.append(_PROMPT_RE.sub("", stripped))
|
|
|
|
return "\n".join(kept) if kept else None
|