{
  "module": "S03 — Pentest and Red Team Harnesses",
  "course": "2A — Building AI Harnesses for Cybersecurity",
  "version": "1.0.0",
  "duration_minutes": 45,
  "total_questions": 20,
  "bloom_distribution": {
    "target": "20% recall / 40% application / 40% analysis-design",
    "actual": { "recall": 4, "application": 8, "analysis": 8 }
  },
  "passing_score_percent": 70,
  "questions": [
    { "id": "Q01", "bloom": "recall", "type": "multiple_choice", "prompt": "Name the five states an attack graph node can be in.", "options": ["open, closed, filtered, pending, done", "unexplored, attempted, succeeded, failed, out_of_scope", "new, active, complete, error, skipped", "todo, in_progress, review, blocked, cancelled"], "answer_index": 1, "rationale": "Attack graph nodes carry MITRE ATT&CK tactic/technique labels and transition through: unexplored → attempted → succeeded/failed/out_of_scope. out_of_scope is terminal (node exists for completeness but no attempts made)." },
    { "id": "Q02", "bloom": "recall", "type": "multiple_choice", "prompt": "Name the four steps of the RedTeamLLM plan correction loop.", "options": ["Scan, analyze, exploit, report", "Detect failure → re-hypothesize → re-plan → retry with alternative", "Try, fail, escalate, abort", "Recon, plan, execute, verify"], "answer_index": 1, "rationale": "RedTeamLLM (Brown et al., 2024): detect hard failure, re-hypothesize (query knowledge base for alternatives), re-plan (mark old node failed, add new unexplored node), retry the NEW node. Never retries the failed technique." },
    { "id": "Q03", "bloom": "recall", "type": "multiple_choice", "prompt": "Name the two independent boundaries of dual containment.", "options": ["Firewall and antivirus", "Outbound scope enforcement + inbound isolation", "Authentication and authorization", "Encryption and hashing"], "answer_index": 1, "rationale": "Outbound scope enforcement: egress proxy + firewall ACL from scope.json — agent connects only to in-scope targets at the network layer. Inbound isolation: no inbound routes from target net to container/host — reverse shells terminate in container. Both required, independent by design." },
    { "id": "Q04", "bloom": "recall", "type": "multiple_choice", "prompt": "What are the three output formats of the report generator, and why one source of truth?", "options": ["Word, Excel, PowerPoint — for Office compatibility", "HTML (interactive), PDF (static signable), JSON (machine-readable) — all from the same finding records", "TXT, CSV, XML — for parsing flexibility", "Markdown, LaTeX, HTML — for documentation systems"], "answer_index": 1, "rationale": "HTML is the interactive primary deliverable (clickable evidence), PDF for static signable artifacts, JSON for ticketing systems. All rendered from the same finding records. Separate report data from engagement data → two sources → they drift." },
    { "id": "Q05", "bloom": "application", "type": "multiple_choice", "prompt": "You are running a 3-day red team engagement with 8 sequential objectives and dependencies between them. Linear plan or attack graph?", "options": ["Linear plan — simpler to reason about", "Attack graph — multi-objective with dependencies is exactly when the graph is required", "Neither — use a flat finding list like S02", "Start with a checklist and add objectives as you go"], "answer_index": 1, "rationale": "Attack graph when multi-objective, multiple techniques per objective, dependencies. A 3-day, 8-objective engagement with dependencies is the canonical graph case. Linear plans work only for single-objective, single-technique, high-confidence tasks." },
    { "id": "Q06", "bloom": "application", "type": "multiple_choice", "prompt": "An exploit returns 'connection refused' after 200ms. Is this a hard failure or soft failure, and what does the harness do?", "options": ["Soft failure — retry with backoff", "Hard failure — the exploit genuinely didn't work. Trigger plan correction: re-hypothesize, re-plan, retry a NEW technique", "Hard failure — abort the engagement", "Indeterminate — ask the operator"], "answer_index": 1, "rationale": "'Connection refused' is a definitive no-result, not a transient timeout. This is a hard failure: target patched, port closed, or service not running. Triggers plan correction — select an alternative technique for the same tactic, never retry the same one." },
    { "id": "Q07", "bloom": "application", "type": "multiple_choice", "prompt": "Your harness has high-confidence recon: the target is running nginx 1.4.1 with a known CVE. Do you generalize or specialize the exploit?", "options": ["Generalize — broad patterns are always safer", "Specialize — high-confidence intel (exact version) is exactly when target-specific payloads win. High impact if it works", "Both simultaneously", "Generalize, then specialize if it fails"], "answer_index": 1, "rationale": "Specialize when you have high-confidence target intelligence (exact version, confirmed config). The CVE-specific payload has high impact. Save generalize as the fallback IF the specialized technique fails (specialized intel is frequently wrong in adversarial environments)." },
    { "id": "Q08", "bloom": "application", "type": "multiple_choice", "prompt": "During an engagement, the agent harvests a target database password. Where does it go?", "options": ["~/.pgpass on the host for convenience", "The host's keychain, encrypted", "A credential quarantine vault inside the container — separate key, destroyed on teardown. Never touches host stores", "A plaintext file in the agent's working directory"], "answer_index": 2, "rationale": "Credential quarantine: engagement creds live in a separate encrypted vault inside the container, own key, destroyed on teardown. Host credential stores (SSH agent, keychain, ~/.aws/credentials) are NOT mounted into the container. Enforced by config, not agent cooperation." },
    { "id": "Q09", "bloom": "application", "type": "multiple_choice", "prompt": "You configured dual containment. How do you verify outbound scope enforcement?", "options": ["Review the firewall rules and confirm they look correct", "From inside the container, connect to an out-of-scope canary host → confirm REFUSED. Connect to an in-scope target → confirm SUCCEEDS", "Ask the agent whether it can reach out-of-scope targets", "Check the Docker network configuration"], "answer_index": 1, "rationale": "Verification = attempt the forbidden thing, observe the block. Untested containment is an untested safety claim. Config review is not verification — actively attempt an out-of-scope connection and confirm it is refused at the network layer." },
    { "id": "Q10", "bloom": "application", "type": "multiple_choice", "prompt": "The report generator produces a CVSS 9.8 Critical for a finding. What must happen before this reaches the client?", "options": ["Nothing — the model is authoritative on CVSS", "Human analyst review. The score is a DRAFT, flagged 'pending analyst review'. LLMs mis-score CVSS by 2+ points regularly", "Automated cross-check against NVD, then ship", "The operator glances at it and approves"], "answer_index": 1, "rationale": "CVSS is a known LLM weakness. The harness policy: always auto-scored, always flagged 'DRAFT — pending analyst review', always routed to human review before the client. The model is never the final authority on severity." },
    { "id": "Q11", "bloom": "application", "type": "multiple_choice", "prompt": "A red team engagement pauses on day 1 with foothold achieved. How does the harness resume correctly on day 2?", "options": ["Continue yesterday's reasoning from the context window", "Load the persisted attack graph, issue an orientation prompt, rebuild reasoning from structural truth. Do NOT trust yesterday's reasoning (may be stale)", "Re-run all of day 1's scans to rebuild state", "Ask the operator to summarize day 1"], "answer_index": 1, "rationale": "The graph is serialized to disk at every state transition. On resume: load graph, orientation prompt ('here's the graph, you're at [nodes], next available: [techniques]'), verify evidence_refs resolve. Resume from structural truth — yesterday's reasoning may be based on changed target state." },
    { "id": "Q12", "bloom": "application", "type": "multiple_choice", "prompt": "Your harness's plan correction loop keeps selecting the same failed exploit with minor parameter variations. What went wrong?", "options": ["The knowledge base is too small", "Exploit lock-in — exclude_failed=True is not set in the knowledge base query, so failed techniques remain candidates. The loop varies instead of substituting", "The rate limiter is too aggressive", "The agent lacks permissions"], "answer_index": 1, "rationale": "The key invariant of plan correction: failed techniques are removed from the candidate set (exclude_failed=True). Without it, the loop retries variations of the failed technique instead of selecting a genuinely different one. This is exploit lock-in — the cure is forcing substitution." },
    { "id": "Q13", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why is the sandbox model from Course 1 (contain the agent) INVERTED for an offensive harness?", "options": ["Offensive agents are more trustworthy", "Course 1 was wrong about containment", "An offensive agent MUST reach out to attack the target. Contain it like a coding agent and it cannot work. But uncontained, a compromised agent (prompt injection, S01.3) attacks YOU. The offensive sandbox must allow outbound to targets while preventing lateral movement", "Offensive engagements don't need sandboxes"], "answer_index": 2, "rationale": "The sandbox inversion: Course 1 contains the agent to protect the host (correct for coding). Offensive requires reaching out. Remove containment and a compromised agent does lateral movement, credential exfil, internal pivot. Resolution is dual containment — outbound to targets, inbound isolation." },
    { "id": "Q14", "bloom": "analysis", "type": "multiple_choice", "prompt": "The scope middleware checks every outbound call. Why is the NETWORK LAYER (egress proxy, firewall) also required?", "options": ["For performance — the middleware is slow", "The middleware checks INTENT ('model wants nmap on X'). The network layer enforces OUTCOME ('even if middleware bypassed, the packet cannot leave'). If middleware is bypassed (misconfig, bug, injection), network layer is the last line of defense", "The network layer is required by Docker", "To reduce token usage"], "answer_index": 1, "rationale": "Belt and suspenders. Middleware and network layer enforce different properties: intent vs. outcome. If the middleware is bypassed via prompt injection or misconfiguration, the egress proxy's ACL (generated from scope.json) blocks the packet at the network layer. Both required for defense in depth." },
    { "id": "Q15", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why must the cloud metadata endpoint (169.254.169.254) be denied at the egress proxy?", "options": ["It is slow", "It is a common SSRF target and accidental-exfiltration path. If the host is a cloud instance and a compromised agent reaches it, the agent steals the HOST's cloud credentials — escalating beyond the engagement", "It causes rate limiting", "It is out of protocol"], "answer_index": 1, "rationale": "The metadata endpoint returns the host instance's cloud credentials (IAM role tokens). A compromised agent reaching it via SSRF or direct connection steals the HOST's credentials, escalating beyond the engagement scope into the host's cloud account. Always denied, every engagement." },
    { "id": "Q16", "bloom": "analysis", "type": "multiple_choice", "prompt": "The attack graph has grown dense (many nodes, many alternative paths). A teammate suggests collapsing it to a linear plan 'for simplicity.' What is the consequence?", "options": ["Good idea — simpler is better", "Harmful — the density IS the value. Collapsing throws away the ALTERNATIVE PATHS the agent will need when the primary technique fails. The graph represents fallback options; a linear plan has none", "Neutral — it doesn't matter", "Required when the graph exceeds 10 nodes"], "answer_index": 1, "rationale": "The 'collapse to linear' anti-pattern. A dense graph encodes multiple paths to each objective. When the primary technique fails, plan correction needs those alternatives. Collapsing to linear discards them — the agent will have no fallback and will abort on first failure. Density is a feature, not a bug." },
    { "id": "Q17", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why is CWE-to-OWASP control mapping a deterministic LOOKUP rather than model-generated?", "options": ["The model is too slow", "The control mapping is what the client's compliance team checks. If the model hallucinates an OWASP reference, compliance rejects the report. A lookup CANNOT hallucinate. The model fills target-specific detail; the framework mapping is deterministic", "Deterministic lookups are cheaper", "The model cannot reference OWASP"], "answer_index": 1, "rationale": "Compliance teams verify control mappings against the actual OWASP ASVS / Top 10. A hallucinated reference invalidates the finding's remediation section. The split: CWE→OWASP control is a lookup table (deterministic); target-specific remediation detail ('patch nginx to X.Y') is model-generated." },
    { "id": "Q18", "bloom": "analysis", "type": "multiple_choice", "prompt": "The report's executive summary reads: 'The engagement identified several security issues of varying severity. Remediation should be prioritized by risk.' What is wrong?", "options": ["Grammar error", "The generic-summary anti-pattern. This could describe any engagement. The summary must be grounded in the SPECIFIC finding set — name the actual top finding, cite the attack path, quote the metric. Swappable = failed", "It is too short", "It lacks a CVSS score"], "answer_index": 1, "rationale": "The executive summary is the one place prose matters most (the C-suite reads it). Generic boilerplate is useless. Cure: ground in the specific finding set. 'The engagement demonstrated Domain Admin in 4 hours via CVE-2024-XXXX chained with misconfigured Kerberos delegation' is specific. If you could swap it with another engagement's summary undetected, it failed." },
    { "id": "Q19", "bloom": "analysis", "type": "multiple_choice", "prompt": "RedTeamLLM's ablation showed harnesses with a generalize-on-failure fallback complete 40% more objectives. Why?", "options": ["Generalized attacks are more powerful", "In adversarial environments, specialized intelligence (exact version, config) is frequently WRONG. When the specialized technique fails, a generalized fallback (broad pattern) provides a second path. Harnesses that only specialize have no fallback when intel is wrong", "Generalized attacks are faster", "Specialized attacks are illegal"], "answer_index": 1, "rationale": "Specialized techniques depend on accurate intel — but recon-reported versions differ from actual, patches get applied, configs change. When specialization fails (and it frequently does), a generalize-on-failure fallback gives the harness a reliable, if noisier, second path. Harnesses without it abort on first specialized failure." },
    { "id": "Q20", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why does the plan correction loop add a NEW unexplored node instead of retrying the failed node with a different payload?", "options": ["New nodes are faster", "Retrying the failed node = variation, not substitution. The failed TECHNIQUE is removed from the candidate set (exclude_failed=True). The new node represents a genuinely different technique for the same tactic. This forces real alternatives, defeating exploit lock-in", "To keep the graph small", "Retry would break the hash chain"], "answer_index": 1, "rationale": "The invariant: failed techniques are excluded from candidates. A new node for a different technique enforces genuine substitution. Retrying the failed node with parameter tweaks is exploit lock-in — the harness has a favorite and won't let go. The new-node approach forces the knowledge base to surface a structurally different path." }
  ]
}