{
  "module": "S06 — Secure Code Review Harnesses",
  "course": "2A — Building AI Harnesses for Cybersecurity",
  "version": "1.0.0",
  "duration_minutes": 45,
  "total_questions": 20,
  "bloom_distribution": {
    "target": "20% recall / 40% application / 40% analysis-design",
    "actual": { "recall": 4, "application": 8, "analysis": 8 }
  },
  "passing_score_percent": 70,
  "questions": [
    { "id": "Q01", "bloom": "recall", "type": "multiple_choice", "prompt": "Name the 5 stages of the layered code review pipeline in order.", "options": ["LLM → Semgrep → CodeQL → AST → synthesis", "AST → Semgrep → CodeQL → LLM → synthesis", "Semgrep → LLM → AST → CodeQL → synthesis", "Synthesis → AST → Semgrep → CodeQL → LLM"], "answer_index": 1, "rationale": "AST parsing (syntax/symbols) → Semgrep (fast pattern rules) → CodeQL (deep data-flow) → LLM (semantic judgement, FP triage) → synthesis (dedup, route). Deterministic first, LLM last." },
    { "id": "Q02", "bloom": "recall", "type": "multiple_choice", "prompt": "What does the LLM triage layer ask about each raw finding?", "options": ["Whether the finding matches a new CWE", "Is the input reachable from untrusted data? Is there a sanitizer? TP or FP? Confidence?", "Whether the scanner ran correctly", "Whether to delete the file"], "answer_index": 1, "rationale": "The model judges reachability, sanitization, verdict (TP/FP), and confidence — with evidence. It does not find the finding; the scanners did. Reader/actor separation." },
    { "id": "Q03", "bloom": "recall", "type": "multiple_choice", "prompt": "What precision/recall shift does LLM triage produce versus raw Semgrep?", "options": ["Raw: 40% recall / 95% precision → triaged: 95% recall / 40% precision", "Raw: ~95% recall / ~40% precision → triaged: ~85% recall / ~92% precision", "No change — triage only affects speed", "Raw: 92% precision → triaged: 40% precision"], "answer_index": 1, "rationale": "Raw scanners are recall machines (~95% recall, ~40% precision). LLM triage trades a small recall loss for a large precision gain (~85% recall, ~92% precision). Worth the trade." },
    { "id": "Q04", "bloom": "recall", "type": "multiple_choice", "prompt": "What does a CodeUnit in the semantic codebase index store?", "options": ["Only the file name and line number", "id, file, symbol, signature, source, callers, callees, embedding", "Only the embedding vector", "Only the CWE classification"], "answer_index": 1, "rationale": "CodeUnit carries metadata (file, symbol, signature), the source body, the static call graph edges (callers, callees), and the embedding vector — enabling both graph traversal and semantic retrieval." },
    { "id": "Q05", "bloom": "application", "type": "multiple_choice", "prompt": "A finding is critical severity, high post-triage confidence. What routing action?", "options": ["File silently in the dashboard", "Auto-comment inline only", "Block the PR (status check fails)", "Discard the finding"], "answer_index": 2, "rationale": "Critical/high severity with high confidence → block the PR. Blocking requires both high severity AND high post-triage confidence to avoid over-blocking, which trains developers to click override." },
    { "id": "Q06", "bloom": "application", "type": "multiple_choice", "prompt": "A human marks a finding as a false positive. What happens next in the harness?", "options": ["Nothing — the verdict is discarded", "The verdict is stored in the feedback store and retrieved as a few-shot example for future triage of similar findings", "The Semgrep rule is deleted", "The LLM is retrained from scratch"], "answer_index": 1, "rationale": "The feedback loop: human verdicts go into a vector index. Future triage retrieves similar past verdicts as few-shot examples, improving precision on that finding class over weeks — a learning harness." },
    { "id": "Q07", "bloom": "application", "type": "multiple_choice", "prompt": "A SQL injection finding is in repository.py, but the tainted source enters in handler.py. What does the harness need to judge it correctly?", "options": ["A stronger Semgrep rule on repository.py", "Cross-file context from the semantic codebase memory — callers (source), callees (sink), sanitizers across file boundaries", "A faster LLM", "Manual code review only"], "answer_index": 1, "rationale": "Single-file analysis misses cross-file taint flows. The semantic index retrieves the call graph context (callers, callees, sanitizers) across files so the triage model can judge reachability and sanitization." },
    { "id": "Q08", "bloom": "application", "type": "multiple_choice", "prompt": "A generated patch passes linter, Semgrep re-scan, and tests. What is the next required gate?", "options": ["Auto-merge — all automated gates passed", "Patch quality score above threshold, then a DRAFT PR for human approval", "Nothing — open a direct commit", "Re-run CodeQL only"], "answer_index": 1, "rationale": "After the automated gates (linter, Semgrep, tests), the patch quality score must clear threshold, then a draft PR opens. Human approval is the final, non-negotiable gate. Auto-merge is an anti-pattern." },
    { "id": "Q09", "bloom": "application", "type": "multiple_choice", "prompt": "Your triage pipeline surfaces 100 findings; 8 are real. What is the precision, and what does it mean?", "options": ["8% precision — almost all surfaced findings are false positives; the operator is buried in noise", "92% precision — excellent", "100% precision — perfect", "8% recall — missing real vulns"], "answer_index": 0, "rationale": "Precision = TP/(TP+FP) = 8/100 = 8%. Of surfaced findings, only 8% are real. Below ~80% precision buries the operator in false positives. The LLM triage layer exists to fix this." },
    { "id": "Q10", "bloom": "application", "type": "multiple_choice", "prompt": "The semantic memory retrieves 30 functions for a finding, totaling 25k tokens. The triage prompt budget is 6k. What does the harness do?", "options": ["Send all 25k tokens — the model needs full context", "Truncate by priority: direct callers/callees and sanitizers first, transitive context last, within the 6k budget", "Increase the budget to 25k", "Skip retrieval entirely"], "answer_index": 1, "rationale": "Context budget management: prioritize direct callers/callees (define the taint path) and sanitizers (resolve FP/TP) over transitive context. Too-large prompts are expensive and dilute the model's attention." },
    { "id": "Q11", "bloom": "application", "type": "multiple_choice", "prompt": "A developer wants to auto-merge high-confidence patches to save review time. What is the correct response?", "options": ["Agree — high-confidence patches are safe to auto-merge", "Refuse — human approval is non-negotiable. LLM patches can introduce new vulns while fixing the reported one. Draft PR, human merge, always", "Agree, but only for low-severity findings", "Agree, but only on weekends"], "answer_index": 1, "rationale": "The approval gate is non-negotiable. One auto-merged regression that ships a CVE outweighs thousands of saved clicks. The deterministic gates catch some regressions, not all — the human is the last line of defense." },
    { "id": "Q12", "bloom": "application", "type": "multiple_choice", "prompt": "A patch fixes the reported finding but introduces a new Semgrep finding elsewhere. What happens?", "options": ["Merge it — the original finding is resolved", "The Semgrep re-scan gate fails; the patch is discarded or revised", "File it for later review", "Open the PR anyway and let the human catch it"], "answer_index": 1, "rationale": "Gate 2 (Semgrep re-scan) requires the finding resolved AND no new findings introduced. A patch that introduces a new weakness fails the gate and is discarded or sent back for revision." },
    { "id": "Q13", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why is running the LLM first (over the raw diff) an anti-pattern?", "options": ["It is too cheap", "Slow, expensive, inconsistent across runs, hallucination-prone. The LLM's strength is judgement on a small bounded set, not brute-force scanning", "It produces too few findings", "The LLM cannot read diffs"], "answer_index": 1, "rationale": "LLM-first scanning misuses the model. Deterministic scanners give recall cheaply and deterministically; the LLM gives precision on the filtered candidate set. Running the LLM first inverts the strengths." },
    { "id": "Q14", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why does the synthesis layer dedup across scanners before routing?", "options": ["To reduce token cost", "Semgrep and CodeQL may flag the same weakness at the same location with different CWEs. Dedup (via dedup_key) presents one finding, not duplicates, to the developer", "To make the pipeline faster", "Because CodeQL is always wrong"], "answer_index": 1, "rationale": "Cross-scanner dedup uses a dedup_key to merge overlapping findings (e.g. Semgrep and CodeQL both flag the same SQL injection). Without it, the developer sees the same issue twice from different tools." },
    { "id": "Q15", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why is the LLM triage model SEPARATE from any model that might have suggested the finding?", "options": ["For performance", "Confirmation bias — the finder's enthusiasm biases the assessment. A separate triage model provides independent judgement (reader/actor separation, S01.3/S02.4)", "To reduce cost", "It is required by Semgrep"], "answer_index": 1, "rationale": "The system that finds a finding should not score it. Separation prevents confirmation bias — the finder's enthusiasm for the finding does not bias the severity/exploitability assessment." },
    { "id": "Q16", "bloom": "analysis", "type": "multiple_choice", "prompt": "Compare production precision vs production recall measurability. Which is correct?", "options": ["Both are directly measurable in production", "Precision is measurable (from human feedback on surfaced findings). Recall is unknowable — you don't know about vulns you missed. Labeled datasets are the proxy", "Recall is measurable; precision is not", "Neither is measurable"], "answer_index": 1, "rationale": "Production precision is measurable: humans verdict the surfaced findings. Production recall is unknowable by definition (missed vulns are unseen). Labeled datasets (OWASP Benchmark, Juliet) validate recall periodically." },
    { "id": "Q17", "bloom": "analysis", "type": "multiple_choice", "prompt": "A team posts every Semgrep finding as an inline PR comment. What principle did they violate?", "options": ["Scope enforcement", "Confidence gating — raw scanner output at scale is low-precision; surfacing all findings buries the PR in noise. Only high-confidence findings should surface as comments", "Hash chaining", "Rate limiting"], "answer_index": 1, "rationale": "The surface-all-findings anti-pattern. Raw Semgrep precision is ~40%; posting every finding trains developers to ignore the bot. Confidence gating routes only high-confidence findings to PR comments; the rest file silently." },
    { "id": "Q18", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why is the semantic codebase index rebuilt incrementally per PR, not fully?", "options": ["Full reindex is impossible", "A full reindex of a large monorepo is expensive. Only changed functions and their callers/callees need re-embedding; tying reindex to the git diff keeps cost bounded", "Incremental is more accurate", "The vector DB does not support full reindex"], "answer_index": 1, "rationale": "Cost control. A full reindex on every PR is prohibitive for a large monorepo. Incremental reindex (changed functions + their call-graph neighbors) bounds cost while keeping the index current where it matters." },
    { "id": "Q19", "bloom": "analysis", "type": "multiple_choice", "prompt": "Your pipeline has 92% precision but 50% recall at the high-confidence threshold. What is the operational risk and fix?", "options": ["No risk — high precision is enough", "Half of real vulns are missed (filed as low-confidence/discarded). The threshold is too strict; relax it to raise recall toward 85%, accepting slightly lower precision", "Precision is too high", "Recall does not matter for code review"], "answer_index": 1, "rationale": "50% recall means half of real vulnerabilities are missed — dangerous. The high-confidence threshold is too aggressive. Tune the threshold to recover recall (~85%) while keeping precision acceptable (~92%)." },
    { "id": "Q20", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why does the patch generation prompt constrain the model to a minimal, targeted diff?", "options": ["To save tokens", "Minimal diff = smaller regression surface, easier human review, lower chance of introducing new findings. The model resolves the specific weakness only — no refactor/rename/reformat", "Because the model cannot write large diffs", "To speed up the linter"], "answer_index": 1, "rationale": "A minimal targeted diff constrains the patch to resolving the finding. Larger diffs (refactors, reformats) increase the regression surface, complicate human review, and raise the chance of introducing new weaknesses." }
  ]
}