{
  "module": "S11 — Smart Contract Audit Harnesses",
  "course": "2A — Building AI Harnesses for Cybersecurity",
  "version": "1.0.0",
  "duration_minutes": 45,
  "total_questions": 20,
  "bloom_distribution": {
    "target": "20% recall / 40% application / 40% analysis-design",
    "actual": { "recall": 4, "application": 8, "analysis": 8 }
  },
  "passing_score_percent": 70,
  "questions": [
    { "id": "Q01", "bloom": "recall", "type": "multiple_choice", "prompt": "What are the three modes of a smart contract security harness as defined by EVMbench?", "options": ["Scan, Report, Remediate", "Detect, Patch, Exploit", "Recon, Test, Deploy", "Audit, Fix, Verify"], "answer_index": 1, "rationale": "EVMbench evaluates agents across Detect (find vulnerabilities), Patch (generate fixes), and Exploit (build and run PoCs). All three are scored independently." },
    { "id": "Q02", "bloom": "recall", "type": "multiple_choice", "prompt": "What does EVMbench's dataset consist of?", "options": ["50 synthetic vulnerabilities in 10 repos", "117 curated vulnerabilities across 40 EVM repositories", "1000 fuzz-generated bugs in test contracts", "25 real-world exploits from 2024"], "answer_index": 1, "rationale": "EVMbench contains 117 curated vulnerabilities from 40 real EVM repositories, spanning reentrancy, integer overflow, access control, flash loan, and business logic classes." },
    { "id": "Q03", "bloom": "recall", "type": "multiple_choice", "prompt": "What are Heimdallr's headline metrics?", "options": ["99% detection at $0.10/10K LOC", "92.45% detection at $2.31/10K LOC, 17/20 real-world attacks reconstructed", "85% detection at $10/10K LOC", "95% detection, 20/20 attacks reconstructed"], "answer_index": 1, "rationale": "Heimdallr achieved 92.45% detection rate at $2.31 per 10K LOC in contest-style auditing, and reconstructed 17 of 20 real-world attacks post-June 2025." },
    { "id": "Q04", "bloom": "recall", "type": "multiple_choice", "prompt": "What are the four cascaded verification gates for a generated patch?", "options": ["Compile, deploy, test, monitor", "Static re-check, test suite, exploit re-run, formal verification", "Lint, unit test, integration test, code review", "Slither, Mythril, Semgrep, manual review"], "answer_index": 1, "rationale": "Gate 1: Slither/Mythril on patched code (finding gone, no new issues). Gate 2: test suite / property tests pass. Gate 3: exploit PoC now fails. Gate 4: formal verification where feasible." },
    { "id": "Q05", "bloom": "application", "type": "multiple_choice", "prompt": "An LLM-only audit flags 40 external calls as reentrancy risks, but only 2 are real. What failure mode is this, and what is the cure?", "options": ["Context overflow; increase context window", "False positives on benign patterns; cure is the static-plus-LLM pipeline — static analysis produces candidates, LLM judges exploitability in context", "Hallucination on semantics; use a Solidity-trained model", "Rate limiting; slow down the analysis"], "answer_index": 1, "rationale": "LLM-only auditing produces false positives on benign patterns (checks-effects-interactions is safe but flagged). The harness pairs static analysis (structured candidates) with LLM reasoning (exploitability judgment), reducing FPs." },
    { "id": "Q06", "bloom": "application", "type": "multiple_choice", "prompt": "Your harness detects an invariant violation that requires the attacker to own 51% of the governance token. What severity and why?", "options": ["Critical — any invariant violation is critical", "Theoretical/low — not economically exploitable via flash loan; requires the attacker to already control 51% of supply. Downgraded unless a realistic path to 51% exists", "High — invariant violations are always high or above", "Informational — log it only"], "answer_index": 1, "rationale": "The LLM confirmation step distinguishes economically exploitable (flash-loan reachable = critical) from theoretical (requires 51% ownership). Severity depends on economic reachability, not just code correctness." },
    { "id": "Q07", "bloom": "application", "type": "multiple_choice", "prompt": "A protocol's withdraw function reads price from a spot DEX pool. The harness traces this data flow. What finding class and severity?", "options": ["Integer overflow, medium", "Flash loan / oracle manipulation path — critical if flash-loan reachable in a single transaction. Spot DEX prices are manipulable via flash loan", "Access control, high", "Reentrancy, critical"], "answer_index": 1, "rationale": "Spot DEX prices are manipulable in a single transaction via flash loan (borrow, swap to move price, exploit, swap back, repay). The harness flags this as a flash loan attack path. TWAP over multiple blocks would be resistant." },
    { "id": "Q08", "bloom": "application", "type": "multiple_choice", "prompt": "You generate a patch that closes a reentrancy. Gate 1 (Slither) passes. Gate 2 (test suite) fails — 3 tests broke. What happened and what do you do?", "options": ["Deploy anyway — the security fix is more important", "The patch changed intended behavior, not just security. Reject the patch; regenerate with the test failures as feedback, or escalate to human auditor", "Rewrite the tests to match the patch", "Skip gate 2 — it's optional"], "answer_index": 1, "rationale": "Gate 2 failing means the patch broke intended behavior. The patch must preserve all intended behavior while removing the vulnerability. Reject, regenerate with failure feedback, or escalate." },
    { "id": "Q09", "bloom": "application", "type": "multiple_choice", "prompt": "A patch makes the harness's PoC fail (gate 3 passes) but Slither still flags the reentrancy pattern (gate 1 fails). What does this indicate?", "options": ["The patch is correct — the PoC failing is enough", "The patch fixed the specific PoC but not the underlying vulnerability. The PoC used 10 ETH; the patch may reject exactly 10 ETH while 50 ETH still exploits. Static re-check catches what exploit re-run alone misses", "Slither is broken — ignore it", "Regenerate the PoC to match the patch"], "answer_index": 1, "rationale": "The patch-that-fixes-the-PoC trap. Exploit re-run (gate 3) is necessary but insufficient. Static re-check (gate 1) and formal verification (gate 4) catch patches that address the specific PoC but leave the underlying vulnerability." },
    { "id": "Q10", "bloom": "application", "type": "multiple_choice", "prompt": "You run an exploit PoC on a clean local chain and it succeeds. Should you report it as a confirmed finding?", "options": ["Yes — a successful PoC is sufficient evidence", "No — run it on forked mainnet first. A clean chain does not reproduce the target's actual storage, integrated token contracts, or oracle prices at the forked block. The exploit may depend on precise on-chain state", "Yes — clean chain is more reliable", "Only if the dollar impact exceeds $1M"], "answer_index": 1, "rationale": "Forked mainnet reproduces the target's actual storage and integrations. Many exploits depend on precise on-chain state (DEX liquidity, oracle prices). A clean-chain PoC may not reproduce the real conditions under which the vulnerability is exploitable." },
    { "id": "Q11", "bloom": "application", "type": "multiple_choice", "prompt": "A DeFi vault contract is 2,000 lines. How does Heimdallr's approach feed it to the LLM for analysis?", "options": ["Feed the entire 2,000 lines at once", "Function-level reorganization: split into function units, feed the LLM the relevant function + called functions + touched storage variables. Minimizes context overhead while preserving needed info", "Summarize the contract first, then analyze the summary", "Feed only the constructor and public functions"], "answer_index": 1, "rationale": "Function-level reorganization avoids context overflow and lossy compaction. The model sees the function under analysis plus its call-graph context — nothing else. This is Heimdallr's first innovation." },
    { "id": "Q12", "bloom": "application", "type": "multiple_choice", "prompt": "A protocol has no test suite. How does the harness verify that a patch preserves behavior (gate 2)?", "options": ["Skip gate 2 — no tests to run", "Generate property tests from the invariants extracted in S11.2 and run those. If the protocol has no tests, the harness creates them from the protocol's intended invariants", "Deploy the patch and monitor on-chain", "Manual review only"], "answer_index": 1, "rationale": "When no test suite exists, the harness generates property tests from extracted invariants (S11.2). These test whether the protocol's intended properties hold — the patch must not break them." },
    { "id": "Q13", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why is the LLM hallucination on Solidity described as 'systematic' rather than 'random'?", "options": ["It occurs on every contract", "The errors cluster around specific semantic mismatches — the model applies Python/JS intuitions (overflow throws, transfer forwards full gas) to opcode-precise EVM semantics (overflow wraps, transfer = 2300 gas). Not random noise; structured misapplication of general-language intuitions", "It only happens with small models", "It is random but frequent"], "answer_index": 1, "rationale": "The hallucination is systematic because the model applies general-language intuitions to a domain where those intuitions mislead. The errors cluster around specific EVM semantic features (overflow wrapping, gas forwarding, balance reading) that differ from Python/JS semantics." },
    { "id": "Q14", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why can static analysis not detect the Beanstalk governance attack ($182M)?", "options": ["Slither was not configured correctly", "The bug is not a pattern — it's a property of the protocol's economic design (governance proposals can execute arbitrary code, and a flash-loan-funded proposal drained the protocol). Static analysis checks patterns, not the relationship between code and intended invariants", "The contract was not open source", "The attack used a zero-day opcode"], "answer_index": 1, "rationale": "Business-logic bugs have no syntactic signature. The code is valid and passes all static checks. The bug is in the relationship between code and intent — governance proposals executing arbitrary code is a design property, not a pattern static analysis can flag." },
    { "id": "Q15", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why does the harness shift the LLM's job from 'find the bug from raw source' to 'is this candidate real and exploitable in context'?", "options": ["To reduce token cost", "Finding bugs from raw source is hard and hallucination-prone (the three LLM failure modes). Judging a structured candidate with location, data flow, and severity hints is tractable and verifiable. The shift leverages each component's strength: static tools find, LLM judges", "To make the pipeline faster", "Because raw source is copyrighted"], "answer_index": 1, "rationale": "The architecture pairs static analysis (good at taint/pattern detection) with LLM reasoning (good at intent/economic judgment). Starting the LLM from structured candidates rather than raw source avoids the three failure modes and makes the LLM's judgment verifiable." },
    { "id": "Q16", "bloom": "analysis", "type": "multiple_choice", "prompt": "Compare a finding that says 'reentrancy in Vault.withdraw()' to one with a transaction hash showing 10K ETH moved. What is the operational difference?", "options": ["No difference — both indicate the same vulnerability", "The first is a hypothesis/claim; the second is evidence. A client can argue the first is theoretical. The second (forked-mainnet PoC with tx trace) is undeniable — the client can run forge test and see the vault drain. Evidence makes the finding actionable", "The first is more reliable because it's simpler", "The second is less reliable because forked mainnet can be wrong"], "answer_index": 1, "rationale": "Detection without exploitation is a hypothesis. A PoC with structured evidence (tx hash, state delta, call trace, economic impact) converts the hypothesis into undeniable evidence. EVMbench's Exploit mode exists for this reason." },
    { "id": "Q17", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why is the exploit re-run gate (gate 3) necessary but insufficient for patch verification?", "options": ["It's not necessary — gates 1 and 2 are enough", "Necessary: it verifies the specific attack path is closed. Insufficient: a patch can fix the specific PoC (e.g., reject 10 ETH deposits) while leaving the underlying vulnerability (50 ETH still exploits). Gates 1 (static) and 4 (formal) catch this — they check the pattern/property, not just the PoC", "It's both necessary and sufficient", "It's insufficient because it's too slow"], "answer_index": 1, "rationale": "The patch-that-fixes-the-PoC trap. Gate 3 verifies the specific exploit fails, but a patch can address only the PoC's parameters. Static re-check (gate 1) still flags the pattern; formal verification (gate 4) shows the invariant is still violable. The cascade of four gates is required." },
    { "id": "Q18", "bloom": "analysis", "type": "multiple_choice", "prompt": "Why is the human approval gate non-negotiable even after all four automated gates pass?", "options": ["The automated gates are unreliable", "Smart contracts are immutable once deployed (upgradeable proxies aside). A buggy patch on mainnet causes irreversible losses. The four gates check what is automatable; some properties (does this patch change economic behavior the team intends?) require human judgment. The harness generates; a human approves and deploys", "It's a legal requirement in all jurisdictions", "To reduce automation cost"], "answer_index": 1, "rationale": "The automated gates check automatable properties. Economic-behavior changes (does the patch alter the protocol's intended economics?) require human judgment. Immutability means a wrong patch cannot be reversed. The harness does NOT deploy — a human approves." },
    { "id": "Q19", "bloom": "analysis", "type": "multiple_choice", "prompt": "How does Heimdallr's function-level reorganization extend to cross-contract logic (flash loan paths)?", "options": ["It doesn't — each contract is analyzed independently", "The harness extends the context window along the call graph. If Vault.withdraw calls Oracle.getPrice which reads a spot DEX pool, all three are included in the LLM's context with data-flow edges. The model reasons over the full attack path, not one contract in isolation", "It compiles all contracts into one file", "It uses a separate LLM call per contract and merges results"], "answer_index": 1, "rationale": "For cross-contract logic, the harness includes all functions along the call graph with their data-flow edges. The LLM can then reason: withdrawal amount comes from a spot price, which is flash-loan manipulable, therefore the withdrawal can be inflated. This is identified by call-graph reasoning, not pattern matching." },
    { "id": "Q20", "bloom": "analysis", "type": "multiple_choice", "prompt": "A harness scores 90% on EVMbench Detect mode but only 30% on Exploit mode. What does this tell you about the harness?", "options": ["It's a good harness — detection is what matters", "It can find vulnerabilities but cannot construct working PoCs. Detection is easier than exploitation. The harness needs work on the Exploit pipeline (forked mainnet PoC construction, Foundry integration). EVMbench scores all three modes independently for this reason", "It's broken — Detect and Exploit should match", "The benchmark is flawed"], "answer_index": 1, "rationale": "EVMbench scores Detect, Patch, and Exploit independently because they require different capabilities. A harness can find bugs (Detect) without being able to exploit them (Exploit). The gap indicates the Exploit pipeline — forked-mainnet PoC construction, Foundry/Hardhat integration — needs improvement." }
  ]
}