{
 "title": "Claude Code security review skills on RealVuln",
 "recorded": "2026-10-01",
 "claude_code": "2.1.280",
 "model": "claude-sonnet-5-5",
 "runs_per_arm_and_repository": 1,
 "skills": {
  "ecc": {
   "source": "https://github.com/affaan-m/ECC",
   "path": "skills/security-review",
   "commit": "c70874f"
  },
  "sentry": {
   "source": "https://github.com/getsentry/skills",
   "path": "skills/security-review",
   "commit": "d18b7aa"
  },
  "owasp": {
   "source": "https://github.com/sergiodxa/agent-skills",
   "path": "skills/owasp-security-check",
   "commit": "40e21b4"
  },
  "supercov": {
   "source": "https://github.com/supercorp-ai/supercov",
   "path": "plugins/supercov",
   "commit": "4908a6c",
   "cli": "supercov 2.0.1 via npx",
   "jev_usd_per_repo": 0.0126,
   "jev_seconds_median": 4.05
  },
  "builtin": {
   "source": "built into Claude Code 2.1.280 (/security-review)"
  }
 },
 "conversion_prompt": "List every vulnerability you reported in your review as JSON in exactly the format below: one entry per reported vulnerability, with its file path relative to the repository root, its line, and its most specific CWE. Include every vulnerability you reported and nothing you did not report. Output only the JSON, no markdown fences.\n\n{\n  \"version\": \"1.0.0\",\n  \"results\": [\n    {\n      \"check_id\": \"python.security.injection.sql-injection\",\n      \"path\": \"app/views.py\",\n      \"start\": {\n        \"line\": 42,\n        \"col\": 5,\n        \"offset\": 1200\n      },\n      \"end\": {\n        \"line\": 42,\n        \"col\": 60,\n        \"offset\": 1255\n      },\n      \"extra\": {\n        \"message\": \"User input from request.GET['query'] is concatenated directly into SQL string passed to cursor.execute(). This allows SQL injection.\",\n        \"severity\": \"ERROR\",\n        \"metadata\": {\n          \"cwe\": [\n            \"CWE-89: SQL Injection\"\n          ],\n          \"confidence\": \"HIGH\",\n          \"category\": \"security\"\n        }\n      }\n    }\n  ]\n}",
 "setup": "One fresh copy per run. Every file is committed on a branch over an empty origin/main, so /security-review sees the whole repository as the branch diff. Claude Code ran headless with --setting-sources project and --strict-mcp-config, file edits disallowed; each skill or plugin was loaded for its session only with --plugin-dir and invoked by its slash name. /security-review and /supercov:security run bare, because their arguments are not a request. The skills-disabled arm adds --disable-slash-commands. A second turn in the same session listed the reported vulnerabilities as JSON.",
 "benchmark": "https://github.com/kolega-ai/Real-Vuln-Benchmark",
 "rule": "same file; the reported CWE in the primary-or-acceptable set, or in the same RealVuln CWE family (config/cwe-families.json) as one of them; within 10 lines; findings deduplicated. A finding without a line (listed at line 1) hits one not-yet-hit labelled vulnerability of an accepted CWE in its file, as RealVuln's own matcher allows.",
 "repositories": [
  {
   "name": "realvuln-dsvw",
   "language": "python",
   "labelled": 27
  },
  {
   "name": "realvuln-nodejs-goof",
   "language": "javascript",
   "labelled": 17
  },
  {
   "name": "realvuln-vulnerable-flask-app",
   "language": "python",
   "labelled": 21
  },
  {
   "name": "realvuln-dvna",
   "language": "javascript",
   "labelled": 17
  },
  {
   "name": "vc-codex-seeded-v2-fintech-lending-fastapi",
   "language": "python",
   "labelled": 37
  },
  {
   "name": "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue",
   "language": "typescript",
   "labelled": 28
  }
 ],
 "arms": [
  {
   "arm": "none",
   "label": "Claude Code, skills disabled",
   "prompt": "Do a security review of this entire repository and report the security vulnerabilities you find in its code. Do not modify any files.",
   "reported": 245,
   "true_positives": 107,
   "labelled": 147,
   "found": 111,
   "precision": 0.4367,
   "recall": 0.7551,
   "f1": 0.5534,
   "review_seconds_median": 56.6,
   "review_list_price_usd_mean": 0.5762,
   "findings_at_line_1": 4,
   "f1_line_findings_only": 0.5534,
   "f1_strict_cwe": 0.5497,
   "f1_with_supercov_classes": 0.6022,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 30,
     "true_positives": 22,
     "labelled": 27,
     "found": 23,
     "precision": 0.7333,
     "recall": 0.8519,
     "f1": 0.7882
    },
    "realvuln-nodejs-goof": {
     "reported": 35,
     "true_positives": 11,
     "labelled": 17,
     "found": 11,
     "precision": 0.3143,
     "recall": 0.6471,
     "f1": 0.4231
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 48,
     "true_positives": 16,
     "labelled": 21,
     "found": 18,
     "precision": 0.3333,
     "recall": 0.8571,
     "f1": 0.48
    },
    "realvuln-dvna": {
     "reported": 31,
     "true_positives": 12,
     "labelled": 17,
     "found": 12,
     "precision": 0.3871,
     "recall": 0.7059,
     "f1": 0.5
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 40,
     "true_positives": 28,
     "labelled": 37,
     "found": 28,
     "precision": 0.7,
     "recall": 0.7568,
     "f1": 0.7273
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 61,
     "true_positives": 18,
     "labelled": 28,
     "found": 19,
     "precision": 0.2951,
     "recall": 0.6786,
     "f1": 0.4113
    }
   },
   "found_by_class": {
    "destination_from_input": {
     "found": 7,
     "labelled": 7
    },
    "injection_sink": {
     "found": 11,
     "labelled": 11
    },
    "insecure_configuration": {
     "found": 11,
     "labelled": 14
    },
    "missing_authorization": {
     "found": 15,
     "labelled": 20
    },
    "path_from_input": {
     "found": 7,
     "labelled": 9
    },
    "secret_in_source": {
     "found": 12,
     "labelled": 15
    },
    "sensitive_data_exposure": {
     "found": 9,
     "labelled": 14
    },
    "unchecked_mass_assignment": {
     "found": 7,
     "labelled": 7
    },
    "unescaped_output": {
     "found": 10,
     "labelled": 10
    },
    "unmapped": {
     "found": 9,
     "labelled": 16
    },
    "unsafe_code_execution": {
     "found": 5,
     "labelled": 6
    },
    "weak_authentication": {
     "found": 6,
     "labelled": 13
    },
    "weak_cryptography": {
     "found": 2,
     "labelled": 5
    }
   }
  },
  {
   "arm": "plain",
   "label": "Claude Code as installed",
   "prompt": "Do a security review of this entire repository and report the security vulnerabilities you find in its code. Do not modify any files.",
   "reported": 182,
   "true_positives": 90,
   "labelled": 147,
   "found": 92,
   "precision": 0.4945,
   "recall": 0.6259,
   "f1": 0.5525,
   "review_seconds_median": 54.3,
   "review_list_price_usd_mean": 0.9163,
   "findings_at_line_1": 3,
   "f1_line_findings_only": 0.5525,
   "f1_strict_cwe": 0.5498,
   "f1_with_supercov_classes": 0.587,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 25,
     "true_positives": 21,
     "labelled": 27,
     "found": 22,
     "precision": 0.84,
     "recall": 0.8148,
     "f1": 0.8272
    },
    "realvuln-nodejs-goof": {
     "reported": 6,
     "true_positives": 5,
     "labelled": 17,
     "found": 5,
     "precision": 0.8333,
     "recall": 0.2941,
     "f1": 0.4348
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 32,
     "true_positives": 14,
     "labelled": 21,
     "found": 16,
     "precision": 0.4375,
     "recall": 0.7619,
     "f1": 0.5558
    },
    "realvuln-dvna": {
     "reported": 36,
     "true_positives": 13,
     "labelled": 17,
     "found": 13,
     "precision": 0.3611,
     "recall": 0.7647,
     "f1": 0.4906
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 71,
     "true_positives": 32,
     "labelled": 37,
     "found": 31,
     "precision": 0.4507,
     "recall": 0.8378,
     "f1": 0.5861
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 12,
     "true_positives": 5,
     "labelled": 28,
     "found": 5,
     "precision": 0.4167,
     "recall": 0.1786,
     "f1": 0.25
    }
   },
   "found_by_class": {
    "destination_from_input": {
     "found": 6,
     "labelled": 7
    },
    "injection_sink": {
     "found": 10,
     "labelled": 11
    },
    "insecure_configuration": {
     "found": 8,
     "labelled": 14
    },
    "missing_authorization": {
     "found": 15,
     "labelled": 20
    },
    "path_from_input": {
     "found": 6,
     "labelled": 9
    },
    "secret_in_source": {
     "found": 7,
     "labelled": 15
    },
    "sensitive_data_exposure": {
     "found": 6,
     "labelled": 14
    },
    "unchecked_mass_assignment": {
     "found": 2,
     "labelled": 7
    },
    "unescaped_output": {
     "found": 10,
     "labelled": 10
    },
    "unmapped": {
     "found": 9,
     "labelled": 16
    },
    "unsafe_code_execution": {
     "found": 6,
     "labelled": 6
    },
    "weak_authentication": {
     "found": 6,
     "labelled": 13
    },
    "weak_cryptography": {
     "found": 1,
     "labelled": 5
    }
   }
  },
  {
   "arm": "builtin",
   "label": "Claude Code /security-review (built in)",
   "prompt": "/security-review",
   "reported": 47,
   "true_positives": 33,
   "labelled": 147,
   "found": 35,
   "precision": 0.7021,
   "recall": 0.2381,
   "f1": 0.3556,
   "review_seconds_median": 298.2,
   "review_list_price_usd_mean": 2.8106,
   "findings_at_line_1": 0,
   "f1_line_findings_only": 0.3556,
   "f1_strict_cwe": 0.3556,
   "f1_with_supercov_classes": 0.383,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 14,
     "true_positives": 14,
     "labelled": 27,
     "found": 14,
     "precision": 1.0,
     "recall": 0.5185,
     "f1": 0.6829
    },
    "realvuln-nodejs-goof": {
     "reported": 6,
     "true_positives": 4,
     "labelled": 17,
     "found": 4,
     "precision": 0.6667,
     "recall": 0.2353,
     "f1": 0.3478
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 7,
     "true_positives": 5,
     "labelled": 21,
     "found": 7,
     "precision": 0.7143,
     "recall": 0.3333,
     "f1": 0.4545
    },
    "realvuln-dvna": {
     "reported": 11,
     "true_positives": 7,
     "labelled": 17,
     "found": 7,
     "precision": 0.6364,
     "recall": 0.4118,
     "f1": 0.5
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 0,
     "true_positives": 0,
     "labelled": 37,
     "found": 0,
     "precision": 0,
     "recall": 0.0,
     "f1": 0
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 9,
     "true_positives": 3,
     "labelled": 28,
     "found": 3,
     "precision": 0.3333,
     "recall": 0.1071,
     "f1": 0.1622
    }
   },
   "found_by_class": {
    "destination_from_input": {
     "found": 0,
     "labelled": 7
    },
    "injection_sink": {
     "found": 9,
     "labelled": 11
    },
    "insecure_configuration": {
     "found": 2,
     "labelled": 14
    },
    "missing_authorization": {
     "found": 3,
     "labelled": 20
    },
    "path_from_input": {
     "found": 2,
     "labelled": 9
    },
    "secret_in_source": {
     "found": 4,
     "labelled": 15
    },
    "sensitive_data_exposure": {
     "found": 0,
     "labelled": 14
    },
    "unchecked_mass_assignment": {
     "found": 2,
     "labelled": 7
    },
    "unescaped_output": {
     "found": 8,
     "labelled": 10
    },
    "unmapped": {
     "found": 2,
     "labelled": 16
    },
    "unsafe_code_execution": {
     "found": 3,
     "labelled": 6
    },
    "weak_authentication": {
     "found": 0,
     "labelled": 13
    },
    "weak_cryptography": {
     "found": 0,
     "labelled": 5
    }
   }
  },
  {
   "arm": "ecc",
   "label": "everything-claude-code security-review skill",
   "prompt": "/ecc:security-review Do a security review of this entire repository and report the security vulnerabilities you find in its code. Do not modify any files.",
   "reported": 270,
   "true_positives": 104,
   "labelled": 147,
   "found": 108,
   "precision": 0.3852,
   "recall": 0.7347,
   "f1": 0.5054,
   "review_seconds_median": 55.0,
   "review_list_price_usd_mean": 0.5481,
   "findings_at_line_1": 5,
   "f1_line_findings_only": 0.5054,
   "f1_strict_cwe": 0.5005,
   "f1_with_supercov_classes": 0.5486,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 28,
     "true_positives": 22,
     "labelled": 27,
     "found": 23,
     "precision": 0.7857,
     "recall": 0.8519,
     "f1": 0.8174
    },
    "realvuln-nodejs-goof": {
     "reported": 27,
     "true_positives": 6,
     "labelled": 17,
     "found": 6,
     "precision": 0.2222,
     "recall": 0.3529,
     "f1": 0.2727
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 40,
     "true_positives": 14,
     "labelled": 21,
     "found": 16,
     "precision": 0.35,
     "recall": 0.7619,
     "f1": 0.4797
    },
    "realvuln-dvna": {
     "reported": 29,
     "true_positives": 14,
     "labelled": 17,
     "found": 14,
     "precision": 0.4828,
     "recall": 0.8235,
     "f1": 0.6087
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 60,
     "true_positives": 29,
     "labelled": 37,
     "found": 28,
     "precision": 0.4833,
     "recall": 0.7568,
     "f1": 0.5899
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 86,
     "true_positives": 19,
     "labelled": 28,
     "found": 21,
     "precision": 0.2209,
     "recall": 0.75,
     "f1": 0.3413
    }
   },
   "found_by_class": {
    "destination_from_input": {
     "found": 7,
     "labelled": 7
    },
    "injection_sink": {
     "found": 11,
     "labelled": 11
    },
    "insecure_configuration": {
     "found": 10,
     "labelled": 14
    },
    "missing_authorization": {
     "found": 15,
     "labelled": 20
    },
    "path_from_input": {
     "found": 8,
     "labelled": 9
    },
    "secret_in_source": {
     "found": 7,
     "labelled": 15
    },
    "sensitive_data_exposure": {
     "found": 7,
     "labelled": 14
    },
    "unchecked_mass_assignment": {
     "found": 7,
     "labelled": 7
    },
    "unescaped_output": {
     "found": 10,
     "labelled": 10
    },
    "unmapped": {
     "found": 10,
     "labelled": 16
    },
    "unsafe_code_execution": {
     "found": 5,
     "labelled": 6
    },
    "weak_authentication": {
     "found": 9,
     "labelled": 13
    },
    "weak_cryptography": {
     "found": 2,
     "labelled": 5
    }
   }
  },
  {
   "arm": "sentry",
   "label": "Sentry security-review skill",
   "prompt": "/sentry:security-review Do a security review of this entire repository and report the security vulnerabilities you find in its code. Do not modify any files.",
   "reported": 90,
   "true_positives": 67,
   "labelled": 147,
   "found": 70,
   "precision": 0.7444,
   "recall": 0.4762,
   "f1": 0.5808,
   "review_seconds_median": 80.5,
   "review_list_price_usd_mean": 0.6501,
   "findings_at_line_1": 1,
   "f1_line_findings_only": 0.5808,
   "f1_strict_cwe": 0.5758,
   "f1_with_supercov_classes": 0.6329,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 19,
     "true_positives": 19,
     "labelled": 27,
     "found": 20,
     "precision": 1.0,
     "recall": 0.7407,
     "f1": 0.8511
    },
    "realvuln-nodejs-goof": {
     "reported": 12,
     "true_positives": 6,
     "labelled": 17,
     "found": 6,
     "precision": 0.5,
     "recall": 0.3529,
     "f1": 0.4138
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 12,
     "true_positives": 9,
     "labelled": 21,
     "found": 11,
     "precision": 0.75,
     "recall": 0.5238,
     "f1": 0.6168
    },
    "realvuln-dvna": {
     "reported": 14,
     "true_positives": 9,
     "labelled": 17,
     "found": 9,
     "precision": 0.6429,
     "recall": 0.5294,
     "f1": 0.5806
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 17,
     "true_positives": 16,
     "labelled": 37,
     "found": 16,
     "precision": 0.9412,
     "recall": 0.4324,
     "f1": 0.5926
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 16,
     "true_positives": 8,
     "labelled": 28,
     "found": 8,
     "precision": 0.5,
     "recall": 0.2857,
     "f1": 0.3636
    }
   },
   "found_by_class": {
    "destination_from_input": {
     "found": 7,
     "labelled": 7
    },
    "injection_sink": {
     "found": 11,
     "labelled": 11
    },
    "insecure_configuration": {
     "found": 4,
     "labelled": 14
    },
    "missing_authorization": {
     "found": 8,
     "labelled": 20
    },
    "path_from_input": {
     "found": 4,
     "labelled": 9
    },
    "secret_in_source": {
     "found": 7,
     "labelled": 15
    },
    "sensitive_data_exposure": {
     "found": 4,
     "labelled": 14
    },
    "unchecked_mass_assignment": {
     "found": 2,
     "labelled": 7
    },
    "unescaped_output": {
     "found": 9,
     "labelled": 10
    },
    "unmapped": {
     "found": 6,
     "labelled": 16
    },
    "unsafe_code_execution": {
     "found": 5,
     "labelled": 6
    },
    "weak_authentication": {
     "found": 1,
     "labelled": 13
    },
    "weak_cryptography": {
     "found": 2,
     "labelled": 5
    }
   }
  },
  {
   "arm": "owasp",
   "label": "sergiodxa owasp-security-check skill",
   "prompt": "/owasp:owasp-security-check Do a security review of this entire repository and report the security vulnerabilities you find in its code. Do not modify any files.",
   "reported": 243,
   "true_positives": 105,
   "labelled": 147,
   "found": 109,
   "precision": 0.4321,
   "recall": 0.7415,
   "f1": 0.546,
   "review_seconds_median": 61.6,
   "review_list_price_usd_mean": 0.5779,
   "findings_at_line_1": 5,
   "f1_line_findings_only": 0.546,
   "f1_strict_cwe": 0.5372,
   "f1_with_supercov_classes": 0.5893,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 26,
     "true_positives": 23,
     "labelled": 27,
     "found": 24,
     "precision": 0.8846,
     "recall": 0.8889,
     "f1": 0.8867
    },
    "realvuln-nodejs-goof": {
     "reported": 31,
     "true_positives": 10,
     "labelled": 17,
     "found": 9,
     "precision": 0.3226,
     "recall": 0.5294,
     "f1": 0.4009
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 28,
     "true_positives": 12,
     "labelled": 21,
     "found": 14,
     "precision": 0.4286,
     "recall": 0.6667,
     "f1": 0.5217
    },
    "realvuln-dvna": {
     "reported": 34,
     "true_positives": 14,
     "labelled": 17,
     "found": 14,
     "precision": 0.4118,
     "recall": 0.8235,
     "f1": 0.549
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 49,
     "true_positives": 28,
     "labelled": 37,
     "found": 28,
     "precision": 0.5714,
     "recall": 0.7568,
     "f1": 0.6512
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 75,
     "true_positives": 18,
     "labelled": 28,
     "found": 20,
     "precision": 0.24,
     "recall": 0.7143,
     "f1": 0.3593
    }
   },
   "found_by_class": {
    "destination_from_input": {
     "found": 7,
     "labelled": 7
    },
    "injection_sink": {
     "found": 11,
     "labelled": 11
    },
    "insecure_configuration": {
     "found": 11,
     "labelled": 14
    },
    "missing_authorization": {
     "found": 14,
     "labelled": 20
    },
    "path_from_input": {
     "found": 8,
     "labelled": 9
    },
    "secret_in_source": {
     "found": 10,
     "labelled": 15
    },
    "sensitive_data_exposure": {
     "found": 6,
     "labelled": 14
    },
    "unchecked_mass_assignment": {
     "found": 6,
     "labelled": 7
    },
    "unescaped_output": {
     "found": 10,
     "labelled": 10
    },
    "unmapped": {
     "found": 9,
     "labelled": 16
    },
    "unsafe_code_execution": {
     "found": 5,
     "labelled": 6
    },
    "weak_authentication": {
     "found": 10,
     "labelled": 13
    },
    "weak_cryptography": {
     "found": 2,
     "labelled": 5
    }
   }
  },
  {
   "arm": "supercov",
   "label": "Supercov plugin /supercov:security",
   "prompt": "/supercov:security",
   "reported": 129,
   "true_positives": 76,
   "labelled": 147,
   "found": 77,
   "precision": 0.5891,
   "recall": 0.5238,
   "f1": 0.5546,
   "review_seconds_median": 25.4,
   "review_list_price_usd_mean": 0.1528,
   "findings_at_line_1": 8,
   "f1_line_findings_only": 0.5473,
   "f1_strict_cwe": 0.5328,
   "f1_with_supercov_classes": 0.6059,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 25,
     "true_positives": 22,
     "labelled": 27,
     "found": 23,
     "precision": 0.88,
     "recall": 0.8519,
     "f1": 0.8657
    },
    "realvuln-nodejs-goof": {
     "reported": 28,
     "true_positives": 10,
     "labelled": 17,
     "found": 10,
     "precision": 0.3571,
     "recall": 0.5882,
     "f1": 0.4444
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 22,
     "true_positives": 12,
     "labelled": 21,
     "found": 13,
     "precision": 0.5455,
     "recall": 0.619,
     "f1": 0.5799
    },
    "realvuln-dvna": {
     "reported": 14,
     "true_positives": 6,
     "labelled": 17,
     "found": 6,
     "precision": 0.4286,
     "recall": 0.3529,
     "f1": 0.3871
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 29,
     "true_positives": 20,
     "labelled": 37,
     "found": 19,
     "precision": 0.6897,
     "recall": 0.5135,
     "f1": 0.5887
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 11,
     "true_positives": 6,
     "labelled": 28,
     "found": 6,
     "precision": 0.5455,
     "recall": 0.2143,
     "f1": 0.3077
    }
   },
   "found_by_class": {
    "destination_from_input": {
     "found": 7,
     "labelled": 7
    },
    "injection_sink": {
     "found": 11,
     "labelled": 11
    },
    "insecure_configuration": {
     "found": 9,
     "labelled": 14
    },
    "missing_authorization": {
     "found": 6,
     "labelled": 20
    },
    "path_from_input": {
     "found": 6,
     "labelled": 9
    },
    "secret_in_source": {
     "found": 9,
     "labelled": 15
    },
    "sensitive_data_exposure": {
     "found": 7,
     "labelled": 14
    },
    "unchecked_mass_assignment": {
     "found": 1,
     "labelled": 7
    },
    "unescaped_output": {
     "found": 8,
     "labelled": 10
    },
    "unmapped": {
     "found": 4,
     "labelled": 16
    },
    "unsafe_code_execution": {
     "found": 5,
     "labelled": 6
    },
    "weak_authentication": {
     "found": 3,
     "labelled": 13
    },
    "weak_cryptography": {
     "found": 1,
     "labelled": 5
    }
   }
  }
 ],
 "references": [
  {
   "arm": "claude-sonnet-5-cc-agentic-v1",
   "label": "RealVuln's run: Claude Code, Sonnet 5, RealVuln's agentic prompt",
   "reported": 107,
   "true_positives": 77,
   "labelled": 147,
   "found": 70,
   "precision": 0.7196,
   "recall": 0.4762,
   "f1": 0.5731,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 26,
     "true_positives": 25,
     "labelled": 27,
     "found": 21,
     "precision": 0.9615,
     "recall": 0.7778,
     "f1": 0.86
    },
    "realvuln-nodejs-goof": {
     "reported": 15,
     "true_positives": 8,
     "labelled": 17,
     "found": 8,
     "precision": 0.5333,
     "recall": 0.4706,
     "f1": 0.5
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 20,
     "true_positives": 12,
     "labelled": 21,
     "found": 11,
     "precision": 0.6,
     "recall": 0.5238,
     "f1": 0.5593
    },
    "realvuln-dvna": {
     "reported": 13,
     "true_positives": 9,
     "labelled": 17,
     "found": 9,
     "precision": 0.6923,
     "recall": 0.5294,
     "f1": 0.6
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 15,
     "true_positives": 12,
     "labelled": 37,
     "found": 11,
     "precision": 0.8,
     "recall": 0.2973,
     "f1": 0.4335
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 18,
     "true_positives": 11,
     "labelled": 28,
     "found": 10,
     "precision": 0.6111,
     "recall": 0.3571,
     "f1": 0.4508
    }
   },
   "scoring": "this study's rule"
  },
  {
   "arm": "semgrep",
   "label": "Semgrep (RealVuln run)",
   "reported": 67,
   "true_positives": 14,
   "labelled": 147,
   "found": 14,
   "precision": 0.209,
   "recall": 0.0952,
   "f1": 0.1308,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 0,
     "true_positives": 0,
     "labelled": 27,
     "found": 0,
     "precision": 0,
     "recall": 0.0,
     "f1": 0
    },
    "realvuln-nodejs-goof": {
     "reported": 26,
     "true_positives": 3,
     "labelled": 17,
     "found": 3,
     "precision": 0.1154,
     "recall": 0.1765,
     "f1": 0.1395
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 11,
     "true_positives": 3,
     "labelled": 21,
     "found": 3,
     "precision": 0.2727,
     "recall": 0.1429,
     "f1": 0.1875
    },
    "realvuln-dvna": {
     "reported": 14,
     "true_positives": 7,
     "labelled": 17,
     "found": 7,
     "precision": 0.5,
     "recall": 0.4118,
     "f1": 0.4516
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 7,
     "true_positives": 0,
     "labelled": 37,
     "found": 0,
     "precision": 0.0,
     "recall": 0.0,
     "f1": 0
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 9,
     "true_positives": 1,
     "labelled": 28,
     "found": 1,
     "precision": 0.1111,
     "recall": 0.0357,
     "f1": 0.0541
    }
   },
   "scoring": "this study's rule"
  },
  {
   "arm": "supercov-september",
   "label": "supercov security 2.0.1, the September 25 benchmark run",
   "reported": 133,
   "true_positives": 76,
   "labelled": 147,
   "found": 87,
   "precision": 0.5714,
   "recall": 0.5918,
   "f1": 0.5815,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 16,
     "true_positives": 15,
     "labelled": 27,
     "found": 18,
     "precision": 0.9375,
     "recall": 0.6667,
     "f1": 0.7792
    },
    "realvuln-nodejs-goof": {
     "reported": 30,
     "true_positives": 12,
     "labelled": 17,
     "found": 12,
     "precision": 0.4,
     "recall": 0.7059,
     "f1": 0.5106
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 20,
     "true_positives": 12,
     "labelled": 21,
     "found": 14,
     "precision": 0.6,
     "recall": 0.6667,
     "f1": 0.6316
    },
    "realvuln-dvna": {
     "reported": 30,
     "true_positives": 13,
     "labelled": 17,
     "found": 13,
     "precision": 0.4333,
     "recall": 0.7647,
     "f1": 0.5532
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 24,
     "true_positives": 17,
     "labelled": 37,
     "found": 21,
     "precision": 0.7083,
     "recall": 0.5676,
     "f1": 0.6302
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 13,
     "true_positives": 7,
     "labelled": 28,
     "found": 9,
     "precision": 0.5385,
     "recall": 0.3214,
     "f1": 0.4026
    }
   },
   "scoring": "the 72-repository benchmark's rule: each check's whole CWE family; a file-level finding counts once in its file"
  },
  {
   "arm": "supercov-this-run",
   "label": "supercov security 2.0.1, this study's run, before Claude read it",
   "reported": 136,
   "true_positives": 76,
   "labelled": 147,
   "found": 86,
   "precision": 0.5588,
   "recall": 0.585,
   "f1": 0.5716,
   "per_repo": {
    "realvuln-dsvw": {
     "reported": 16,
     "true_positives": 15,
     "labelled": 27,
     "found": 18,
     "precision": 0.9375,
     "recall": 0.6667,
     "f1": 0.7792
    },
    "realvuln-nodejs-goof": {
     "reported": 30,
     "true_positives": 12,
     "labelled": 17,
     "found": 12,
     "precision": 0.4,
     "recall": 0.7059,
     "f1": 0.5106
    },
    "realvuln-vulnerable-flask-app": {
     "reported": 20,
     "true_positives": 12,
     "labelled": 21,
     "found": 14,
     "precision": 0.6,
     "recall": 0.6667,
     "f1": 0.6316
    },
    "realvuln-dvna": {
     "reported": 32,
     "true_positives": 13,
     "labelled": 17,
     "found": 13,
     "precision": 0.4062,
     "recall": 0.7647,
     "f1": 0.5306
    },
    "vc-codex-seeded-v2-fintech-lending-fastapi": {
     "reported": 24,
     "true_positives": 17,
     "labelled": 37,
     "found": 20,
     "precision": 0.7083,
     "recall": 0.5405,
     "f1": 0.6132
    },
    "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue": {
     "reported": 14,
     "true_positives": 7,
     "labelled": 28,
     "found": 9,
     "precision": 0.5,
     "recall": 0.3214,
     "f1": 0.3913
    }
   },
   "scoring": "the 72-repository benchmark's rule: each check's whole CWE family; a file-level finding counts once in its file"
  }
 ],
 "rerun_pairs": {
  "note": "The as-installed and skills-disabled arms used no skill on these four repositories, so each pair is the same setup run twice.",
  "repositories": {
   "realvuln-dsvw": {
    "first": {
     "reported": 25,
     "true_positives": 21,
     "labelled": 27,
     "found": 22,
     "precision": 0.84,
     "recall": 0.8148,
     "f1": 0.8272
    },
    "second": {
     "reported": 30,
     "true_positives": 22,
     "labelled": 27,
     "found": 23,
     "precision": 0.7333,
     "recall": 0.8519,
     "f1": 0.7882
    }
   },
   "realvuln-vulnerable-flask-app": {
    "first": {
     "reported": 32,
     "true_positives": 14,
     "labelled": 21,
     "found": 16,
     "precision": 0.4375,
     "recall": 0.7619,
     "f1": 0.5558
    },
    "second": {
     "reported": 48,
     "true_positives": 16,
     "labelled": 21,
     "found": 18,
     "precision": 0.3333,
     "recall": 0.8571,
     "f1": 0.48
    }
   },
   "realvuln-dvna": {
    "first": {
     "reported": 36,
     "true_positives": 13,
     "labelled": 17,
     "found": 13,
     "precision": 0.3611,
     "recall": 0.7647,
     "f1": 0.4906
    },
    "second": {
     "reported": 31,
     "true_positives": 12,
     "labelled": 17,
     "found": 12,
     "precision": 0.3871,
     "recall": 0.7059,
     "f1": 0.5
    }
   },
   "vc-codex-seeded-v2-fintech-lending-fastapi": {
    "first": {
     "reported": 71,
     "true_positives": 32,
     "labelled": 37,
     "found": 31,
     "precision": 0.4507,
     "recall": 0.8378,
     "f1": 0.5861
    },
    "second": {
     "reported": 40,
     "true_positives": 28,
     "labelled": 37,
     "found": 28,
     "precision": 0.7,
     "recall": 0.7568,
     "f1": 0.7273
    }
   }
  }
 },
 "note": "The first built-in and Supercov runs reused a Supercov assessment saved in the benchmark checkouts. Both arms were rerun from clean copies; only the clean runs are published.",
 "as_installed_used_security_review_on": [
  "realvuln-nodejs-goof",
  "vc-deepseek-v4-flash-seeded-v3-logistics-dispatch-fastify-vue"
 ]
}