{
  "schema_version": 1,
  "retrieved_on": "2026-11-18",
  "scope": "Separate numerical source snapshots. No composite ranking or inferred model joins.",
  "reuse": "Each component retains its own license; no blanket catalog license applies.",
  "sources": [
    {
      "id": "gpqa",
      "name": "GPQA Diamond · Epoch own runs",
      "attribution": "Epoch AI, Capabilities & benchmarking",
      "license": "CC BY 4.0",
      "license_url": "https://creativecommons.org/licenses/by/4.0/",
      "license_notice": "docs/benchmark-licenses/cc-by-4.0.txt",
      "scope": "Graduate-level biology, chemistry and physics. Mean accuracy is a fraction, not ECI. Inference settings and budgets differ; stderr is source-reported, not a confidence interval.",
      "selection": "All scored own-run gpqa_diamond.csv rows; no external imported scores.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "Export SHA-256 retained in input hashes",
      "source_url": "https://epoch.ai/benchmarks/use-this-data",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/gpqa.json",
      "record_count": 340,
      "sha256": "82cef472d9f314a2583f47449035301741e13ce239ce6850cc4c84096fe8f8fa"
    },
    {
      "id": "arena",
      "name": "Arena · text preferences",
      "attribution": "Arena / LMArena, leaderboard-dataset",
      "license": "CC BY 4.0",
      "license_url": "https://creativecommons.org/licenses/by/4.0/",
      "license_notice": "docs/benchmark-licenses/cc-by-4.0.txt",
      "scope": "Pairwise human preferences, not factual accuracy. Rating points are not percentages or directly comparable across categories or dates. Confidence bounds and vote counts retained. The source model license field describes the model, not this numerical dataset.",
      "selection": "Complete frozen text/latest Parquet, overall category only; category scores are not mixed.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "b17ca88fd34809e9161df1ef89fd444acaf598e4",
      "source_url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/b17ca88fd34809e9161df1ef89fd444acaf598e4/text/latest-00000-of-00001.parquet",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/arena.json",
      "record_count": 421,
      "sha256": "f9fa231fcf4f60dfbae28cd281caa7c137374488760498de18152a08715e869e"
    },
    {
      "id": "tau",
      "name": "τ²-bench · text agent tasks",
      "attribution": "Sierra Research and τ-bench result submitters",
      "license": "MIT",
      "license_url": "https://github.com/sierra-research/tau2-bench/blob/c06b07dd7c4d4ec61ba6ff9bcdf0b15dd7ef36d4/LICENSE",
      "license_notice": "docs/benchmark-licenses/tau2-bench-license.txt",
      "scope": "Agent task success. Pass^1 is shown; other pass^k values and methodology remain in details. Versions, simulators, domains, retrieval, reasoning and agent setups can differ. Source-reported results, not independent reproductions.",
      "selection": "All active text submission.json records in the pinned manifest; voice, legacy and S3 trajectories excluded.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "c06b07dd7c4d4ec61ba6ff9bcdf0b15dd7ef36d4",
      "source_url": "https://raw.githubusercontent.com/sierra-research/tau2-bench/c06b07dd7c4d4ec61ba6ff9bcdf0b15dd7ef36d4/web/leaderboard/public/submissions/manifest.json",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/tau.json",
      "record_count": 54,
      "sha256": "203f217be4db00e8f3ca4ce46de406bae7af3434cc34b26f1c7d16b5cbbd9d0c"
    },
    {
      "id": "chembench",
      "name": "ChemBench · legacy reports",
      "attribution": "LAMA Lab / ChemBench authors",
      "license": "MIT",
      "license_url": "https://github.com/lamalab-org/chembench/blob/45f8bad062fe552810c52be3a328d5da8597ed30/LICENSE",
      "license_notice": "docs/benchmark-licenses/chembench-license.txt",
      "scope": "Chemical question performance in two historical repository reports. Source folder names are retained; exact model release and evaluation dates are not stated. This is not the current ChemBench-Results leaderboard.",
      "selection": "Only gpt-4o and claude3.5 aggregate fraction_correct; question-level outputs and tasks excluded.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "45f8bad062fe552810c52be3a328d5da8597ed30",
      "source_url": "https://raw.githubusercontent.com/lamalab-org/chembench/45f8bad062fe552810c52be3a328d5da8597ed30/reports/gpt-4o/gpt-4o.json",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/chembench.json",
      "record_count": 2,
      "sha256": "0247b5561d6cc4d25329b6f657e7329a29a7955aea7aca83c25fb4c1a8008bb4"
    },
    {
      "id": "bixbench",
      "name": "BixBench v1.5 · zero-shot baselines",
      "attribution": "FutureHouse / BixBench authors",
      "license": "Apache-2.0",
      "license_url": "https://github.com/Future-House/BixBench/blob/49311180bdacb324c596f2e07596c126f2004008/LICENSE",
      "license_notice": "docs/benchmark-licenses/BixBench-license.txt",
      "scope": "Bioinformatics zero-shot baseline accuracy on 205 questions. MCQ, refusal-enabled and open-ended settings differ; these are not the paper agent runs. Precision and coverage retained.",
      "selection": "All six configurations in zero_shot_baselines.json.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "49311180bdacb324c596f2e07596c126f2004008",
      "source_url": "https://raw.githubusercontent.com/Future-House/BixBench/49311180bdacb324c596f2e07596c126f2004008/bixbench-v1.5_results/zero_shot_baselines.json",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/bixbench.json",
      "record_count": 6,
      "sha256": "4f5cac1da0ec773cdf4dd1e9d0513b412e2156a0776e4bcdb3745da3740129a1"
    },
    {
      "id": "mmlu-pro",
      "name": "MMLU-Pro · historical README results",
      "attribution": "TIGER-AI-Lab; Yubo Wang and MMLU-Pro coauthors",
      "license": "Apache-2.0",
      "license_url": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/LICENSE",
      "license_notice": "docs/benchmark-licenses/MMLU-Pro-license.txt",
      "scope": "Multitask knowledge/reasoning accuracy. Historical mini-leaderboard; dates and full inference configurations are not reported in the selected table. It does not represent the current external leaderboard.",
      "selection": "All numerical rows in the licensed README mini-leaderboard.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "f418b116db00b065c2aea046518d8fcf74d39872",
      "source_url": "https://raw.githubusercontent.com/TIGER-AI-Lab/MMLU-Pro/f418b116db00b065c2aea046518d8fcf74d39872/README.md",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/mmlu-pro.json",
      "record_count": 17,
      "sha256": "33908e39453141b5d163199869446fb6b844542117d6e9b23a5a583098ff81ce"
    },
    {
      "id": "charxiv",
      "name": "CharXiv · chart understanding",
      "attribution": "Princeton NLP / CharXiv authors",
      "license": "CC BY-SA 4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "license_notice": "docs/benchmark-licenses/charxiv-data-license.txt",
      "scope": "Scientific chart question performance. Descriptive and reasoning validation scores are separate; human/random references are labeled as in source. Released evaluations are historical (2024); current website scores are not imported. No paper chart images or question text included.",
      "selection": "All released stats-*.json aggregate evaluations; source numbers unchanged. This adapted numerical component is shared under CC BY-SA 4.0. Attribution, license and change notices must travel with adaptations.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "f441eb632fc62f6f777830a0f47619e6e86459b0",
      "source_url": "https://huggingface.co/datasets/princeton-nlp/CharXiv/resolve/f441eb632fc62f6f777830a0f47619e6e86459b0/README.md",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/charxiv.json",
      "record_count": 78,
      "sha256": "2b2abecbecdcf6d2a86dd73da96177df358387428bab27b0a92e487d03e6d2fe"
    },
    {
      "id": "ruler",
      "name": "RULER · long-context accuracy",
      "attribution": "NVIDIA / RULER authors; contributing model authors identified in the source notes",
      "license": "Apache-2.0",
      "license_url": "https://github.com/NVIDIA/RULER/blob/db43f92ff634bbce1c33d1fb15858aec4bba509a/LICENSE",
      "license_notice": "docs/benchmark-licenses/RULER-license.txt",
      "scope": "Source README accuracy at six measured context lengths. Claimed and effective length differ. Includes contributing-author reports; see original notes for FILM, Qwen, Jamba and EXAONE protocol exceptions. Missing lengths are absent, never zero. No third-party task corpora redistributed.",
      "selection": "All available numeric 4K–128K context columns for model rows; aggregate weighted rankings not imported.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "db43f92ff634bbce1c33d1fb15858aec4bba509a",
      "source_url": "https://raw.githubusercontent.com/NVIDIA/RULER/db43f92ff634bbce1c33d1fb15858aec4bba509a/README.md",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/ruler.json",
      "record_count": 264,
      "sha256": "a763ff8c8cc44b9489ece703d252728f5e2c552d07369c35b47291b25bbbe93b"
    },
    {
      "id": "agentdojo",
      "name": "AgentDojo · prompt injection robustness",
      "attribution": "ETH Zurich SPY Lab / AgentDojo authors",
      "license": "MIT",
      "license_url": "https://github.com/ethz-spylab/agentdojo/blob/089ed468cf3ed0322acc66b0211f26d9d90dbf60/LICENSE",
      "license_notice": "docs/benchmark-licenses/agentdojo-license.txt",
      "scope": "Agent utility and targeted attack success are separate outcomes. Lower attack success is better. Defenses and attacks differ; dates are source table dates. No attack content, traces or conversations redistributed.",
      "selection": "All numeric utility, utility-under-attack and targeted-ASR cells in the repository results table.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "089ed468cf3ed0322acc66b0211f26d9d90dbf60",
      "source_url": "https://raw.githubusercontent.com/ethz-spylab/agentdojo/089ed468cf3ed0322acc66b0211f26d9d90dbf60/docs/results-table.html",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/agentdojo.json",
      "record_count": 84,
      "sha256": "89ddc08c6479b65d367dce348e48be33df4d27aa7f6f3af06180b03d5b2de110"
    },
    {
      "id": "swe-bench-pro",
      "name": "SWE-bench Pro · initial repository runs",
      "attribution": "Scale AI, SWE-bench Pro authors",
      "license": "MIT",
      "license_url": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/66f92766bba642462d4bbe5479e83f91f9211862/LICENSE",
      "license_notice": "docs/benchmark-licenses/SWE-bench_Pro-os-license.txt",
      "scope": "Initial SWE-Agent runs on 730 problems with uncapped cost and a 250-turn limit. Source explicitly says results are preliminary and may change. Distinct from SWE-bench Verified and from separate Scale commercial/private leaderboard data.",
      "selection": "Only seven source-committed HTML numerical scores; no client-computed confidence intervals, tasks or private results.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "upstream_revision": "66f92766bba642462d4bbe5479e83f91f9211862",
      "source_url": "https://raw.githubusercontent.com/scaleapi/SWE-bench_Pro-os/66f92766bba642462d4bbe5479e83f91f9211862/index.html",
      "retrieved_on": "2026-11-18",
      "path": "data/benchmarks/swe-bench-pro.json",
      "record_count": 7,
      "sha256": "439d6467f11f4532168e95010fc69acf3d72dc279ef912913db37595aaca4db9"
    }
  ]
}
