{
  "schema_version": 1,
  "scope": "Current public collection files; snapshot identity is not a review or publication timestamp.",
  "snapshot_sha256": "f1a478d9982c0c31956443b714ce7ad3a0c0c32bebf32f05d1388ab4b80dc93f",
  "components": {
    "discoveries": {
      "path": "data/discoveries.json",
      "label": "Scientific discoveries",
      "sha256": "84ecc25621da14bdb62ef7034eb202347cb5bade05c7104f89c01acf76cf7132",
      "bytes": 2955947,
      "count": 1364,
      "unit": "active scientific catalog entries",
      "dates": {
        "publication_dates": {
          "first": "2020-01-01",
          "last": "2026-11-19"
        },
        "addition_dates": {
          "first": "2026-11-11",
          "last": "2026-11-20"
        }
      },
      "source_urls": [
        "https://aisciencetracker.org/methodology.html"
      ],
      "license": "CC BY 4.0",
      "coverage": "Manual and incomplete; entries can overlap or bundle findings.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": "https://doi.org/10.5281/zenodo.24217686",
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license."
    },
    "archived-discoveries": {
      "path": "data/archived-discoveries.json",
      "label": "Archived discoveries",
      "sha256": "948fb2739c23a8e682cac19d488e777b045f709a0a1786ba8fa04661bce09ff2",
      "bytes": 35223,
      "count": 20,
      "unit": "historical catalog entries",
      "dates": {
        "publication_dates": {
          "first": "2022-11-07",
          "last": "2026-11-09"
        },
        "addition_dates": {
          "first": "2026-11-11",
          "last": "2026-11-18"
        }
      },
      "source_urls": [
        "https://aisciencetracker.org/methodology.html"
      ],
      "license": "CC BY 4.0",
      "coverage": "Retained historical records; excluded from active science totals.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": "https://doi.org/10.5281/zenodo.24217686",
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license."
    },
    "vulnerabilities": {
      "path": "data/vulnerabilities.json",
      "label": "AI-found vulnerabilities",
      "sha256": "8cccffad5d18d0b3ee5fca147e1d47484dca47ebd82bd5df90018f3873958eb3",
      "bytes": 4303057,
      "count": 1658,
      "unit": "source-attributed cases",
      "dates": {
        "disclosure_date": {
          "first": "2024-10-16",
          "last": "2026-11-19"
        },
        "ai_attribution_date": {
          "first": "2024-11-01",
          "last": "2026-11-19"
        },
        "checked_date": {
          "first": "2026-11-13",
          "last": "2026-11-20"
        }
      },
      "source_urls": [
        "https://aisciencetracker.org/security.html#t-context"
      ],
      "license": "CC BY 4.0",
      "coverage": "Manual and incomplete; cases and distinct CVEs are different units.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": "https://doi.org/10.5281/zenodo.24217686",
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license.",
      "distinct_cves": 1547
    },
    "models": {
      "path": "data/models.json",
      "label": "Notable model releases",
      "sha256": "37a0905d92b7ee11e8129a0ed4b6de22e512275eff7925895de6f82bd6a6a5f9",
      "bytes": 181795,
      "count": 440,
      "unit": "selected Epoch model releases",
      "dates": {
        "generated_on": "2026-11-20",
        "source_downloaded_on": null,
        "release_dates": {
          "first": "2019-08-22",
          "last": "2026-11-10"
        }
      },
      "source_urls": [
        "https://epoch.ai/data/ai-models"
      ],
      "license": "CC BY 4.0; attribute Epoch AI",
      "coverage": "Selected notable releases; overlaps model-progress inventory.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": null,
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license."
    },
    "model-progress": {
      "path": "data/model-progress.json",
      "label": "Model progress",
      "sha256": "a6aabd4f2c3dabd3f3c9b45b821947464a85797beb2b747e1ab869effbe69a25",
      "bytes": 541178,
      "count": 573,
      "unit": "Epoch source entries",
      "dates": {
        "snapshot_retrieved_on": "2026-11-20",
        "recorded_dates": {
          "first": "2019-10-23",
          "last": "2026-11-18"
        }
      },
      "source_urls": [
        "https://epoch.ai/data/benchmark_data.zip"
      ],
      "license": "CC BY 4.0; attribute Epoch AI",
      "coverage": "Selected source entries; missing ECI and benchmark values are not zero.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": null,
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license.",
      "coverage_counts": {
        "eci": 209,
        "any_displayed_benchmark": 99,
        "total": 573
      }
    },
    "swe-bench": {
      "path": "data/swe-bench.json",
      "label": "SWE-bench Verified",
      "sha256": "9511d100759c9d62ddb574631ae119e33a7cc8c8fdf1469b2b3909f128404aad",
      "bytes": 29091,
      "count": 47,
      "unit": "mini-SWE-agent evaluation runs",
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "evaluation_dates": {
          "first": "2025-07-20",
          "last": "2026-02-26"
        }
      },
      "source_urls": [
        "https://github.com/SWE-bench/swe-bench.github.io/blob/master/data/leaderboards.json"
      ],
      "license": "CC BY-NC 4.0",
      "coverage": "Separate configurations, not unique models; excluded from current Hugging Face catalog export. Noncommercial reuse only.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": null,
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license.",
      "license_url": "https://creativecommons.org/licenses/by-nc/4.0/",
      "license_notice": "docs/swe-bench-license.txt",
      "attribution": "SWE-bench Team, SWE-bench website leaderboard",
      "changes": "Selected Verified mini-SWE-agent runs; omitted warning-marked and unscored runs; renamed fields and sorted records. No scores recalculated.",
      "hugging_face_export": false
    },
    "citations": {
      "path": "data/citations.json",
      "label": "Citation metadata",
      "sha256": "58c0f5bbbf19a365ac7cedd0b93b1e8fc151d22cc1f8d3c0da9f6a5db0c70f68",
      "bytes": 104238,
      "count": 504,
      "unit": "OpenAlex records matched to catalog IDs",
      "dates": {
        "metadata_checked_on": "2026-11-15"
      },
      "source_urls": [
        "https://openalex.org"
      ],
      "license": "CC0; OpenAlex metadata",
      "coverage": "Selective matching; citation counts do not measure scientific evidence strength.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": null,
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license."
    },
    "entity-aliases": {
      "path": "data/entity-aliases.json",
      "label": "Reviewed entity aliases",
      "sha256": "5af444822a8c4502f3c2cc6eaddd920afdbf286f180bed645198be172893d497",
      "bytes": 3294,
      "count": 3,
      "unit": "reviewed entity groups",
      "dates": {
        "alias_review_dates": {
          "first": "2026-11-18",
          "last": "2026-11-18"
        }
      },
      "source_urls": [
        "https://aisle.com/cve-discoveries/cve-2026-125201",
        "https://www.mit.edu/about/"
      ],
      "license": "CC BY 4.0",
      "coverage": "Exact aliases only; original labels and model-version uncertainty remain in records.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": null,
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license."
    },
    "discoveries-csv": {
      "path": "data/discoveries.csv",
      "label": "Scientific discoveries",
      "sha256": "429ca542fa463d1c2f3125e3e624a4558663fb5b56eade22c6d1eb7a5d547283",
      "bytes": 2418599,
      "count": 1364,
      "unit": "active scientific catalog entries",
      "dates": {
        "publication_dates": {
          "first": "2020-01-01",
          "last": "2026-11-19"
        },
        "addition_dates": {
          "first": "2026-11-11",
          "last": "2026-11-20"
        }
      },
      "source_urls": [
        "https://aisciencetracker.org/methodology.html"
      ],
      "license": "CC BY 4.0",
      "coverage": "Manual and incomplete; entries can overlap or bundle findings.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": "https://doi.org/10.5281/zenodo.24217686",
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license."
    },
    "vulnerabilities-csv": {
      "path": "data/vulnerabilities.csv",
      "label": "AI-found vulnerabilities",
      "sha256": "dcbd8beaded37d7779be58345d1bf2ed98a71c03e0273be2f1a3b639feb1694a",
      "bytes": 3397188,
      "count": 1658,
      "unit": "source-attributed cases",
      "dates": {
        "disclosure_date": {
          "first": "2024-10-16",
          "last": "2026-11-19"
        },
        "ai_attribution_date": {
          "first": "2024-11-01",
          "last": "2026-11-19"
        },
        "checked_date": {
          "first": "2026-11-13",
          "last": "2026-11-20"
        }
      },
      "source_urls": [
        "https://aisciencetracker.org/security.html#t-context"
      ],
      "license": "CC BY 4.0",
      "coverage": "Manual and incomplete; cases and distinct CVEs are different units.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases",
      "catalog_doi": "https://doi.org/10.5281/zenodo.24217686",
      "doi_note": "Catalog archive DOI; upstream sources retain their own identity and license.",
      "distinct_cves": 1547
    },
    "catalog-summary": {
      "path": "data/catalog-summary.csv",
      "label": "Catalog coverage summary",
      "sha256": "86fbbff27fd4a558b69d75015b10d8a86716cc56daf45d37d1436f636e83c883",
      "bytes": 524,
      "count": 18,
      "unit": "overlapping year/field aggregate rows",
      "dates": {
        "publication_dates": {
          "first": "2020-01-01",
          "last": "2026-11-19"
        },
        "addition_dates": {
          "first": "2026-11-11",
          "last": "2026-11-20"
        }
      },
      "source_urls": [
        "https://aisciencetracker.org/profiles/catalog-evidence.html"
      ],
      "license": "CC BY 4.0",
      "coverage": "Each grouping separately sums to active catalog entries; never add the year and field groupings together. Counts describe collection coverage, not independent findings.",
      "catalog_doi": "https://doi.org/10.5281/zenodo.24217686",
      "doi_note": "Catalog archive DOI; choose a matching release for stable citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-registry": {
      "path": "data/benchmarks/index.json",
      "label": "Independent benchmark directory",
      "sha256": "b90125c0d9bde94b04a91a38d47a4835177a06e457fc8a85ab30cbb018790991",
      "bytes": 16084,
      "count": 11,
      "unit": "separately licensed source components",
      "dates": {
        "directory_retrieved_on": "2026-11-20"
      },
      "source_urls": [
        "https://aisciencetracker.org/docs/public-benchmarks.md"
      ],
      "license": "Per-component terms; no blanket catalog license",
      "coverage": "Separate numerical source snapshots. No composite ranking or inferred model joins.",
      "group": "public-benchmarks",
      "catalog_doi": null,
      "doi_note": "Source components retain their own identity and license.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-gpqa": {
      "path": "data/benchmarks/gpqa.json",
      "label": "GPQA Diamond · Epoch own runs",
      "sha256": "9d97f3e1db9ea870351278cac554b33f955317bc5bc05c58cd1e0ffb1dff4722",
      "bytes": 354378,
      "count": 342,
      "unit": "source numerical observations",
      "record_units": [
        "fraction"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-20",
        "source_observation_dates": {
          "first": "2025-01-27",
          "last": "2026-11-19"
        }
      },
      "source_date_basis": [
        "Evaluation start"
      ],
      "source_urls": [
        "https://epoch.ai/benchmarks/use-this-data"
      ],
      "license": "CC BY 4.0",
      "license_url": "https://creativecommons.org/licenses/by/4.0/",
      "license_notice": "docs/benchmark-licenses/cc-by-4.0.txt",
      "attribution": "Epoch AI, Capabilities & benchmarking",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Graduate-level biology, chemistry and physics. Mean accuracy is a fraction, not ECI. Inference settings and budgets differ; stderr is source-reported, not a confidence interval.",
      "selection": "All scored own-run gpqa_diamond.csv rows; no external imported scores.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-arena": {
      "path": "data/benchmarks/arena.json",
      "label": "Arena · text preferences",
      "sha256": "4ecfbcf313a277ccd09b57ec3a1eb38ba2ea48c13ac029217e2b384c11d9a341",
      "bytes": 362708,
      "count": 426,
      "unit": "source numerical observations",
      "record_units": [
        "rating points"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-20",
        "source_observation_dates": {
          "first": "2026-11-19",
          "last": "2026-11-19"
        }
      },
      "source_date_basis": [
        "Leaderboard publication"
      ],
      "source_urls": [
        "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/d3517dc2bbc13f82d925d8524f9c5cb5e4563d36/README.md"
      ],
      "license": "CC BY 4.0",
      "license_url": "https://creativecommons.org/licenses/by/4.0/",
      "license_notice": "docs/benchmark-licenses/cc-by-4.0.txt",
      "attribution": "Arena / LMArena, leaderboard-dataset",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Pairwise human preferences, not factual accuracy. Rating points are not percentages or directly comparable across categories or dates. Confidence bounds and vote counts retained. The source model license field describes the model, not this numerical dataset.",
      "selection": "Complete frozen text/latest Parquet, overall category only; category scores are not mixed.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-tau": {
      "path": "data/benchmarks/tau.json",
      "label": "τ²-bench · text agent tasks",
      "sha256": "203f217be4db00e8f3ca4ce46de406bae7af3434cc34b26f1c7d16b5cbbd9d0c",
      "bytes": 111976,
      "count": 54,
      "unit": "source numerical observations",
      "record_units": [
        "%"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": {
          "first": "2026-02-24",
          "last": "2026-11-03"
        }
      },
      "source_date_basis": [
        "Evaluation date, or submission date if absent"
      ],
      "source_urls": [
        "https://raw.githubusercontent.com/sierra-research/tau2-bench/c06b07dd7c4d4ec61ba6ff9bcdf0b15dd7ef36d4/web/leaderboard/public/submissions/manifest.json"
      ],
      "license": "MIT",
      "license_url": "https://github.com/sierra-research/tau2-bench/blob/c06b07dd7c4d4ec61ba6ff9bcdf0b15dd7ef36d4/LICENSE",
      "license_notice": "docs/benchmark-licenses/tau2-bench-license.txt",
      "attribution": "Sierra Research and τ-bench result submitters",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Agent task success. Pass^1 is shown; other pass^k values and methodology remain in details. Versions, simulators, domains, retrieval, reasoning and agent setups can differ. Source-reported results, not independent reproductions.",
      "selection": "All active text submission.json records in the pinned manifest; voice, legacy and S3 trajectories excluded.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-chembench": {
      "path": "data/benchmarks/chembench.json",
      "label": "ChemBench · legacy reports",
      "sha256": "0247b5561d6cc4d25329b6f657e7329a29a7955aea7aca83c25fb4c1a8008bb4",
      "bytes": 3117,
      "count": 2,
      "unit": "source numerical observations",
      "record_units": [
        "fraction"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": null
      },
      "source_date_basis": [
        "Not reported"
      ],
      "source_urls": [
        "https://raw.githubusercontent.com/lamalab-org/chembench/45f8bad062fe552810c52be3a328d5da8597ed30/reports/gpt-4o/gpt-4o.json"
      ],
      "license": "MIT",
      "license_url": "https://github.com/lamalab-org/chembench/blob/45f8bad062fe552810c52be3a328d5da8597ed30/LICENSE",
      "license_notice": "docs/benchmark-licenses/chembench-license.txt",
      "attribution": "LAMA Lab / ChemBench authors",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Chemical question performance in two historical repository reports. Source folder names are retained; exact model release and evaluation dates are not stated. This is not the current ChemBench-Results leaderboard.",
      "selection": "Only gpt-4o and claude3.5 aggregate fraction_correct; question-level outputs and tasks excluded.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-bixbench": {
      "path": "data/benchmarks/bixbench.json",
      "label": "BixBench v1.5 · zero-shot baselines",
      "sha256": "4f5cac1da0ec773cdf4dd1e9d0513b412e2156a0776e4bcdb3745da3740129a1",
      "bytes": 6091,
      "count": 6,
      "unit": "source numerical observations",
      "record_units": [
        "fraction"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": null
      },
      "source_date_basis": [
        "Not reported"
      ],
      "source_urls": [
        "https://raw.githubusercontent.com/Future-House/BixBench/49311180bdacb324c596f2e07596c126f2004008/bixbench-v1.5_results/zero_shot_baselines.json"
      ],
      "license": "Apache-2.0",
      "license_url": "https://github.com/Future-House/BixBench/blob/49311180bdacb324c596f2e07596c126f2004008/LICENSE",
      "license_notice": "docs/benchmark-licenses/BixBench-license.txt",
      "attribution": "FutureHouse / BixBench authors",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Bioinformatics zero-shot baseline accuracy on 205 questions. MCQ, refusal-enabled and open-ended settings differ; these are not the paper agent runs. Precision and coverage retained.",
      "selection": "All six configurations in zero_shot_baselines.json.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-mmlu-pro": {
      "path": "data/benchmarks/mmlu-pro.json",
      "label": "MMLU-Pro · historical README results",
      "sha256": "33908e39453141b5d163199869446fb6b844542117d6e9b23a5a583098ff81ce",
      "bytes": 11484,
      "count": 17,
      "unit": "source numerical observations",
      "record_units": [
        "%"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": null
      },
      "source_date_basis": [
        "Not reported"
      ],
      "source_urls": [
        "https://raw.githubusercontent.com/TIGER-AI-Lab/MMLU-Pro/f418b116db00b065c2aea046518d8fcf74d39872/README.md"
      ],
      "license": "Apache-2.0",
      "license_url": "https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/f418b116db00b065c2aea046518d8fcf74d39872/LICENSE",
      "license_notice": "docs/benchmark-licenses/MMLU-Pro-license.txt",
      "attribution": "TIGER-AI-Lab; Yubo Wang and MMLU-Pro coauthors",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Multitask knowledge/reasoning accuracy. Historical mini-leaderboard; dates and full inference configurations are not reported in the selected table. It does not represent the current external leaderboard.",
      "selection": "All numerical rows in the licensed README mini-leaderboard.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-charxiv": {
      "path": "data/benchmarks/charxiv.json",
      "label": "CharXiv · chart understanding",
      "sha256": "2b2abecbecdcf6d2a86dd73da96177df358387428bab27b0a92e487d03e6d2fe",
      "bytes": 152675,
      "count": 78,
      "unit": "source numerical observations",
      "record_units": [
        "%"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": null
      },
      "source_date_basis": [
        "Not reported"
      ],
      "source_urls": [
        "https://huggingface.co/datasets/princeton-nlp/CharXiv/resolve/f441eb632fc62f6f777830a0f47619e6e86459b0/README.md"
      ],
      "license": "CC BY-SA 4.0",
      "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
      "license_notice": "docs/benchmark-licenses/charxiv-data-license.txt",
      "attribution": "Princeton NLP / CharXiv authors",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Scientific chart question performance. Descriptive and reasoning validation scores are separate; human/random references are labeled as in source. Released evaluations are historical (2024); current website scores are not imported. No paper chart images or question text included.",
      "selection": "All released stats-*.json aggregate evaluations; source numbers unchanged. This adapted numerical component is shared under CC BY-SA 4.0. Attribution, license and change notices must travel with adaptations.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-ruler": {
      "path": "data/benchmarks/ruler.json",
      "label": "RULER · long-context accuracy",
      "sha256": "a763ff8c8cc44b9489ece703d252728f5e2c552d07369c35b47291b25bbbe93b",
      "bytes": 193722,
      "count": 264,
      "unit": "source numerical observations",
      "record_units": [
        "%"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": null
      },
      "source_date_basis": [
        "Not reported"
      ],
      "source_urls": [
        "https://raw.githubusercontent.com/NVIDIA/RULER/db43f92ff634bbce1c33d1fb15858aec4bba509a/README.md"
      ],
      "license": "Apache-2.0",
      "license_url": "https://github.com/NVIDIA/RULER/blob/db43f92ff634bbce1c33d1fb15858aec4bba509a/LICENSE",
      "license_notice": "docs/benchmark-licenses/RULER-license.txt",
      "attribution": "NVIDIA / RULER authors; contributing model authors identified in the source notes",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Source README accuracy at six measured context lengths. Claimed and effective length differ. Includes contributing-author reports; see original notes for FILM, Qwen, Jamba and EXAONE protocol exceptions. Missing lengths are absent, never zero. No third-party task corpora redistributed.",
      "selection": "All available numeric 4K–128K context columns for model rows; aggregate weighted rankings not imported.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-agentdojo": {
      "path": "data/benchmarks/agentdojo.json",
      "label": "AgentDojo · prompt injection robustness",
      "sha256": "89ddc08c6479b65d367dce348e48be33df4d27aa7f6f3af06180b03d5b2de110",
      "bytes": 55601,
      "count": 84,
      "unit": "source numerical observations",
      "record_units": [
        "%"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": {
          "first": "2021-09-05",
          "last": "2025-02-24"
        }
      },
      "source_date_basis": [
        "Source table date"
      ],
      "source_urls": [
        "https://raw.githubusercontent.com/ethz-spylab/agentdojo/089ed468cf3ed0322acc66b0211f26d9d90dbf60/docs/results-table.html"
      ],
      "license": "MIT",
      "license_url": "https://github.com/ethz-spylab/agentdojo/blob/089ed468cf3ed0322acc66b0211f26d9d90dbf60/LICENSE",
      "license_notice": "docs/benchmark-licenses/agentdojo-license.txt",
      "attribution": "ETH Zurich SPY Lab / AgentDojo authors",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Agent utility and targeted attack success are separate outcomes. Lower attack success is better. Defenses and attacks differ; dates are source table dates. No attack content, traces or conversations redistributed.",
      "selection": "All numeric utility, utility-under-attack and targeted-ASR cells in the repository results table.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-swe-bench-pro": {
      "path": "data/benchmarks/swe-bench-pro.json",
      "label": "SWE-bench Pro · initial repository runs",
      "sha256": "439d6467f11f4532168e95010fc69acf3d72dc279ef912913db37595aaca4db9",
      "bytes": 6788,
      "count": 7,
      "unit": "source numerical observations",
      "record_units": [
        "%"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": null
      },
      "source_date_basis": [
        "Not reported"
      ],
      "source_urls": [
        "https://raw.githubusercontent.com/scaleapi/SWE-bench_Pro-os/66f92766bba642462d4bbe5479e83f91f9211862/index.html"
      ],
      "license": "MIT",
      "license_url": "https://github.com/scaleapi/SWE-bench_Pro-os/blob/66f92766bba642462d4bbe5479e83f91f9211862/LICENSE",
      "license_notice": "docs/benchmark-licenses/SWE-bench_Pro-os-license.txt",
      "attribution": "Scale AI, SWE-bench Pro authors",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Initial SWE-Agent runs on 730 problems with uncapped cost and a 250-turn limit. Source explicitly says results are preliminary and may change. Distinct from SWE-bench Verified and from separate Scale commercial/private leaderboard data.",
      "selection": "Only seven source-committed HTML numerical scores; no client-computed confidence intervals, tasks or private results.",
      "maintenance": "Checked daily for updates within this exact selection. Source licenses and supported formats are checked before acceptance; blocked sources retain the last accepted snapshot. Historical and preliminary scope still applies.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    },
    "benchmark-metr": {
      "path": "data/benchmarks/metr.json",
      "label": "METR · task-completion time horizons",
      "sha256": "9b929e4234b1ef2fed08a0109c215500001e77038aa302efc4c64123d1fa395e",
      "bytes": 142027,
      "count": 112,
      "unit": "source numerical observations",
      "record_units": [
        "human-baseline minutes"
      ],
      "dates": {
        "snapshot_retrieved_on": "2026-11-18",
        "source_observation_dates": {
          "first": "2019-02-14",
          "last": "2026-04-07"
        }
      },
      "source_date_basis": [
        "Source model release date; evaluation date not reported"
      ],
      "source_urls": [
        "https://metr.org/time-horizons/"
      ],
      "license": "CC BY 4.0",
      "license_url": "https://creativecommons.org/licenses/by/4.0/",
      "license_notice": "docs/benchmark-licenses/cc-by-4.0.txt",
      "attribution": "METR, Measuring AI Ability to Complete Long Tasks (2025), arXiv:2503.14499",
      "changes": "Selected numerical results; fields renamed and evaluation slices labeled. Original numerical scales retained. Questions, paper images, transcripts, contacts and trajectories excluded.",
      "coverage": "Human-expert task duration at 50% or 80% predicted agent success, measured in minutes. This is task difficulty, not AI runtime, uninterrupted autonomy or job-automation coverage. Versions 1.1 and 1.0 use different suites; scaffolds differ. Estimates above 16 hours are unreliable with this task suite. Source-reported 95% bootstrapped intervals are retained.",
      "selection": "All scored 50% and 80% horizons in the official public METR-Horizon-v1.1 and v1.0 exports. Version 1.1 is shown first; versions and success thresholds remain separate. Exact source aliases, scaffolds, dataset versions and model-release dates retained. No inferred Epoch joins, trend fits or private task data. Three identical v1.0 baselines stitched into the v1.1 export are retained once under v1.0, not counted as v1.1 reruns.",
      "maintenance": "Checked daily for changes to these public exports; blocked sources retain the last accepted snapshot. METR states the source page is no longer actively updated.",
      "group": "public-benchmarks",
      "hugging_face_export": false,
      "catalog_doi": null,
      "doi_note": "No component DOI assigned here; the catalog archive does not replace source citation.",
      "release_url": "https://github.com/marcello-pichini/ai-science-tracker/releases"
    }
  }
}
