{
  "fetched_on": "2026-11-18",
  "source_url": "https://epoch.ai/data/benchmark_data.zip",
  "zip_sha256": "c7769ce95e1739442a5f3757cb8c52c6328d54e52e17848c163616f2aab17289",
  "catalog_snapshot": "0c411dbaafd96eb1166545cfe13f760131e4860f18a1d41d222ba77697341107",
  "license": "CC BY 4.0; attribute Epoch AI",
  "labs": [
    "OpenAI",
    "Google DeepMind",
    "Anthropic",
    "Meta",
    "Alibaba",
    "DeepSeek",
    "xAI",
    "Mistral"
  ],
  "benchmarks": [
    {
      "id": "frontiermath_tier_4_v2.csv",
      "name": "FrontierMath Tier 4 (v2)",
      "domain": "Mathematics"
    },
    {
      "id": "mirrorcode.csv",
      "name": "MirrorCode",
      "domain": "Software engineering"
    },
    {
      "id": "simpleqa_verified.csv",
      "name": "SimpleQA Verified",
      "domain": "World knowledge"
    },
    {
      "id": "mystery_game_puzzles.csv",
      "name": "Mystery Game Puzzles",
      "domain": "Reasoning / games"
    }
  ],
  "unmatched_benchmark_rows": {},
  "models": [
    {
      "id": "GPT-6.2 Sol",
      "name": "GPT-6.2 Sol",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-11-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-6.2-sol_high",
        "gpt-6.2-sol_low",
        "gpt-6.2-sol_max",
        "gpt-6.2-sol_medium",
        "gpt-6.2-sol_unknown",
        "gpt-6.2-sol_xhigh"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-6.2 Astra",
      "name": "GPT-6.2 Astra",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-11-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 174.17,
      "eci_low": 170.29,
      "eci_high": 178.58,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-6.2-astra_max"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 100.0,
          "variant": "gpt-6.2-astra_max",
          "configurations": [
            {
              "variant": "gpt-6.2-astra_max",
              "score": 100.0
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 67.77777777777779,
          "variant": "gpt-6.2-astra_max",
          "configurations": [
            {
              "variant": "gpt-6.2-astra_max",
              "score": 67.77777777777779
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 78.5,
          "variant": "gpt-6.2-astra_max",
          "configurations": [
            {
              "variant": "gpt-6.2-astra_max",
              "score": 78.5
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 95.0,
          "variant": "gpt-6.2-astra_max",
          "configurations": [
            {
              "variant": "gpt-6.2-astra_max",
              "score": 95.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 4 Hydrogen",
      "name": "Gemini 4 Hydrogen",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-11-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-4-hydrogen_high",
        "gemini-4-hydrogen_low",
        "gemini-4-hydrogen_medium",
        "gemini-4-hydrogen_unknown",
        "gemini-4-hydrogen_xhigh"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 4 Carbon",
      "name": "Gemini 4 Carbon",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-11-10",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-11-10",
      "catalog_releases": [
        {
          "name": "Gemini 4 Carbon",
          "date": "2026-11-10",
          "link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/"
        }
      ],
      "eci": 169.78,
      "eci_low": 166.59,
      "eci_high": 173.31,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-4-carbon_high",
        "gemini-4-carbon_low",
        "gemini-4-carbon_medium",
        "gemini-4-carbon_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 95.1219512195122,
          "variant": "gemini-4-carbon_high",
          "configurations": [
            {
              "variant": "gemini-4-carbon_high",
              "score": 95.1219512195122
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 77.7,
          "variant": "gemini-4-carbon_high",
          "configurations": [
            {
              "variant": "gemini-4-carbon_high",
              "score": 77.7
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 73.0,
          "variant": "gemini-4-carbon_high",
          "configurations": [
            {
              "variant": "gemini-4-carbon_high",
              "score": 73.0
            }
          ]
        }
      },
      "catalog_link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/introducing-gemini-4-carbon/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Ember",
      "name": "Muse Ember",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-11-09",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "muse-ember-1.0_high",
        "muse-ember-1.0_low",
        "muse-ember-1.0_max",
        "muse-ember-1.0_medium",
        "muse-ember-1.0_minimal",
        "muse-ember-1.0_unknown",
        "muse-ember-1.0_xhigh"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Mythos 5.2",
      "name": "Claude Mythos 5.2",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-11-09",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-11-09",
      "catalog_releases": [
        {
          "name": "Claude Mythos 5.2",
          "date": "2026-11-09",
          "link": "https://www.anthropic.com/claude-fable-and-mythos-5-2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Limited access",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/claude-fable-and-mythos-5-2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Fable 5.2",
      "name": "Claude Fable 5.2",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-11-09",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-11-09",
      "catalog_releases": [
        {
          "name": "Claude Fable 5.2",
          "date": "2026-11-09",
          "link": "https://www.anthropic.com/claude-fable-and-mythos-5-2"
        }
      ],
      "eci": 173.5,
      "eci_low": 170.09,
      "eci_high": 177.36,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-fable-5-2-fallback_max",
        "claude-fable-5-2_high",
        "claude-fable-5-2_low",
        "claude-fable-5-2_max",
        "claude-fable-5-2_medium",
        "claude-fable-5-2_unknown",
        "claude-fable-5-2_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 100.0,
          "variant": "claude-fable-5-2_max",
          "configurations": [
            {
              "variant": "claude-fable-5-2_max",
              "score": 100.0
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 94.0,
          "variant": "claude-fable-5-2_max",
          "configurations": [
            {
              "variant": "claude-fable-5-2_max",
              "score": 94.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 90.0,
          "variant": "claude-fable-5-2_max",
          "configurations": [
            {
              "variant": "claude-fable-5-2_max",
              "score": 90.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/claude-fable-and-mythos-5-2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Large 4",
      "name": "Mistral Large 4",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2026-10-29",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-10-06",
      "catalog_releases": [
        {
          "name": "Mistral Large 4",
          "date": "2026-10-06",
          "link": "https://mistral.ai/news/mistral-large-4/\n\nhttps://mistral.ai/news/mistral-large-4-oss/"
        }
      ],
      "eci": 154.56,
      "eci_low": 151.79,
      "eci_high": 157.42,
      "access": "Open weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": 1050000000000.0,
      "compute_flop": 1.24e+25,
      "variants": [
        "mistral-large-4-0-oss_high",
        "mistral-large-4-0-oss_medium",
        "mistral-large-4-0-oss_unknown"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 22.400000000000002,
          "variant": "mistral-large-4-0-oss_high",
          "configurations": [
            {
              "variant": "mistral-large-4-0-oss_high",
              "score": 22.400000000000002
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 20.0,
          "variant": "mistral-large-4-0-oss_high",
          "configurations": [
            {
              "variant": "mistral-large-4-0-oss_high",
              "score": 20.0
            }
          ]
        }
      },
      "catalog_link": "https://mistral.ai/news/mistral-large-4/\n\nhttps://mistral.ai/news/mistral-large-4-oss/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-6.1 Terra",
      "name": "GPT-6.1 Terra",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-10-26",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 167.72,
      "eci_low": 164.78,
      "eci_high": 170.92,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-6.1-terra_high",
        "gpt-6.1-terra_low",
        "gpt-6.1-terra_max",
        "gpt-6.1-terra_medium",
        "gpt-6.1-terra_none",
        "gpt-6.1-terra_unknown",
        "gpt-6.1-terra_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 92.6829268292683,
          "variant": "gpt-6.1-terra_max",
          "configurations": [
            {
              "variant": "gpt-6.1-terra_max",
              "score": 92.6829268292683
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 76.66666666666667,
          "variant": "gpt-6.1-terra_max",
          "configurations": [
            {
              "variant": "gpt-6.1-terra_max",
              "score": 76.66666666666667
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 57.3,
          "variant": "gpt-6.1-terra_max",
          "configurations": [
            {
              "variant": "gpt-6.1-terra_max",
              "score": 57.3
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 74.0,
          "variant": "gpt-6.1-terra_max",
          "configurations": [
            {
              "variant": "gpt-6.1-terra_max",
              "score": 74.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek V4.1 Flash 1017",
      "name": "DeepSeek V4.1 Flash 1017",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2026-10-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 159.4,
      "eci_low": 157.16,
      "eci_high": 161.93,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "deepseek-v4.1-flash-1017_high",
        "deepseek-v4.1-flash-1017_low",
        "deepseek-v4.1-flash-1017_max",
        "deepseek-v4.1-flash-1017_medium",
        "deepseek-v4.1-flash-1017_none",
        "deepseek-v4.1-flash-1017_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 46.34146341463415,
          "variant": "deepseek-v4.1-flash-1017_max",
          "configurations": [
            {
              "variant": "deepseek-v4.1-flash-1017_max",
              "score": 46.34146341463415
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 42.5,
          "variant": "deepseek-v4.1-flash-1017_max",
          "configurations": [
            {
              "variant": "deepseek-v4.1-flash-1017_max",
              "score": 42.5
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 49.0,
          "variant": "deepseek-v4.1-flash-1017_max",
          "configurations": [
            {
              "variant": "deepseek-v4.1-flash-1017_max",
              "score": 49.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Spark Open",
      "name": "Muse Spark Open",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-10-15",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 155.45,
      "eci_low": 153.37,
      "eci_high": 157.73,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Muse-Spark-Open-816B-A42B_high",
        "Muse-Spark-Open-816B-A42B_low",
        "Muse-Spark-Open-816B-A42B_medium",
        "Muse-Spark-Open-816B-A42B_unknown",
        "Muse-Spark-Open-816B-A42B_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 31.70731707317073,
          "variant": "Muse-Spark-Open-816B-A42B_xhigh",
          "configurations": [
            {
              "variant": "Muse-Spark-Open-816B-A42B_xhigh",
              "score": 31.70731707317073
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 56.599999999999994,
          "variant": "Muse-Spark-Open-816B-A42B_xhigh",
          "configurations": [
            {
              "variant": "Muse-Spark-Open-816B-A42B_xhigh",
              "score": 56.599999999999994
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok Voice 3.0",
      "name": "Grok Voice 3.0",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2026-10-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-10-14",
      "catalog_releases": [
        {
          "name": "Grok Voice 3.0",
          "date": "2026-10-14",
          "link": "https://x.ai/news/grok-voice-3"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Audio,Speech",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://x.ai/news/grok-voice-3",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 4 Argon",
      "name": "Gemini 4 Argon",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-10-13",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-10-13",
      "catalog_releases": [
        {
          "name": "Gemini 4 Argon",
          "date": "2026-10-13",
          "link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/"
        }
      ],
      "eci": 166.57,
      "eci_low": 163.94,
      "eci_high": 169.51,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-4-argon_high",
        "gemini-4-argon_low",
        "gemini-4-argon_medium",
        "gemini-4-argon_minimal",
        "gemini-4-argon_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 78.04878048780488,
          "variant": "gemini-4-argon_high",
          "configurations": [
            {
              "variant": "gemini-4-argon_high",
              "score": 78.04878048780488
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 58.88888888888889,
          "variant": "gemini-4-argon_high",
          "configurations": [
            {
              "variant": "gemini-4-argon_high",
              "score": 58.88888888888889
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 76.0,
          "variant": "gemini-4-argon_high",
          "configurations": [
            {
              "variant": "gemini-4-argon_high",
              "score": 76.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 64.0,
          "variant": "gemini-4-argon_high",
          "configurations": [
            {
              "variant": "gemini-4-argon_high",
              "score": 64.0
            }
          ]
        }
      },
      "catalog_link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Wonder Full",
      "name": "Muse Wonder Full",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-10-12",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-10-12",
      "catalog_releases": [
        {
          "name": "Muse Wonder Full",
          "date": "2026-10-12",
          "link": "https://research.meta.ai/blog/muse-wonder\n\nhttps://research.meta.ai/static/muse-wonder-model-card"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Vision,Video,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://research.meta.ai/blog/muse-wonder\n\nhttps://research.meta.ai/static/muse-wonder-model-card",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Wonder",
      "name": "Muse Wonder",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-10-12",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-10-12",
      "catalog_releases": [
        {
          "name": "Muse Wonder",
          "date": "2026-10-12",
          "link": "https://research.meta.ai/blog/muse-wonder"
        }
      ],
      "eci": 161.74,
      "eci_low": 159.36,
      "eci_high": 164.32,
      "access": "Closed weights",
      "domain": "Language,Vision,Video,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "muse-wonder-1.0_high",
        "muse-wonder-1.0_low",
        "muse-wonder-1.0_max",
        "muse-wonder-1.0_medium",
        "muse-wonder-1.0_minimal",
        "muse-wonder-1.0_unknown",
        "muse-wonder-1.0_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 70.73170731707317,
          "variant": "muse-wonder-1.0_max",
          "configurations": [
            {
              "variant": "muse-wonder-1.0_max",
              "score": 70.73170731707317
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 45.55555555555556,
          "variant": "muse-wonder-1.0_max",
          "configurations": [
            {
              "variant": "muse-wonder-1.0_max",
              "score": 45.55555555555556
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 51.0,
          "variant": "muse-wonder-1.0_max",
          "configurations": [
            {
              "variant": "muse-wonder-1.0_max",
              "score": 51.0
            }
          ]
        }
      },
      "catalog_link": "https://research.meta.ai/blog/muse-wonder",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Haiku 5.5",
      "name": "Claude Haiku 5.5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-10-07",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 160.29,
      "eci_low": 158.12,
      "eci_high": 162.69,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-haiku-5-5_high",
        "claude-haiku-5-5_low",
        "claude-haiku-5-5_max",
        "claude-haiku-5-5_medium",
        "claude-haiku-5-5_none",
        "claude-haiku-5-5_unknown",
        "claude-haiku-5-5_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 46.34146341463415,
          "variant": "claude-haiku-5-5_max",
          "configurations": [
            {
              "variant": "claude-haiku-5-5_max",
              "score": 46.34146341463415
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 78.88888888888889,
          "variant": "claude-haiku-5-5_max",
          "configurations": [
            {
              "variant": "claude-haiku-5-5_max",
              "score": 78.88888888888889
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 23.799999999999997,
          "variant": "claude-haiku-5-5_max",
          "configurations": [
            {
              "variant": "claude-haiku-5-5_max",
              "score": 23.799999999999997
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 30.0,
          "variant": "claude-haiku-5-5_max",
          "configurations": [
            {
              "variant": "claude-haiku-5-5_max",
              "score": 30.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Large 4 Preview",
      "name": "Mistral Large 4 Preview",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2026-10-06",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 152.71,
      "eci_low": 150.81,
      "eci_high": 154.6,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.24e+25,
      "variants": [
        "mistral-large-4-0_high",
        "mistral-large-4-0_none",
        "mistral-large-4-0_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 26.82926829268293,
          "variant": "mistral-large-4-0_high",
          "configurations": [
            {
              "variant": "mistral-large-4-0_high",
              "score": 26.82926829268293
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 40.0,
          "variant": "mistral-large-4-0_high",
          "configurations": [
            {
              "variant": "mistral-large-4-0_high",
              "score": 40.0
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 20.0,
          "variant": "mistral-large-4-0_high",
          "configurations": [
            {
              "variant": "mistral-large-4-0_high",
              "score": 20.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 17.0,
          "variant": "mistral-large-4-0_high",
          "configurations": [
            {
              "variant": "mistral-large-4-0_high",
              "score": 17.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-6.1 Sol",
      "name": "GPT-6.1 Sol",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-09-29",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-29",
      "catalog_releases": [
        {
          "name": "GPT-6.1 Sol",
          "date": "2026-09-29",
          "link": "https://openai.com/index/introducing-gpt-6-1-sol/"
        }
      ],
      "eci": 168.63,
      "eci_low": 165.7,
      "eci_high": 171.91,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-6.1-sol_high",
        "gpt-6.1-sol_low",
        "gpt-6.1-sol_max",
        "gpt-6.1-sol_medium",
        "gpt-6.1-sol_unknown",
        "gpt-6.1-sol_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 100.0,
          "variant": "gpt-6.1-sol_max",
          "configurations": [
            {
              "variant": "gpt-6.1-sol_max",
              "score": 100.0
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 61.111111111111114,
          "variant": "gpt-6.1-sol_max",
          "configurations": [
            {
              "variant": "gpt-6.1-sol_max",
              "score": 61.111111111111114
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 73.9,
          "variant": "gpt-6.1-sol_max",
          "configurations": [
            {
              "variant": "gpt-6.1-sol_max",
              "score": 73.9
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 80.0,
          "variant": "gpt-6.1-sol_max",
          "configurations": [
            {
              "variant": "gpt-6.1-sol_max",
              "score": 80.0
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-gpt-6-1-sol/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Sonnet 5.5",
      "name": "Claude Sonnet 5.5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-09-28",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-28",
      "catalog_releases": [
        {
          "name": "Claude Sonnet 5.5",
          "date": "2026-09-28",
          "link": "https://www.anthropic.com/claude-sonnet-5-5"
        }
      ],
      "eci": 166.59,
      "eci_low": 163.83,
      "eci_high": 169.65,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-sonnet-5-5_high",
        "claude-sonnet-5-5_low",
        "claude-sonnet-5-5_max",
        "claude-sonnet-5-5_medium",
        "claude-sonnet-5-5_unknown",
        "claude-sonnet-5-5_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 80.48780487804879,
          "variant": "claude-sonnet-5-5_max",
          "configurations": [
            {
              "variant": "claude-sonnet-5-5_max",
              "score": 80.48780487804879
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 97.77777777777777,
          "variant": "claude-sonnet-5-5_max",
          "configurations": [
            {
              "variant": "claude-sonnet-5-5_max",
              "score": 97.77777777777777
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 46.5,
          "variant": "claude-sonnet-5-5_max",
          "configurations": [
            {
              "variant": "claude-sonnet-5-5_max",
              "score": 46.5
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 65.0,
          "variant": "claude-sonnet-5-5_max",
          "configurations": [
            {
              "variant": "claude-sonnet-5-5_max",
              "score": 65.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/claude-sonnet-5-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-6 Sol",
      "name": "GPT-6 Sol",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-09-22",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-22",
      "catalog_releases": [
        {
          "name": "GPT-6 Sol",
          "date": "2026-09-22",
          "link": "https://openai.com/index/introducing-gpt-6-sol-and-luna/"
        }
      ],
      "eci": 163.82,
      "eci_low": 161.4,
      "eci_high": 166.57,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-6-sol_high",
        "gpt-6-sol_low",
        "gpt-6-sol_max",
        "gpt-6-sol_medium",
        "gpt-6-sol_none",
        "gpt-6-sol_unknown",
        "gpt-6-sol_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 90.0,
          "variant": "gpt-6-sol_max",
          "configurations": [
            {
              "variant": "gpt-6-sol_max",
              "score": 90.0
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 48.888888888888886,
          "variant": "gpt-6-sol_max",
          "configurations": [
            {
              "variant": "gpt-6-sol_max",
              "score": 48.888888888888886
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 60.699999999999996,
          "variant": "gpt-6-sol_max",
          "configurations": [
            {
              "variant": "gpt-6-sol_max",
              "score": 60.699999999999996
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 56.00000000000001,
          "variant": "gpt-6-sol_max",
          "configurations": [
            {
              "variant": "gpt-6-sol_max",
              "score": 56.00000000000001
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-gpt-6-sol-and-luna/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-6 Luna Decisions",
      "name": "GPT-6 Luna Decisions",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-09-22",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-22",
      "catalog_releases": [
        {
          "name": "GPT-6 Luna Decisions",
          "date": "2026-09-22",
          "link": "https://openai.com/index/decisions-api/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Vision,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-6-luna-decisions"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/decisions-api/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-6 Luna",
      "name": "GPT-6 Luna",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-09-22",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-22",
      "catalog_releases": [
        {
          "name": "GPT-6 Luna",
          "date": "2026-09-22",
          "link": "https://openai.com/index/introducing-gpt-6-sol-and-luna/"
        }
      ],
      "eci": 156.39,
      "eci_low": 154.5,
      "eci_high": 158.47,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-6-luna_high",
        "gpt-6-luna_low",
        "gpt-6-luna_max",
        "gpt-6-luna_medium",
        "gpt-6-luna_none",
        "gpt-6-luna_unknown",
        "gpt-6-luna_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 56.09756097560976,
          "variant": "gpt-6-luna_max",
          "configurations": [
            {
              "variant": "gpt-6-luna_max",
              "score": 56.09756097560976
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 26.666666666666668,
          "variant": "gpt-6-luna_max",
          "configurations": [
            {
              "variant": "gpt-6-luna_max",
              "score": 26.666666666666668
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 41.4,
          "variant": "gpt-6-luna_max",
          "configurations": [
            {
              "variant": "gpt-6-luna_max",
              "score": 41.4
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 7.000000000000001,
          "variant": "gpt-6-luna_max",
          "configurations": [
            {
              "variant": "gpt-6-luna_max",
              "score": 7.000000000000001
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-gpt-6-sol-and-luna/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Opus 5.5",
      "name": "Claude Opus 5.5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-09-22",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-22",
      "catalog_releases": [
        {
          "name": "Claude Opus 5.5",
          "date": "2026-09-22",
          "link": "https://www.anthropic.com/claude-opus-5-5"
        }
      ],
      "eci": 168.7,
      "eci_low": 165.87,
      "eci_high": 171.89,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-opus-5-5-fallback_max",
        "claude-opus-5-5_high",
        "claude-opus-5-5_low",
        "claude-opus-5-5_max",
        "claude-opus-5-5_medium",
        "claude-opus-5-5_unknown",
        "claude-opus-5-5_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 95.0,
          "variant": "claude-opus-5-5_max",
          "configurations": [
            {
              "variant": "claude-opus-5-5_max",
              "score": 95.0
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 95.55555555555556,
          "variant": "claude-opus-5-5_max",
          "configurations": [
            {
              "variant": "claude-opus-5-5_max",
              "score": 95.55555555555556
            },
            {
              "variant": "claude-opus-5-5_high",
              "score": 78.89
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 72.2,
          "variant": "claude-opus-5-5_max",
          "configurations": [
            {
              "variant": "claude-opus-5-5_max",
              "score": 72.2
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 71.0,
          "variant": "claude-opus-5-5_max",
          "configurations": [
            {
              "variant": "claude-opus-5-5_max",
              "score": 71.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/claude-opus-5-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4.7",
      "name": "Grok 4.7",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2026-09-21",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-21",
      "catalog_releases": [
        {
          "name": "Grok 4.7",
          "date": "2026-09-21",
          "link": "https://x.ai/news/grok-4-7"
        }
      ],
      "eci": 156.43,
      "eci_low": 154.47,
      "eci_high": 158.52,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": 2100000000000.0,
      "compute_flop": null,
      "variants": [
        "grok-4.7_high",
        "grok-4.7_low",
        "grok-4.7_medium",
        "grok-4.7_unknown",
        "grok-4.7_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 17.073170731707318,
          "variant": "grok-4.7_xhigh",
          "configurations": [
            {
              "variant": "grok-4.7_xhigh",
              "score": 17.073170731707318
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 54.44444444444444,
          "variant": "grok-4.7_xhigh",
          "configurations": [
            {
              "variant": "grok-4.7_xhigh",
              "score": 54.44444444444444
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 56.00000000000001,
          "variant": "grok-4.7_xhigh",
          "configurations": [
            {
              "variant": "grok-4.7_xhigh",
              "score": 56.00000000000001
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 28.999999999999996,
          "variant": "grok-4.7_xhigh",
          "configurations": [
            {
              "variant": "grok-4.7_xhigh",
              "score": 28.999999999999996
            }
          ]
        }
      },
      "catalog_link": "https://x.ai/news/grok-4-7",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek V4.1 Flash",
      "name": "DeepSeek V4.1 Flash",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2026-09-09",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-09",
      "catalog_releases": [
        {
          "name": "DeepSeek V4.1 Flash",
          "date": "2026-09-09",
          "link": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/main/DeepSeek_V41_Tech_Report.pdf"
        }
      ],
      "eci": 157.0,
      "eci_low": 155.08,
      "eci_high": 159.03,
      "access": "Open weights",
      "domain": "Language,Vision,Multimodal",
      "parameters": 552000000000.0,
      "compute_flop": 4.32e+24,
      "variants": [
        "deepseek-v4.1-flash_high",
        "deepseek-v4.1-flash_low",
        "deepseek-v4.1-flash_max",
        "deepseek-v4.1-flash_none",
        "deepseek-v4.1-flash_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 26.82926829268293,
          "variant": "deepseek-v4.1-flash_max",
          "configurations": [
            {
              "variant": "deepseek-v4.1-flash_max",
              "score": 26.82926829268293
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 64.44444444444444,
          "variant": "deepseek-v4.1-flash_max",
          "configurations": [
            {
              "variant": "deepseek-v4.1-flash_max",
              "score": 64.44444444444444
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 42.9,
          "variant": "deepseek-v4.1-flash_max",
          "configurations": [
            {
              "variant": "deepseek-v4.1-flash_max",
              "score": 42.9
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 43.0,
          "variant": "deepseek-v4.1-flash_max",
          "configurations": [
            {
              "variant": "deepseek-v4.1-flash_max",
              "score": 43.0
            }
          ]
        }
      },
      "catalog_link": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/main/DeepSeek_V41_Tech_Report.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-Image-2.5 Sunburst",
      "name": "GPT-Image-2.5 Sunburst",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-09-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-09-08",
      "catalog_releases": [
        {
          "name": "GPT-Image-2.5 Sunburst",
          "date": "2026-09-08",
          "link": "https://openai.com/index/introducing-chatgpt-images-2-5/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-chatgpt-images-2-5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-Image-2.5 Flare",
      "name": "GPT-Image-2.5 Flare",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-09-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-09-08",
      "catalog_releases": [
        {
          "name": "GPT-Image-2.5 Flare",
          "date": "2026-09-08",
          "link": "https://openai.com/index/introducing-chatgpt-images-2-5/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-chatgpt-images-2-5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ChatGPT Images 2.5",
      "name": "ChatGPT Images 2.5",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-09-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-09-08",
      "catalog_releases": [
        {
          "name": "ChatGPT Images 2.5",
          "date": "2026-09-08",
          "link": "https://openai.com/index/introducing-chatgpt-images-2-5/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Image generation,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-chatgpt-images-2-5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "WeatherNext 3",
      "name": "WeatherNext 3",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,Google Research",
      "lab_inferred": false,
      "date": "2026-09-03",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-09-03",
      "catalog_releases": [
        {
          "name": "WeatherNext 3",
          "date": "2026-09-03",
          "link": "https://arxiv.org/abs/2609.03582"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Earth science",
      "parameters": null,
      "compute_flop": 2.0725198384e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2609.03582",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-6 Astra",
      "name": "GPT-6 Astra",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-09-03",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-03",
      "catalog_releases": [
        {
          "name": "GPT-6 Astra",
          "date": "2026-09-03",
          "link": ""
        }
      ],
      "eci": 169.2,
      "eci_low": 166.33,
      "eci_high": 172.48,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": 9.4e+26,
      "variants": [
        "gpt-6-astra_high",
        "gpt-6-astra_low",
        "gpt-6-astra_max",
        "gpt-6-astra_medium",
        "gpt-6-astra_none",
        "gpt-6-astra_promax",
        "gpt-6-astra_unknown",
        "gpt-6-astra_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 97.6,
          "variant": "gpt-6-astra_high",
          "configurations": [
            {
              "variant": "gpt-6-astra_none",
              "score": 82.92682926829268
            },
            {
              "variant": "gpt-6-astra_low",
              "score": 87.8048780487805
            },
            {
              "variant": "gpt-6-astra_medium",
              "score": 97.5609756097561
            },
            {
              "variant": "gpt-6-astra_high",
              "score": 97.6
            },
            {
              "variant": "gpt-6-astra_xhigh",
              "score": 97.6
            },
            {
              "variant": "gpt-6-astra_max",
              "score": 97.6
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 60.0,
          "variant": "gpt-6-astra_max",
          "configurations": [
            {
              "variant": "gpt-6-astra_max",
              "score": 60.0
            },
            {
              "variant": "gpt-6-astra_high",
              "score": 46.7
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 75.6,
          "variant": "gpt-6-astra_max",
          "configurations": [
            {
              "variant": "gpt-6-astra_max",
              "score": 75.6
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 84.0,
          "variant": "gpt-6-astra_max",
          "configurations": [
            {
              "variant": "gpt-6-astra_max",
              "score": 84.0
            }
          ]
        }
      },
      "catalog_link": "",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Spark 1.3",
      "name": "Muse Spark 1.3",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-09-02",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-02",
      "catalog_releases": [
        {
          "name": "Muse Spark 1.3",
          "date": "2026-09-02",
          "link": "https://research.meta.ai/blog/introducing-muse-spark-1-3"
        }
      ],
      "eci": 158.25,
      "eci_low": 156.26,
      "eci_high": 160.41,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "muse-spark-1.3_high",
        "muse-spark-1.3_low",
        "muse-spark-1.3_max",
        "muse-spark-1.3_medium",
        "muse-spark-1.3_minimal",
        "muse-spark-1.3_unknown",
        "muse-spark-1.3_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 46.34146341463415,
          "variant": "muse-spark-1.3_max",
          "configurations": [
            {
              "variant": "muse-spark-1.3_max",
              "score": 46.34146341463415
            },
            {
              "variant": "muse-spark-1.3_xhigh",
              "score": 41.46341463414634
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 54.44444444444444,
          "variant": "muse-spark-1.3_max",
          "configurations": [
            {
              "variant": "muse-spark-1.3_max",
              "score": 54.44444444444444
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 25.0,
          "variant": "muse-spark-1.3_max",
          "configurations": [
            {
              "variant": "muse-spark-1.3_max",
              "score": 25.0
            },
            {
              "variant": "muse-spark-1.3_xhigh",
              "score": 14.14141414141414
            }
          ]
        }
      },
      "catalog_link": "https://research.meta.ai/blog/introducing-muse-spark-1-3",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3.8 Flash",
      "name": "Gemini 3.8 Flash",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-09-02",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-02",
      "catalog_releases": [
        {
          "name": "Gemini 3.8 Flash",
          "date": "2026-09-02",
          "link": "https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash"
        }
      ],
      "eci": 158.83,
      "eci_low": 156.86,
      "eci_high": 160.97,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Image generation,Video,Audio",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3.8-flash_high",
        "gemini-3.8-flash_low",
        "gemini-3.8-flash_medium",
        "gemini-3.8-flash_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 21.951219512195124,
          "variant": "gemini-3.8-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.8-flash_high",
              "score": 21.951219512195124
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 38.88888888888889,
          "variant": "gemini-3.8-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.8-flash_high",
              "score": 38.88888888888889
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 69.69999999999999,
          "variant": "gemini-3.8-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.8-flash_high",
              "score": 69.69999999999999
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 47.0,
          "variant": "gemini-3.8-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.8-flash_high",
              "score": 47.0
            }
          ]
        }
      },
      "catalog_link": "https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.8 Max (0902)",
      "name": "Qwen3.8 Max (0902)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-09-01",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 155.98,
      "eci_low": 153.89,
      "eci_high": 158.27,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.8-max-0902_unknown",
        "qwen3.8-max-0902_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 34.146341463414636,
          "variant": "qwen3.8-max-0902_xhigh",
          "configurations": [
            {
              "variant": "qwen3.8-max-0902_xhigh",
              "score": 34.146341463414636
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 47.77777777777778,
          "variant": "qwen3.8-max-0902_xhigh",
          "configurations": [
            {
              "variant": "qwen3.8-max-0902_xhigh",
              "score": 47.77777777777778
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 47.3,
          "variant": "qwen3.8-max-0902_xhigh",
          "configurations": [
            {
              "variant": "qwen3.8-max-0902_xhigh",
              "score": 47.3
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Fable 5.1",
      "name": "Claude Fable 5.1",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-09-01",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-09-01",
      "catalog_releases": [
        {
          "name": "Claude Fable 5.1",
          "date": "2026-09-01",
          "link": "https://www.anthropic.com/claude-fable-and-mythos-5-1"
        }
      ],
      "eci": 167.09,
      "eci_low": 164.43,
      "eci_high": 170.11,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-fable-5-1_high",
        "claude-fable-5-1_low",
        "claude-fable-5-1_max",
        "claude-fable-5-1_medium",
        "claude-fable-5-1_unknown",
        "claude-fable-5-1_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 87.8048780487805,
          "variant": "claude-fable-5-1_max",
          "configurations": [
            {
              "variant": "claude-fable-5-1_max",
              "score": 87.8048780487805
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 93.33333333333333,
          "variant": "claude-fable-5-1_max",
          "configurations": [
            {
              "variant": "claude-fable-5-1_high",
              "score": 73.3
            },
            {
              "variant": "claude-fable-5-1_max",
              "score": 93.33333333333333
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 70.8,
          "variant": "claude-fable-5-1_max",
          "configurations": [
            {
              "variant": "claude-fable-5-1_max",
              "score": 70.8
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 57.99999999999999,
          "variant": "claude-fable-5-1_max",
          "configurations": [
            {
              "variant": "claude-fable-5-1_max",
              "score": 57.99999999999999
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/claude-fable-and-mythos-5-1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.8 Flash",
      "name": "Qwen3.8 Flash",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-08-26",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.8-flash_xhigh"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen 3.8 27B",
      "name": "Qwen 3.8 27B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-08-14",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 152.65,
      "eci_low": 150.7,
      "eci_high": 154.76,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.8-27b",
        "qwen3.8-27b_low",
        "qwen3.8-27b_medium",
        "qwen3.8-27b_none",
        "qwen3.8-27b_xhigh"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3.7 Flash",
      "name": "Gemini 3.7 Flash",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-08-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 158.38,
      "eci_low": 156.41,
      "eci_high": 160.53,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3.7-flash_high",
        "gemini-3.7-flash_low",
        "gemini-3.7-flash_medium",
        "gemini-3.7-flash_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 36.58536585365854,
          "variant": "gemini-3.7-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.7-flash_high",
              "score": 36.58536585365854
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 41.11111111111111,
          "variant": "gemini-3.7-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.7-flash_high",
              "score": 41.11111111111111
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 69.19999999999999,
          "variant": "gemini-3.7-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.7-flash_high",
              "score": 69.19999999999999
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 37.0,
          "variant": "gemini-3.7-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.7-flash_high",
              "score": 37.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek V4 Pro 0813",
      "name": "DeepSeek V4 Pro 0813",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2026-08-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 155.96,
      "eci_low": 154.1,
      "eci_high": 157.99,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 9.702e+24,
      "variants": [
        "deepseek-v4-pro-0813_high",
        "deepseek-v4-pro-0813_low",
        "deepseek-v4-pro-0813_max",
        "deepseek-v4-pro-0813_none",
        "deepseek-v4-pro-0813_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 26.82926829268293,
          "variant": "deepseek-v4-pro-0813_max",
          "configurations": [
            {
              "variant": "deepseek-v4-pro-0813_max",
              "score": 26.82926829268293
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 33.33333333333333,
          "variant": "deepseek-v4-pro-0813_max",
          "configurations": [
            {
              "variant": "deepseek-v4-pro-0813_max",
              "score": 33.33333333333333
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 52.90581162324649,
          "variant": "deepseek-v4-pro-0813_max",
          "configurations": [
            {
              "variant": "deepseek-v4-pro-0813_max",
              "score": 52.90581162324649
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 43.0,
          "variant": "deepseek-v4-pro-0813_max",
          "configurations": [
            {
              "variant": "deepseek-v4-pro-0813_max",
              "score": 43.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4.6",
      "name": "Grok 4.6",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2026-08-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 157.82,
      "eci_low": 155.86,
      "eci_high": 160.0,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-4.6_high",
        "grok-4.6_low",
        "grok-4.6_medium",
        "grok-4.6_unknown",
        "grok-4.6_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 31.70731707317073,
          "variant": "grok-4.6_xhigh",
          "configurations": [
            {
              "variant": "grok-4.6_xhigh",
              "score": 31.70731707317073
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 41.11111111111111,
          "variant": "grok-4.6_xhigh",
          "configurations": [
            {
              "variant": "grok-4.6_xhigh",
              "score": 41.11111111111111
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 49.3,
          "variant": "grok-4.6_high",
          "configurations": [
            {
              "variant": "grok-4.6_high",
              "score": 49.3
            },
            {
              "variant": "grok-4.6_xhigh",
              "score": 48.9
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 34.0,
          "variant": "grok-4.6_xhigh",
          "configurations": [
            {
              "variant": "grok-4.6_xhigh",
              "score": 34.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.6 Cyber",
      "name": "GPT-5.6 Cyber",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-08-11",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-08-11",
      "catalog_releases": [
        {
          "name": "GPT-5.6 Cyber",
          "date": "2026-08-11",
          "link": "https://openai.com/index/expanding-daybreak-as-the-cyber-defense-window-narrows/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Cybersecurity,Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/expanding-daybreak-as-the-cyber-defense-window-narrows/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Glimmer",
      "name": "Muse Glimmer",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-08-10",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "muse-glimmer_high"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.8-2.4T-A95B",
      "name": "Qwen3.8-2.4T-A95B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-08-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-08-08",
      "catalog_releases": [
        {
          "name": "Qwen3.8-2.4T-A95B",
          "date": "2026-08-08",
          "link": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": 2400000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Spark 1.2",
      "name": "Muse Spark 1.2",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-08-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-08-05",
      "catalog_releases": [
        {
          "name": "Muse Spark 1.2",
          "date": "2026-08-05",
          "link": "https://research.meta.ai/blog/introducing-muse-code-and-muse-spark-1-2"
        }
      ],
      "eci": 155.06,
      "eci_low": 152.79,
      "eci_high": 157.48,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "muse-spark-1.2",
        "muse-spark-1.2_xhigh"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 60.30150753768844,
          "variant": "muse-spark-1.2_xhigh",
          "configurations": [
            {
              "variant": "muse-spark-1.2_xhigh",
              "score": 60.30150753768844
            }
          ]
        }
      },
      "catalog_link": "https://research.meta.ai/blog/introducing-muse-code-and-muse-spark-1-2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen 3.8 Max",
      "name": "Qwen 3.8 Max",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-08-02",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-07-19",
      "catalog_releases": [
        {
          "name": "Qwen 3.8 Max",
          "date": "2026-07-19",
          "link": "https://qwen.ai/blog?id=qwen3.8"
        }
      ],
      "eci": 157.59,
      "eci_low": 155.47,
      "eci_high": 159.94,
      "access": "Closed weights",
      "domain": "Language,Vision,Video",
      "parameters": 2400000000000.0,
      "compute_flop": null,
      "variants": [
        "qwen3.8-max_unknown",
        "qwen3.8-max_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 46.34146341463415,
          "variant": "qwen3.8-max_xhigh",
          "configurations": [
            {
              "variant": "qwen3.8-max_xhigh",
              "score": 46.34146341463415
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 52.22222222222223,
          "variant": "qwen3.8-max_xhigh",
          "configurations": [
            {
              "variant": "qwen3.8-max_xhigh",
              "score": 52.22222222222223
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 45.800000000000004,
          "variant": "qwen3.8-max_xhigh",
          "configurations": [
            {
              "variant": "qwen3.8-max_xhigh",
              "score": 45.800000000000004
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 38.0,
          "variant": "qwen3.8-max_xhigh",
          "configurations": [
            {
              "variant": "qwen3.8-max_xhigh",
              "score": 38.0
            }
          ]
        }
      },
      "catalog_link": "https://qwen.ai/blog?id=qwen3.8",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek V4 Flash 0731",
      "name": "DeepSeek V4 Flash 0731",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2026-07-31",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 154.43,
      "eci_low": 152.55,
      "eci_high": 156.42,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.496e+24,
      "variants": [
        "deepseek-v4-flash-0731_high",
        "deepseek-v4-flash-0731_low",
        "deepseek-v4-flash-0731_max",
        "deepseek-v4-flash-0731_none",
        "deepseek-v4-flash-0731_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 24.390243902439025,
          "variant": "deepseek-v4-flash-0731_max",
          "configurations": [
            {
              "variant": "deepseek-v4-flash-0731_max",
              "score": 24.390243902439025
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 33.633633633633636,
          "variant": "deepseek-v4-flash-0731_max",
          "configurations": [
            {
              "variant": "deepseek-v4-flash-0731_max",
              "score": 33.633633633633636
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 34.0,
          "variant": "deepseek-v4-flash-0731_max",
          "configurations": [
            {
              "variant": "deepseek-v4-flash-0731_max",
              "score": 34.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.7 Flash",
      "name": "Qwen3.7 Flash",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-07-27",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.7-flash",
        "qwen3.7-flash_none"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 15.0,
          "variant": "qwen3.7-flash",
          "configurations": [
            {
              "variant": "qwen3.7-flash",
              "score": 15.0
            },
            {
              "variant": "qwen3.7-flash_none",
              "score": 14.000000000000002
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Opus 5",
      "name": "Claude Opus 5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-07-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-07-24",
      "catalog_releases": [
        {
          "name": "Claude Opus 5",
          "date": "2026-07-24",
          "link": "https://www.anthropic.com/news/claude-opus-5"
        }
      ],
      "eci": 164.74,
      "eci_low": 162.3,
      "eci_high": 167.66,
      "access": "Closed weights",
      "domain": "Language,Vision,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-opus-5",
        "claude-opus-5_high",
        "claude-opus-5_low",
        "claude-opus-5_max",
        "claude-opus-5_medium",
        "claude-opus-5_unknown",
        "claude-opus-5_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 73.17073170731707,
          "variant": "claude-opus-5_max",
          "configurations": [
            {
              "variant": "claude-opus-5_max",
              "score": 73.17073170731707
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 87.77777777777777,
          "variant": "claude-opus-5_max",
          "configurations": [
            {
              "variant": "claude-opus-5_max",
              "score": 87.77777777777777
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 59.9,
          "variant": "claude-opus-5_max",
          "configurations": [
            {
              "variant": "claude-opus-5_max",
              "score": 59.9
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 59.0,
          "variant": "claude-opus-5_max",
          "configurations": [
            {
              "variant": "claude-opus-5",
              "score": 37.0
            },
            {
              "variant": "claude-opus-5_max",
              "score": 59.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/news/claude-opus-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3.6 Flash",
      "name": "Gemini 3.6 Flash",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-07-21",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 155.24,
      "eci_low": 153.42,
      "eci_high": 157.18,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3.6-flash_high",
        "gemini-3.6-flash_low",
        "gemini-3.6-flash_medium",
        "gemini-3.6-flash_minimal",
        "gemini-3.6-flash_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 21.951219512195124,
          "variant": "gemini-3.6-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.6-flash_high",
              "score": 21.951219512195124
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 23.333333333333332,
          "variant": "gemini-3.6-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.6-flash_high",
              "score": 23.333333333333332
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 66.2,
          "variant": "gemini-3.6-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.6-flash_high",
              "score": 66.2
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 30.0,
          "variant": "gemini-3.6-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.6-flash_minimal",
              "score": 25.0
            },
            {
              "variant": "gemini-3.6-flash_low",
              "score": 23.0
            },
            {
              "variant": "gemini-3.6-flash_high",
              "score": 30.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3.5 Flash-Lite",
      "name": "Gemini 3.5 Flash-Lite",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-07-21",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 145.65,
      "eci_low": 143.98,
      "eci_high": 147.32,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3.5-flash-lite",
        "gemini-3.5-flash-lite_high",
        "gemini-3.5-flash-lite_low",
        "gemini-3.5-flash-lite_medium",
        "gemini-3.5-flash-lite_minimal"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 0.0,
          "variant": "gemini-3.5-flash-lite_high",
          "configurations": [
            {
              "variant": "gemini-3.5-flash-lite_high",
              "score": 0.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 19.0,
          "variant": "gemini-3.5-flash-lite_low",
          "configurations": [
            {
              "variant": "gemini-3.5-flash-lite_minimal",
              "score": 12.0
            },
            {
              "variant": "gemini-3.5-flash-lite_low",
              "score": 19.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Spark 1.1",
      "name": "Muse Spark 1.1",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-07-09",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 154.92,
      "eci_low": 152.33,
      "eci_high": 157.65,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "muse-spark-1.1",
        "muse-spark-1.1_high",
        "muse-spark-1.1_medium",
        "muse-spark-1.1_xhigh"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 57.78894472361809,
          "variant": "muse-spark-1.1",
          "configurations": [
            {
              "variant": "muse-spark-1.1",
              "score": 57.78894472361809
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.6 Terra",
      "name": "GPT-5.6 Terra",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-07-09",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 160.78,
      "eci_low": 158.58,
      "eci_high": 163.23,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.6-terra_high",
        "gpt-5.6-terra_low",
        "gpt-5.6-terra_max",
        "gpt-5.6-terra_medium",
        "gpt-5.6-terra_none",
        "gpt-5.6-terra_unknown",
        "gpt-5.6-terra_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 70.73170731707317,
          "variant": "gpt-5.6-terra_max",
          "configurations": [
            {
              "variant": "gpt-5.6-terra_max",
              "score": 70.73170731707317
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 41.11111111111111,
          "variant": "gpt-5.6-terra_max",
          "configurations": [
            {
              "variant": "gpt-5.6-terra_max",
              "score": 41.11111111111111
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 43.2,
          "variant": "gpt-5.6-terra_max",
          "configurations": [
            {
              "variant": "gpt-5.6-terra_max",
              "score": 43.2
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 35.0,
          "variant": "gpt-5.6-terra_max",
          "configurations": [
            {
              "variant": "gpt-5.6-terra_medium",
              "score": 10.0
            },
            {
              "variant": "gpt-5.6-terra_low",
              "score": 19.0
            },
            {
              "variant": "gpt-5.6-terra_none",
              "score": 14.000000000000002
            },
            {
              "variant": "gpt-5.6-terra_max",
              "score": 35.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.6 Sol",
      "name": "GPT-5.6 Sol",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-07-09",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 163.05,
      "eci_low": 160.76,
      "eci_high": 165.59,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.6-sol_high",
        "gpt-5.6-sol_low",
        "gpt-5.6-sol_max",
        "gpt-5.6-sol_medium",
        "gpt-5.6-sol_none",
        "gpt-5.6-sol_promax",
        "gpt-5.6-sol_prounknown",
        "gpt-5.6-sol_proxhigh",
        "gpt-5.6-sol_unknown",
        "gpt-5.6-sol_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 82.92682926829268,
          "variant": "gpt-5.6-sol_max",
          "configurations": [
            {
              "variant": "gpt-5.6-sol_promax",
              "score": 80.48780487804879
            },
            {
              "variant": "gpt-5.6-sol_max",
              "score": 82.92682926829268
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 20.0,
          "variant": "gpt-5.6-sol_high",
          "configurations": [
            {
              "variant": "gpt-5.6-sol_high",
              "score": 20.0
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 69.69999999999999,
          "variant": "gpt-5.6-sol_max",
          "configurations": [
            {
              "variant": "gpt-5.6-sol_max",
              "score": 69.69999999999999
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 57.99999999999999,
          "variant": "gpt-5.6-sol_max",
          "configurations": [
            {
              "variant": "gpt-5.6-sol_low",
              "score": 26.0
            },
            {
              "variant": "gpt-5.6-sol_none",
              "score": 33.0
            },
            {
              "variant": "gpt-5.6-sol_max",
              "score": 57.99999999999999
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.6 Luna",
      "name": "GPT-5.6 Luna",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-07-09",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 156.7,
      "eci_low": 154.87,
      "eci_high": 158.77,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.6-luna_high",
        "gpt-5.6-luna_low",
        "gpt-5.6-luna_max",
        "gpt-5.6-luna_medium",
        "gpt-5.6-luna_none",
        "gpt-5.6-luna_unknown",
        "gpt-5.6-luna_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 60.97560975609756,
          "variant": "gpt-5.6-luna_max",
          "configurations": [
            {
              "variant": "gpt-5.6-luna_max",
              "score": 60.97560975609756
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 21.11111111111111,
          "variant": "gpt-5.6-luna_max",
          "configurations": [
            {
              "variant": "gpt-5.6-luna_max",
              "score": 21.11111111111111
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 41.0,
          "variant": "gpt-5.6-luna_max",
          "configurations": [
            {
              "variant": "gpt-5.6-luna_max",
              "score": 41.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 21.0,
          "variant": "gpt-5.6-luna_max",
          "configurations": [
            {
              "variant": "gpt-5.6-luna_none",
              "score": 20.0
            },
            {
              "variant": "gpt-5.6-luna_medium",
              "score": 12.0
            },
            {
              "variant": "gpt-5.6-luna_low",
              "score": 17.0
            },
            {
              "variant": "gpt-5.6-luna_max",
              "score": 21.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4.5",
      "name": "Grok 4.5",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2026-07-08",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 154.97,
      "eci_low": 153.23,
      "eci_high": 156.91,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-4.5_high",
        "grok-4.5_low",
        "grok-4.5_medium",
        "grok-4.5_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 24.390243902439025,
          "variant": "grok-4.5_high",
          "configurations": [
            {
              "variant": "grok-4.5_high",
              "score": 24.390243902439025
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 32.22222222222222,
          "variant": "grok-4.5_high",
          "configurations": [
            {
              "variant": "grok-4.5_high",
              "score": 32.22222222222222
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 48.3,
          "variant": "grok-4.5_high",
          "configurations": [
            {
              "variant": "grok-4.5_high",
              "score": 48.3
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Sonnet 5",
      "name": "Claude Sonnet 5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-06-30",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 156.9,
      "eci_low": 154.91,
      "eci_high": 159.08,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-sonnet-5",
        "claude-sonnet-5_16K",
        "claude-sonnet-5_high",
        "claude-sonnet-5_low",
        "claude-sonnet-5_max",
        "claude-sonnet-5_medium",
        "claude-sonnet-5_unknown",
        "claude-sonnet-5_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 29.268292682926827,
          "variant": "claude-sonnet-5_max",
          "configurations": [
            {
              "variant": "claude-sonnet-5_max",
              "score": 29.268292682926827
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 63.33333333333333,
          "variant": "claude-sonnet-5_max",
          "configurations": [
            {
              "variant": "claude-sonnet-5_max",
              "score": 63.33333333333333
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 33.7,
          "variant": "claude-sonnet-5_max",
          "configurations": [
            {
              "variant": "claude-sonnet-5_xhigh",
              "score": 32.9
            },
            {
              "variant": "claude-sonnet-5_max",
              "score": 33.7
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 35.0,
          "variant": "claude-sonnet-5_max",
          "configurations": [
            {
              "variant": "claude-sonnet-5",
              "score": 16.0
            },
            {
              "variant": "claude-sonnet-5_max",
              "score": 35.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Mythos 5",
      "name": "Claude Mythos 5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-06-09",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-06-09",
      "catalog_releases": [
        {
          "name": "Claude Mythos 5",
          "date": "2026-06-09",
          "link": "https://www.anthropic.com/news/claude-fable-5-mythos-5"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/news/claude-fable-5-mythos-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Fable 5",
      "name": "Claude Fable 5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-06-09",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-06-09",
      "catalog_releases": [
        {
          "name": "Claude Fable 5",
          "date": "2026-06-09",
          "link": "https://www.anthropic.com/news/claude-fable-5-mythos-5"
        }
      ],
      "eci": 164.21,
      "eci_low": 161.78,
      "eci_high": 166.86,
      "access": "Closed weights",
      "domain": "Language,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-fable-5",
        "claude-fable-5-fallback_max",
        "claude-fable-5_high",
        "claude-fable-5_low",
        "claude-fable-5_max",
        "claude-fable-5_medium",
        "claude-fable-5_unknown",
        "claude-fable-5_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 90.2,
          "variant": "claude-fable-5_max",
          "configurations": [
            {
              "variant": "claude-fable-5_max",
              "score": 90.2
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 70.0,
          "variant": "claude-fable-5_max",
          "configurations": [
            {
              "variant": "claude-fable-5_max",
              "score": 70.0
            },
            {
              "variant": "claude-fable-5_high",
              "score": 63.88888888888889
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 70.7,
          "variant": "claude-fable-5_xhigh",
          "configurations": [
            {
              "variant": "claude-fable-5_xhigh",
              "score": 70.7
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 52.0,
          "variant": "claude-fable-5_max",
          "configurations": [
            {
              "variant": "claude-fable-5_max",
              "score": 52.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/news/claude-fable-5-mythos-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.7-Plus",
      "name": "Qwen3.7-Plus",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-06-02",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-06-03",
      "catalog_releases": [
        {
          "name": "Qwen3.7-Plus",
          "date": "2026-06-03",
          "link": "https://qwen.ai/blog?id=qwen3.7-plus\n\nhttps://www.qwencloud.com/models/qwen3.7-plus"
        }
      ],
      "eci": 147.95,
      "eci_low": 145.91,
      "eci_high": 150.04,
      "access": "Closed weights",
      "domain": "Language,Vision,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.7-plus",
        "qwen3.7-plus_none"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 17.0,
          "variant": "qwen3.7-plus",
          "configurations": [
            {
              "variant": "qwen3.7-plus",
              "score": 17.0
            },
            {
              "variant": "qwen3.7-plus_none",
              "score": 17.0
            }
          ]
        }
      },
      "catalog_link": "https://qwen.ai/blog?id=qwen3.7-plus\n\nhttps://www.qwencloud.com/models/qwen3.7-plus",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok Build 0.1",
      "name": "Grok Build 0.1",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2026-05-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-build-0.1"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Opus 4.8",
      "name": "Claude Opus 4.8",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-05-28",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-05-28",
      "catalog_releases": [
        {
          "name": "Claude Opus 4.8",
          "date": "2026-05-28",
          "link": "https://www.anthropic.com/news/claude-opus-4-8"
        }
      ],
      "eci": 159.91,
      "eci_low": 157.82,
      "eci_high": 162.21,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-opus-4-8_24K",
        "claude-opus-4-8_high",
        "claude-opus-4-8_low",
        "claude-opus-4-8_max",
        "claude-opus-4-8_medium",
        "claude-opus-4-8_none",
        "claude-opus-4-8_unknown",
        "claude-opus-4-8_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 56.09756097560976,
          "variant": "claude-opus-4-8_max",
          "configurations": [
            {
              "variant": "claude-opus-4-8_max",
              "score": 56.09756097560976
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 65.55555555555556,
          "variant": "claude-opus-4-8_max",
          "configurations": [
            {
              "variant": "claude-opus-4-8_max",
              "score": 65.55555555555556
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 53.0,
          "variant": "claude-opus-4-8_max",
          "configurations": [
            {
              "variant": "claude-opus-4-8_max",
              "score": 53.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 36.0,
          "variant": "claude-opus-4-8_max",
          "configurations": [
            {
              "variant": "claude-opus-4-8_xhigh",
              "score": 31.0
            },
            {
              "variant": "claude-opus-4-8_max",
              "score": 36.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/news/claude-opus-4-8",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.7-Max",
      "name": "Qwen3.7-Max",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-05-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 154.58,
      "eci_low": 152.57,
      "eci_high": 156.67,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.7-max",
        "qwen3.7-max_none"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 34.146341463414636,
          "variant": "qwen3.7-max",
          "configurations": [
            {
              "variant": "qwen3.7-max",
              "score": 34.146341463414636
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 55.76730190571715,
          "variant": "qwen3.7-max",
          "configurations": [
            {
              "variant": "qwen3.7-max",
              "score": 55.76730190571715
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 32.0,
          "variant": "qwen3.7-max",
          "configurations": [
            {
              "variant": "qwen3.7-max_none",
              "score": 26.0
            },
            {
              "variant": "qwen3.7-max",
              "score": 32.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3.5 Flash",
      "name": "Gemini 3.5 Flash",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-05-19",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-05-19",
      "catalog_releases": [
        {
          "name": "Gemini 3.5 Flash",
          "date": "2026-05-19",
          "link": "https://deepmind.google/models/model-cards/gemini-3-5-flash/"
        }
      ],
      "eci": 155.35,
      "eci_low": 153.52,
      "eci_high": 157.38,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Audio,Video,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3.5-flash_high",
        "gemini-3.5-flash_low",
        "gemini-3.5-flash_medium",
        "gemini-3.5-flash_minimal",
        "gemini-3.5-flash_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 26.82926829268293,
          "variant": "gemini-3.5-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.5-flash_high",
              "score": 26.82926829268293
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 66.2,
          "variant": "gemini-3.5-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.5-flash_high",
              "score": 66.2
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 32.0,
          "variant": "gemini-3.5-flash_high",
          "configurations": [
            {
              "variant": "gemini-3.5-flash_low",
              "score": 28.000000000000004
            },
            {
              "variant": "gemini-3.5-flash_high",
              "score": 32.0
            }
          ]
        }
      },
      "catalog_link": "https://deepmind.google/models/model-cards/gemini-3-5-flash/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AI Co-Mathematician",
      "name": "AI Co-Mathematician",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-05-08",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gdm-ai-co-mathematician"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 75.6,
          "variant": "gdm-ai-co-mathematician",
          "configurations": [
            {
              "variant": "gdm-ai-co-mathematician",
              "score": 75.6
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.5 Cyber",
      "name": "GPT-5.5 Cyber",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-05-07",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-05-07",
      "catalog_releases": [
        {
          "name": "GPT-5.5 Cyber",
          "date": "2026-05-07",
          "link": "https://openai.com/index/gpt-5-5-with-trusted-access-for-cyber/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Cybersecurity",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.5-cyber"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/gpt-5-5-with-trusted-access-for-cyber/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.5 Instant",
      "name": "GPT-5.5 Instant",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-05-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-05-05",
      "catalog_releases": [
        {
          "name": "GPT-5.5 Instant",
          "date": "2026-05-05",
          "link": "https://openai.com/index/gpt-5-5-instant/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.5-instant"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 2.4390243902439024,
          "variant": "gpt-5.5-instant",
          "configurations": [
            {
              "variant": "gpt-5.5-instant",
              "score": 2.4390243902439024
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/gpt-5-5-instant/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Medium 3.5",
      "name": "Mistral Medium 3.5",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2026-04-28",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-29",
      "catalog_releases": [
        {
          "name": "Mistral Medium 3.5",
          "date": "2026-04-29",
          "link": "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B"
        }
      ],
      "eci": 141.57,
      "eci_low": 138.4,
      "eci_high": 144.68,
      "access": "Open weights",
      "domain": "Language,Multimodal",
      "parameters": 128000000000.0,
      "compute_flop": null,
      "variants": [
        "mistral-medium-2604"
      ],
      "benchmarks": {},
      "catalog_link": "https://huggingface.co/mistralai/Mistral-Medium-3.5-128B",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen 3.6 Flash",
      "name": "Qwen 3.6 Flash",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-04-27",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 143.76,
      "eci_low": 141.76,
      "eci_high": 145.7,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.6-flash",
        "qwen3.6-flash_none"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 15.915915915915916,
          "variant": "qwen3.6-flash",
          "configurations": [
            {
              "variant": "qwen3.6-flash",
              "score": 15.915915915915916
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 18.0,
          "variant": "qwen3.6-flash_none",
          "configurations": [
            {
              "variant": "qwen3.6-flash",
              "score": 12.0
            },
            {
              "variant": "qwen3.6-flash_none",
              "score": 18.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V4-Pro",
      "name": "DeepSeek-V4-Pro",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2026-04-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-24",
      "catalog_releases": [
        {
          "name": "DeepSeek-V4-Pro",
          "date": "2026-04-24",
          "link": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf"
        }
      ],
      "eci": 150.21,
      "eci_low": 148.47,
      "eci_high": 152.04,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 1600000000000.0,
      "compute_flop": 9.702e+24,
      "variants": [
        "deepseek-v4-pro_high",
        "deepseek-v4-pro_max",
        "deepseek-v4-pro_none",
        "deepseek-v4-pro_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 2.4390243902439024,
          "variant": "deepseek-v4-pro_max",
          "configurations": [
            {
              "variant": "deepseek-v4-pro_max",
              "score": 2.4390243902439024
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 46.9939879759519,
          "variant": "deepseek-v4-pro_max",
          "configurations": [
            {
              "variant": "deepseek-v4-pro_max",
              "score": 46.9939879759519
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 17.0,
          "variant": "deepseek-v4-pro_none",
          "configurations": [
            {
              "variant": "deepseek-v4-pro_none",
              "score": 17.0
            }
          ]
        }
      },
      "catalog_link": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V4-Flash",
      "name": "DeepSeek-V4-Flash",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2026-04-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-24",
      "catalog_releases": [
        {
          "name": "DeepSeek-V4-Flash",
          "date": "2026-04-24",
          "link": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf"
        }
      ],
      "eci": 146.84,
      "eci_low": 143.6,
      "eci_high": 150.16,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 284000000000.0,
      "compute_flop": 2.496e+24,
      "variants": [
        "deepseek-v4-flash_high",
        "deepseek-v4-flash_max",
        "deepseek-v4-flash_none",
        "deepseek-v4-flash_unknown"
      ],
      "benchmarks": {},
      "catalog_link": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/DeepSeek_V4.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.5 Pro",
      "name": "GPT-5.5 Pro",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-04-23",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-23",
      "catalog_releases": [
        {
          "name": "GPT-5.5 Pro",
          "date": "2026-04-23",
          "link": "https://openai.com/index/introducing-gpt-5-5/"
        }
      ],
      "eci": 162.77,
      "eci_low": 160.14,
      "eci_high": 165.77,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.5-pro-pre-release_high",
        "gpt-5.5-pro-pre-release_xhigh",
        "gpt-5.5-pro_high",
        "gpt-5.5-pro_unknown",
        "gpt-5.5-pro_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 78.04878048780488,
          "variant": "gpt-5.5-pro_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.5-pro_xhigh",
              "score": 78.04878048780488
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-gpt-5-5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.5",
      "name": "GPT-5.5",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-04-23",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-23",
      "catalog_releases": [
        {
          "name": "GPT-5.5",
          "date": "2026-04-23",
          "link": "https://openai.com/index/introducing-gpt-5-5/"
        }
      ],
      "eci": 160.19,
      "eci_low": 158.2,
      "eci_high": 162.43,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.5-pre-release_xhigh",
        "gpt-5.5_high",
        "gpt-5.5_low",
        "gpt-5.5_medium",
        "gpt-5.5_none",
        "gpt-5.5_unknown",
        "gpt-5.5_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 72.5,
          "variant": "gpt-5.5_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.5_xhigh",
              "score": 72.5
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 23.333333333333332,
          "variant": "gpt-5.5_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.5_xhigh",
              "score": 23.333333333333332
            },
            {
              "variant": "gpt-5.5_high",
              "score": 10.0
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 63.0,
          "variant": "gpt-5.5_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.5_xhigh",
              "score": 63.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 56.00000000000001,
          "variant": "gpt-5.5_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.5_low",
              "score": 28.000000000000004
            },
            {
              "variant": "gpt-5.5_none",
              "score": 18.0
            },
            {
              "variant": "gpt-5.5_high",
              "score": 52.0
            },
            {
              "variant": "gpt-5.5_xhigh",
              "score": 56.00000000000001
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-gpt-5-5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.6 27B",
      "name": "Qwen3.6 27B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-04-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 146.78,
      "eci_low": 144.68,
      "eci_high": 148.89,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.6-27b",
        "qwen3.6-27b_none"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 7.000000000000001,
          "variant": "qwen3.6-27b_none",
          "configurations": [
            {
              "variant": "qwen3.6-27b_none",
              "score": 7.000000000000001
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT Image 2",
      "name": "GPT Image 2",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-04-21",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-04-21",
      "catalog_releases": [
        {
          "name": "GPT Image 2",
          "date": "2026-04-21",
          "link": "https://openai.com/index/introducing-chatgpt-images-2-0/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-chatgpt-images-2-0/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen 3.6 Max (Preview)",
      "name": "Qwen 3.6 Max (Preview)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-04-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 149.88,
      "eci_low": 147.87,
      "eci_high": 152.07,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.6-max-preview",
        "qwen3.6-max-preview_none"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 52.00400801603207,
          "variant": "qwen3.6-max-preview",
          "configurations": [
            {
              "variant": "qwen3.6-max-preview",
              "score": 52.00400801603207
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 19.0,
          "variant": "qwen3.6-max-preview_none",
          "configurations": [
            {
              "variant": "qwen3.6-max-preview_none",
              "score": 19.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4.3 Beta",
      "name": "Grok 4.3 Beta",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2026-04-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 149.91,
      "eci_low": 147.94,
      "eci_high": 151.92,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-4.3_high",
        "grok-4.3_none",
        "grok-4.3_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 14.634146341463413,
          "variant": "grok-4.3_high",
          "configurations": [
            {
              "variant": "grok-4.3_high",
              "score": 14.634146341463413
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 33.2,
          "variant": "grok-4.3_high",
          "configurations": [
            {
              "variant": "grok-4.3_high",
              "score": 33.2
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Opus 4.7",
      "name": "Claude Opus 4.7",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-04-16",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-16",
      "catalog_releases": [
        {
          "name": "Claude Opus 4.7",
          "date": "2026-04-16",
          "link": "https://www.anthropic.com/news/claude-opus-4-7"
        }
      ],
      "eci": 157.5,
      "eci_low": 155.63,
      "eci_high": 159.46,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-opus-4-7",
        "claude-opus-4-7_high",
        "claude-opus-4-7_low",
        "claude-opus-4-7_max",
        "claude-opus-4-7_medium",
        "claude-opus-4-7_unknown",
        "claude-opus-4-7_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 31.70731707317073,
          "variant": "claude-opus-4-7_max",
          "configurations": [
            {
              "variant": "claude-opus-4-7_max",
              "score": 31.70731707317073
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 43.333333333333336,
          "variant": "claude-opus-4-7_max",
          "configurations": [
            {
              "variant": "claude-opus-4-7_max",
              "score": 43.333333333333336
            },
            {
              "variant": "claude-opus-4-7_high",
              "score": 31.11111111111111
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 51.7,
          "variant": "claude-opus-4-7_xhigh",
          "configurations": [
            {
              "variant": "claude-opus-4-7_xhigh",
              "score": 51.7
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 28.000000000000004,
          "variant": "claude-opus-4-7_max",
          "configurations": [
            {
              "variant": "claude-opus-4-7",
              "score": 13.0
            },
            {
              "variant": "claude-opus-4-7_max",
              "score": 28.000000000000004
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/news/claude-opus-4-7",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini Flash 3.1 TTS",
      "name": "Gemini Flash 3.1 TTS",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-04-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-04-15",
      "catalog_releases": [
        {
          "name": "Gemini Flash 3.1 TTS",
          "date": "2026-04-15",
          "link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-tts/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Audio",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-tts/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen 3.6 35B-A3B",
      "name": "Qwen 3.6 35B-A3B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-04-14",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 144.11,
      "eci_low": 142.23,
      "eci_high": 146.0,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.6-35b-a3b",
        "qwen3.6-35b-a3b_none"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 22.0,
          "variant": "qwen3.6-35b-a3b_none",
          "configurations": [
            {
              "variant": "qwen3.6-35b-a3b_none",
              "score": 22.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini Robotics-ER 1.6",
      "name": "Gemini Robotics-ER 1.6",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2026-04-14",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-robotics-er-1.6-preview"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.4-Cyber",
      "name": "GPT-5.4-Cyber",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-04-14",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-14",
      "catalog_releases": [
        {
          "name": "GPT-5.4-Cyber",
          "date": "2026-04-14",
          "link": "https://openai.com/index/scaling-trusted-access-for-cyber-defense/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Limited access",
      "domain": "Cybersecurity,Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.4-cyber"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/scaling-trusted-access-for-cyber-defense/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Muse Spark",
      "name": "Muse Spark",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2026-04-08",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-08",
      "catalog_releases": [
        {
          "name": "Muse Spark",
          "date": "2026-04-08",
          "link": "https://ai.meta.com/blog/introducing-muse-spark-msl/"
        }
      ],
      "eci": 153.16,
      "eci_low": 150.94,
      "eci_high": 155.46,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "muse-spark"
      ],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/blog/introducing-muse-spark-msl/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Mythos Preview",
      "name": "Claude Mythos Preview",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-04-07",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-07",
      "catalog_releases": [
        {
          "name": "Claude Mythos Preview",
          "date": "2026-04-07",
          "link": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf\nhttps://www.anthropic.com/glasswing"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Limited access",
      "domain": "Language,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-mythos-preview",
        "claude-mythos-preview-early"
      ],
      "benchmarks": {},
      "catalog_link": "https://www-cdn.anthropic.com/7624816413e9b4d2e3ba620c5a5e091b98b190a5/Claude%20Mythos%20Preview%20System%20Card.pdf\nhttps://www.anthropic.com/glasswing",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 4 31B IT",
      "name": "Gemma 4 31B IT",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-04-02",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 144.12,
      "eci_low": 141.72,
      "eci_high": 146.59,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemma-4-31b-it",
        "gemma-4-31b-it_minimal"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 10.4,
          "variant": "gemma-4-31b-it",
          "configurations": [
            {
              "variant": "gemma-4-31b-it",
              "score": 10.4
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 4 26B A4B",
      "name": "Gemma 4 26B A4B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-04-02",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 141.58,
      "eci_low": 139.14,
      "eci_high": 144.02,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemma-4-26b-a4b",
        "gemma-4-26b-a4b-it_minimal",
        "gemma-4-26b-a4b_minimal"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen 3.6 Plus",
      "name": "Qwen 3.6 Plus",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-03-31",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-04-01",
      "catalog_releases": [
        {
          "name": "Qwen 3.6 Plus",
          "date": "2026-04-01",
          "link": "https://qwen.ai/blog?id=qwen3.6"
        }
      ],
      "eci": 148.91,
      "eci_low": 146.96,
      "eci_high": 150.92,
      "access": "Closed weights",
      "domain": "Language,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.6-plus",
        "qwen3.6-plus_none"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 44.14414414414414,
          "variant": "qwen3.6-plus",
          "configurations": [
            {
              "variant": "qwen3.6-plus",
              "score": 44.14414414414414
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 12.0,
          "variant": "qwen3.6-plus_none",
          "configurations": [
            {
              "variant": "qwen3.6-plus_none",
              "score": 12.0
            }
          ]
        }
      },
      "catalog_link": "https://qwen.ai/blog?id=qwen3.6",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-Omni-Plus",
      "name": "Qwen3.5-Omni-Plus",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-03-29",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-03-29",
      "catalog_releases": [
        {
          "name": "Qwen3.5-Omni-Plus",
          "date": "2026-03-29",
          "link": "https://arxiv.org/abs/2604.15804"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Multimodal,Vision,Audio",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2604.15804",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-Omni-Flash",
      "name": "Qwen3.5-Omni-Flash",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-03-29",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-03-29",
      "catalog_releases": [
        {
          "name": "Qwen3.5-Omni-Flash",
          "date": "2026-03-29",
          "link": "https://arxiv.org/abs/2604.15804"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Multimodal,Vision,Audio",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2604.15804",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.4 Nano",
      "name": "GPT-5.4 Nano",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-03-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 146.33,
      "eci_low": 144.67,
      "eci_high": 148.02,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.4-nano-2026-03-17_high",
        "gpt-5.4-nano-2026-03-17_low",
        "gpt-5.4-nano-2026-03-17_medium",
        "gpt-5.4-nano-2026-03-17_none",
        "gpt-5.4-nano-2026-03-17_unknown",
        "gpt-5.4-nano-2026-03-17_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 12.195121951219512,
          "variant": "gpt-5.4-nano-2026-03-17_high",
          "configurations": [
            {
              "variant": "gpt-5.4-nano-2026-03-17_high",
              "score": 12.195121951219512
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 11.700000000000001,
          "variant": "gpt-5.4-nano-2026-03-17_high",
          "configurations": [
            {
              "variant": "gpt-5.4-nano-2026-03-17_high",
              "score": 11.700000000000001
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 9.0,
          "variant": "gpt-5.4-nano-2026-03-17_none",
          "configurations": [
            {
              "variant": "gpt-5.4-nano-2026-03-17_high",
              "score": 5.0
            },
            {
              "variant": "gpt-5.4-nano-2026-03-17_medium",
              "score": 6.0
            },
            {
              "variant": "gpt-5.4-nano-2026-03-17_low",
              "score": 3.0
            },
            {
              "variant": "gpt-5.4-nano-2026-03-17_none",
              "score": 9.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.4 Mini",
      "name": "GPT-5.4 Mini",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-03-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 149.35,
      "eci_low": 147.71,
      "eci_high": 151.06,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.4-mini-2026-03-17_high",
        "gpt-5.4-mini-2026-03-17_low",
        "gpt-5.4-mini-2026-03-17_medium",
        "gpt-5.4-mini-2026-03-17_none",
        "gpt-5.4-mini-2026-03-17_unknown",
        "gpt-5.4-mini-2026-03-17_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 9.75609756097561,
          "variant": "gpt-5.4-mini-2026-03-17_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.4-mini-2026-03-17_xhigh",
              "score": 9.75609756097561
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 29.4,
          "variant": "gpt-5.4-mini-2026-03-17_high",
          "configurations": [
            {
              "variant": "gpt-5.4-mini-2026-03-17_high",
              "score": 29.4
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 11.0,
          "variant": "gpt-5.4-mini-2026-03-17_none",
          "configurations": [
            {
              "variant": "gpt-5.4-mini-2026-03-17_medium",
              "score": 7.000000000000001
            },
            {
              "variant": "gpt-5.4-mini-2026-03-17_low",
              "score": 8.0
            },
            {
              "variant": "gpt-5.4-mini-2026-03-17_none",
              "score": 11.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Small 4",
      "name": "Mistral Small 4",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2026-03-16",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-small-2603"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.4 Pro",
      "name": "GPT-5.4 Pro",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-03-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-03-05",
      "catalog_releases": [
        {
          "name": "GPT-5.4 Pro",
          "date": "2026-03-05",
          "link": "https://openai.com/index/introducing-gpt-5-4/"
        }
      ],
      "eci": 160.32,
      "eci_low": 157.87,
      "eci_high": 163.03,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.4-pro-2026-03-05-web-app",
        "gpt-5.4-pro-2026-03-05_none",
        "gpt-5.4-pro-2026-03-05_unknown",
        "gpt-5.4-pro-2026-03-05_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 58.536585365853654,
          "variant": "gpt-5.4-pro-2026-03-05_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.4-pro-2026-03-05_xhigh",
              "score": 58.536585365853654
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 46.300000000000004,
          "variant": "gpt-5.4-pro-2026-03-05_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.4-pro-2026-03-05_xhigh",
              "score": 46.300000000000004
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-gpt-5-4/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.4",
      "name": "GPT-5.4",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-03-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-03-05",
      "catalog_releases": [
        {
          "name": "GPT-5.4",
          "date": "2026-03-05",
          "link": "https://openai.com/index/introducing-gpt-5-4/"
        }
      ],
      "eci": 157.87,
      "eci_low": 155.96,
      "eci_high": 159.97,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.4-2026-03-05_high",
        "gpt-5.4-2026-03-05_low",
        "gpt-5.4-2026-03-05_medium",
        "gpt-5.4-2026-03-05_none",
        "gpt-5.4-2026-03-05_unknown",
        "gpt-5.4-2026-03-05_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 49.0,
          "variant": "gpt-5.4-2026-03-05_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.4-2026-03-05_xhigh",
              "score": 49.0
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 23.333333333333332,
          "variant": "gpt-5.4-2026-03-05_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.4-2026-03-05_xhigh",
              "score": 23.333333333333332
            },
            {
              "variant": "gpt-5.4-2026-03-05_high",
              "score": 15.555555555555555
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 45.1,
          "variant": "gpt-5.4-2026-03-05_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.4-2026-03-05_xhigh",
              "score": 45.1
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 37.0,
          "variant": "gpt-5.4-2026-03-05_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.4-2026-03-05_none",
              "score": 16.0
            },
            {
              "variant": "gpt-5.4-2026-03-05_medium",
              "score": 28.000000000000004
            },
            {
              "variant": "gpt-5.4-2026-03-05_low",
              "score": 17.0
            },
            {
              "variant": "gpt-5.4-2026-03-05_xhigh",
              "score": 37.0
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-gpt-5-4/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3.1 Flash-Lite",
      "name": "Gemini 3.1 Flash-Lite",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2026-03-03",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 145.03,
      "eci_low": 142.88,
      "eci_high": 147.25,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3.1-flash-lite",
        "gemini-3.1-flash-lite_high",
        "gemini-3.1-flash-lite_low",
        "gemini-3.1-flash-lite_minimal"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3.0 Flash-lite",
      "name": "Gemini 3.0 Flash-lite",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-03-03",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-03-03",
      "catalog_releases": [
        {
          "name": "Gemini 3.0 Flash-lite",
          "date": "2026-03-03",
          "link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-lite/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-flash-lite/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen 3.5 Flash (hosted 35B-A3B)",
      "name": "Qwen 3.5 Flash (hosted 35B-A3B)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 143.87,
      "eci_low": 142.01,
      "eci_high": 145.75,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.5-flash",
        "qwen3.5-flash_none"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 20.32032032032032,
          "variant": "qwen3.5-flash",
          "configurations": [
            {
              "variant": "qwen3.5-flash",
              "score": 20.32032032032032
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 20.0,
          "variant": "qwen3.5-flash",
          "configurations": [
            {
              "variant": "qwen3.5-flash",
              "score": 20.0
            },
            {
              "variant": "qwen3.5-flash_none",
              "score": 16.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-9B",
      "name": "Qwen3.5-9B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 139.48,
      "eci_low": 137.26,
      "eci_high": 141.69,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.5-9B",
        "qwen3.5-9B_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-4B",
      "name": "Qwen3.5-4B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.5-4b_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-35B-A3B",
      "name": "Qwen3.5-35B-A3B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-02-24",
      "catalog_releases": [
        {
          "name": "Qwen3.5-35B-A3B",
          "date": "2026-02-24",
          "link": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B"
        }
      ],
      "eci": 144.08,
      "eci_low": 141.59,
      "eci_high": 146.7,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 35000000000.0,
      "compute_flop": null,
      "variants": [
        "qwen3.5-35b-a3b",
        "qwen3.5-35b-a3b_none"
      ],
      "benchmarks": {},
      "catalog_link": "https://huggingface.co/Qwen/Qwen3.5-35B-A3B",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-2B",
      "name": "Qwen3.5-2B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.5-2b_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-27B",
      "name": "Qwen3.5-27B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.5-27B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-122B-A10B",
      "name": "Qwen3.5-122B-A10B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-02-24",
      "catalog_releases": [
        {
          "name": "Qwen3.5-122B-A10B",
          "date": "2026-02-24",
          "link": "https://github.com/QwenLM/Qwen3.5?tab=readme-ov-file"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": 122000000000.0,
      "compute_flop": null,
      "variants": [
        "qwen3.5-122b-a10b_none"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 17.0,
          "variant": "qwen3.5-122b-a10b_none",
          "configurations": [
            {
              "variant": "qwen3.5-122b-a10b_none",
              "score": 17.0
            }
          ]
        }
      },
      "catalog_link": "https://github.com/QwenLM/Qwen3.5?tab=readme-ov-file",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5-0.8B",
      "name": "Qwen3.5-0.8B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.5-0.8b",
        "qwen3.5-0.8b_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3.1 Pro",
      "name": "Gemini 3.1 Pro",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-02-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 156.25,
      "eci_low": 154.49,
      "eci_high": 158.15,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3.1-pro-preview",
        "gemini-3.1-pro-preview-customtools",
        "gemini-3.1-pro-preview_high",
        "gemini-3.1-pro-preview_low",
        "gemini-3.1-pro-preview_medium"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 26.82926829268293,
          "variant": "gemini-3.1-pro-preview",
          "configurations": [
            {
              "variant": "gemini-3.1-pro-preview",
              "score": 26.82926829268293
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 8.888888888888888,
          "variant": "gemini-3.1-pro-preview_high",
          "configurations": [
            {
              "variant": "gemini-3.1-pro-preview_high",
              "score": 8.888888888888888
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 73.5,
          "variant": "gemini-3.1-pro-preview_high",
          "configurations": [
            {
              "variant": "gemini-3.1-pro-preview_high",
              "score": 73.5
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 34.0,
          "variant": "gemini-3.1-pro-preview_high",
          "configurations": [
            {
              "variant": "gemini-3.1-pro-preview_medium",
              "score": 32.0
            },
            {
              "variant": "gemini-3.1-pro-preview_low",
              "score": 28.999999999999996
            },
            {
              "variant": "gemini-3.1-pro-preview_high",
              "score": 34.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4.20",
      "name": "Grok 4.20",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2026-02-17",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-02-17",
      "catalog_releases": [
        {
          "name": "Grok 4.20",
          "date": "2026-02-17",
          "link": "https://docs.x.ai/developers/models"
        }
      ],
      "eci": 152.3,
      "eci_low": 150.58,
      "eci_high": 154.09,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": 500000000000.0,
      "compute_flop": null,
      "variants": [
        "grok-4-20",
        "grok-4.20-0309-reasoning"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 17.073170731707318,
          "variant": "grok-4.20-0309-reasoning",
          "configurations": [
            {
              "variant": "grok-4.20-0309-reasoning",
              "score": 17.073170731707318
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 30.2,
          "variant": "grok-4.20-0309-reasoning",
          "configurations": [
            {
              "variant": "grok-4.20-0309-reasoning",
              "score": 30.2
            }
          ]
        }
      },
      "catalog_link": "https://docs.x.ai/developers/models",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Sonnet 4.6",
      "name": "Claude Sonnet 4.6",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-02-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 152.86,
      "eci_low": 151.17,
      "eci_high": 154.66,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-sonnet-4-6",
        "claude-sonnet-4-6_16K",
        "claude-sonnet-4-6_32K",
        "claude-sonnet-4-6_60K",
        "claude-sonnet-4-6_high",
        "claude-sonnet-4-6_low",
        "claude-sonnet-4-6_max",
        "claude-sonnet-4-6_medium",
        "claude-sonnet-4-6_unknown"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 35.5,
          "variant": "claude-sonnet-4-6_high",
          "configurations": [
            {
              "variant": "claude-sonnet-4-6_max",
              "score": 32.800000000000004
            },
            {
              "variant": "claude-sonnet-4-6_high",
              "score": 35.5
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 16.0,
          "variant": "claude-sonnet-4-6_low",
          "configurations": [
            {
              "variant": "claude-sonnet-4-6",
              "score": 14.000000000000002
            },
            {
              "variant": "claude-sonnet-4-6_low",
              "score": 16.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen 3.5 Plus (hosted 397B-A17B)",
      "name": "Qwen 3.5 Plus (hosted 397B-A17B)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-16",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 147.3,
      "eci_low": 145.38,
      "eci_high": 149.2,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3.5-plus",
        "qwen3.5-plus_none"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 25.35070140280561,
          "variant": "qwen3.5-plus",
          "configurations": [
            {
              "variant": "qwen3.5-plus",
              "score": 25.35070140280561
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 17.0,
          "variant": "qwen3.5-plus_none",
          "configurations": [
            {
              "variant": "qwen3.5-plus_none",
              "score": 17.0
            },
            {
              "variant": "qwen3.5-plus",
              "score": 16.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3.5 397B-A17B",
      "name": "Qwen3.5 397B-A17B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-13",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-02-15",
      "catalog_releases": [
        {
          "name": "Qwen3.5 397B-A17B",
          "date": "2026-02-15",
          "link": "https://qwen.ai/blog?id=qwen3.5"
        }
      ],
      "eci": 147.95,
      "eci_low": 145.8,
      "eci_high": 150.07,
      "access": "Open weights",
      "domain": "Language,Vision",
      "parameters": 397000000000.0,
      "compute_flop": null,
      "variants": [
        "qwen3.5-397b-a17b",
        "qwen3.5-397b-a17b_none"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 18.0,
          "variant": "qwen3.5-397b-a17b_none",
          "configurations": [
            {
              "variant": "qwen3.5-397b-a17b_none",
              "score": 18.0
            }
          ]
        }
      },
      "catalog_link": "https://qwen.ai/blog?id=qwen3.5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3 Deep Think",
      "name": "Gemini 3 Deep Think",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2026-02-12",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-11-18",
      "catalog_releases": [
        {
          "name": "Gemini 3 Deep Think",
          "date": "2025-11-18",
          "link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-deep-think/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3-deep-think-preview"
      ],
      "benchmarks": {},
      "catalog_link": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-deep-think/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.3-Codex-Spark",
      "name": "GPT-5.3-Codex-Spark",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-02-12",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-02-12",
      "catalog_releases": [
        {
          "name": "GPT-5.3-Codex-Spark",
          "date": "2026-02-12",
          "link": "https://openai.com/index/introducing-gpt-5-3-codex-spark/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-gpt-5-3-codex-spark/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.3 Codex",
      "name": "GPT-5.3 Codex",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2026-02-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-02-05",
      "catalog_releases": [
        {
          "name": "GPT-5.3 Codex",
          "date": "2026-02-05",
          "link": "https://openai.com/index/introducing-gpt-5-3-codex/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.3-codex",
        "gpt-5.3-codex_high",
        "gpt-5.3-codex_xhigh"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-gpt-5-3-codex/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Opus 4.6",
      "name": "Claude Opus 4.6",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2026-02-05",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 156.35,
      "eci_low": 154.59,
      "eci_high": 158.29,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-opus-4-6",
        "claude-opus-4-6_120K",
        "claude-opus-4-6_32K",
        "claude-opus-4-6_64K",
        "claude-opus-4-6_high",
        "claude-opus-4-6_low",
        "claude-opus-4-6_max",
        "claude-opus-4-6_medium",
        "claude-opus-4-6_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 26.82926829268293,
          "variant": "claude-opus-4-6_max",
          "configurations": [
            {
              "variant": "claude-opus-4-6_max",
              "score": 26.82926829268293
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 33.33333333333333,
          "variant": "claude-opus-4-6_max",
          "configurations": [
            {
              "variant": "claude-opus-4-6_max",
              "score": 33.33333333333333
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 47.0,
          "variant": "claude-opus-4-6_max",
          "configurations": [
            {
              "variant": "claude-opus-4-6_max",
              "score": 47.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 25.0,
          "variant": "claude-opus-4-6_max",
          "configurations": [
            {
              "variant": "claude-opus-4-6",
              "score": 15.0
            },
            {
              "variant": "claude-opus-4-6_low",
              "score": 7.000000000000001
            },
            {
              "variant": "claude-opus-4-6_max",
              "score": 25.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "qwen3-coder-next",
      "name": "qwen3-coder-next",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-02-02",
      "date_basis": "Benchmark release date",
      "catalog_date": "2026-02-02",
      "catalog_releases": [
        {
          "name": "Qwen3-Coder-Next",
          "date": "2026-02-02",
          "link": "https://qwen.ai/blog?id=qwen3-coder-next"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 80000000000.0,
      "compute_flop": null,
      "variants": [
        "qwen3-coder-next"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwen.ai/blog?id=qwen3-coder-next",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-Max-Thinking",
      "name": "Qwen3-Max-Thinking",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2026-01-25",
      "date_basis": "Catalog publication date",
      "catalog_date": "2026-01-25",
      "catalog_releases": [
        {
          "name": "Qwen3-Max-Thinking",
          "date": "2026-01-25",
          "link": "https://qwen.ai/blog?id=qwen3-max-thinking"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://qwen.ai/blog?id=qwen3-max-thinking",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.2 Codex",
      "name": "GPT-5.2 Codex",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-12-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-12-18",
      "catalog_releases": [
        {
          "name": "GPT-5.2 Codex",
          "date": "2025-12-18",
          "link": "https://openai.com/index/introducing-gpt-5-2-codex/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.2-codex"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-gpt-5-2-codex/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3 Flash",
      "name": "Gemini 3 Flash",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-12-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 152.34,
      "eci_low": 150.79,
      "eci_high": 153.98,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3-flash-preview",
        "gemini-3-flash-preview_high",
        "gemini-3-flash-preview_low",
        "gemini-3-flash-preview_minimal"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 17.073170731707318,
          "variant": "gemini-3-flash-preview",
          "configurations": [
            {
              "variant": "gemini-3-flash-preview",
              "score": 17.073170731707318
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 66.8,
          "variant": "gemini-3-flash-preview_high",
          "configurations": [
            {
              "variant": "gemini-3-flash-preview_high",
              "score": 66.8
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 26.0,
          "variant": "gemini-3-flash-preview_low",
          "configurations": [
            {
              "variant": "gemini-3-flash-preview_minimal",
              "score": 25.0
            },
            {
              "variant": "gemini-3-flash-preview_high",
              "score": 20.0
            },
            {
              "variant": "gemini-3-flash-preview_low",
              "score": 26.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT Image 1.5",
      "name": "GPT Image 1.5",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-12-16",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-12-16",
      "catalog_releases": [
        {
          "name": "GPT Image 1.5",
          "date": "2025-12-16",
          "link": "https://openai.com/index/new-chatgpt-images-is-here/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation,Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/new-chatgpt-images-is-here/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.2 Pro",
      "name": "GPT-5.2 Pro",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-12-11",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-12-11",
      "catalog_releases": [
        {
          "name": "GPT-5.2 Pro",
          "date": "2025-12-11",
          "link": "https://openai.com/index/introducing-gpt-5-2/"
        }
      ],
      "eci": 157.31,
      "eci_low": 155.11,
      "eci_high": 159.77,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.2-pro-2025-12-11",
        "gpt-5.2-pro-2025-12-11-webapp",
        "gpt-5.2-pro-2025-12-11_high",
        "gpt-5.2-pro-2025-12-11_medium",
        "gpt-5.2-pro-2025-12-11_xhigh",
        "robin-alpha-pro-2025-11-24",
        "robin-alpha-pro-2025-11-24_high",
        "robin-alpha-pro-2025-11-24_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 46.0,
          "variant": "gpt-5.2-pro-2025-12-11_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.2-pro-2025-12-11_xhigh",
              "score": 46.0
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-gpt-5-2/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.2",
      "name": "GPT-5.2",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-12-11",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-12-11",
      "catalog_releases": [
        {
          "name": "GPT-5.2",
          "date": "2025-12-11",
          "link": ""
        }
      ],
      "eci": 154.52,
      "eci_low": 152.94,
      "eci_high": 156.31,
      "access": "Closed weights",
      "domain": "",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.2-2025-12-11_high",
        "gpt-5.2-2025-12-11_low",
        "gpt-5.2-2025-12-11_medium",
        "gpt-5.2-2025-12-11_none",
        "gpt-5.2-2025-12-11_unknown",
        "gpt-5.2-2025-12-11_xhigh",
        "robin-alpha-next-2025-11-24_high",
        "robin-alpha-next-2025-11-24_xhigh"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 31.7,
          "variant": "gpt-5.2-2025-12-11_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.2-2025-12-11_xhigh",
              "score": 31.7
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 32.22222222222222,
          "variant": "gpt-5.2-2025-12-11_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.2-2025-12-11_xhigh",
              "score": 32.22222222222222
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 37.1,
          "variant": "gpt-5.2-2025-12-11_xhigh",
          "configurations": [
            {
              "variant": "gpt-5.2-2025-12-11_low",
              "score": 32.800000000000004
            },
            {
              "variant": "gpt-5.2-2025-12-11_medium",
              "score": 32.7
            },
            {
              "variant": "gpt-5.2-2025-12-11_high",
              "score": 34.300000000000004
            },
            {
              "variant": "gpt-5.2-2025-12-11_xhigh",
              "score": 37.1
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 23.0,
          "variant": "gpt-5.2-2025-12-11_high",
          "configurations": [
            {
              "variant": "gpt-5.2-2025-12-11_medium",
              "score": 22.0
            },
            {
              "variant": "gpt-5.2-2025-12-11_low",
              "score": 10.0
            },
            {
              "variant": "gpt-5.2-2025-12-11_none",
              "score": 14.000000000000002
            },
            {
              "variant": "gpt-5.2-2025-12-11_high",
              "score": 23.0
            }
          ]
        }
      },
      "catalog_link": "",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Devstral Small 2",
      "name": "Devstral Small 2",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-12-09",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "devstral-small-2512"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "SIMA 2",
      "name": "SIMA 2",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-12-04",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-12-04",
      "catalog_releases": [
        {
          "name": "SIMA 2",
          "date": "2025-12-04",
          "link": "https://arxiv.org/abs/2512.04797v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "3D modeling",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2512.04797v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Large 3",
      "name": "Mistral Large 3",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-12-02",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-large-2512"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V3.2-Speciale",
      "name": "DeepSeek-V3.2-Speciale",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-12-01",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 4.2e+24,
      "variants": [
        "DeepSeek-V3.2-Speciale"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V3.2",
      "name": "DeepSeek-V3.2",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-12-01",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 146.26,
      "eci_low": 144.75,
      "eci_high": 147.75,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 4.2e+24,
      "variants": [
        "deepseek-chat",
        "deepseek-reasoner",
        "deepseek/deepseek-v3.2",
        "fireworks/deepseek-v3p2"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeekMath-V2",
      "name": "DeepSeekMath-V2",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-11-27",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-11-27",
      "catalog_releases": [
        {
          "name": "DeepSeekMath-V2",
          "date": "2025-11-27",
          "link": "https://arxiv.org/abs/2511.22570"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 685000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2511.22570",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Opus 4.5",
      "name": "Claude Opus 4.5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-11-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-11-24",
      "catalog_releases": [
        {
          "name": "Claude Opus 4.5",
          "date": "2025-11-24",
          "link": "https://www.anthropic.com/news/claude-opus-4-5"
        }
      ],
      "eci": 151.88,
      "eci_low": 150.32,
      "eci_high": 153.53,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-opus-4-5-20251101",
        "claude-opus-4-5-20251101_128K",
        "claude-opus-4-5-20251101_16K",
        "claude-opus-4-5-20251101_32K",
        "claude-opus-4-5-20251101_48K",
        "claude-opus-4-5-20251101_59K",
        "claude-opus-4-5-20251101_64K",
        "claude-opus-4-5-20251101_8K",
        "claude-opus-4-5-20251101_high",
        "claude-opus-4-5-20251101_low",
        "claude-opus-4-5-20251101_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 4.878048780487805,
          "variant": "claude-opus-4-5-20251101_32K",
          "configurations": [
            {
              "variant": "claude-opus-4-5-20251101_32K",
              "score": 4.878048780487805
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 45.7,
          "variant": "claude-opus-4-5-20251101_32K",
          "configurations": [
            {
              "variant": "claude-opus-4-5-20251101_32K",
              "score": 45.7
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 22.0,
          "variant": "claude-opus-4-5-20251101_48K",
          "configurations": [
            {
              "variant": "claude-opus-4-5-20251101_48K",
              "score": 22.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/news/claude-opus-4-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3 Pro Image (Nano Banana Pro)",
      "name": "Gemini 3 Pro Image (Nano Banana Pro)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-11-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-11-20",
      "catalog_releases": [
        {
          "name": "Gemini 3 Pro Image (Nano Banana Pro)",
          "date": "2025-11-20",
          "link": "https://blog.google/innovation-and-ai/products/nano-banana-pro/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://blog.google/innovation-and-ai/products/nano-banana-pro/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4.1 Fast",
      "name": "Grok 4.1 Fast",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2025-11-19",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-11-19",
      "catalog_releases": [
        {
          "name": "Grok 4.1 Fast",
          "date": "2025-11-19",
          "link": "https://x.ai/news/grok-4-1-fast"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-4-1-fast-non-reasoning",
        "grok-4-1-fast-reasoning"
      ],
      "benchmarks": {},
      "catalog_link": "https://x.ai/news/grok-4-1-fast",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.1-Codex-Max",
      "name": "GPT-5.1-Codex-Max",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-11-19",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-11-19",
      "catalog_releases": [
        {
          "name": "GPT-5.1-Codex-Max",
          "date": "2025-11-19",
          "link": "https://openai.com/index/gpt-5-1-codex-max/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.1-codex-max",
        "gpt-5.1-codex-max_xhigh"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/gpt-5-1-codex-max/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 3 Pro",
      "name": "Gemini 3 Pro",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-11-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-11-18",
      "catalog_releases": [
        {
          "name": "Gemini 3 Pro",
          "date": "2025-11-18",
          "link": "https://blog.google/products/gemini/gemini-3/"
        }
      ],
      "eci": 153.04,
      "eci_low": 151.45,
      "eci_high": 154.78,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-3-pro-preview",
        "gemini-3-pro-preview_low"
      ],
      "benchmarks": {},
      "catalog_link": "https://blog.google/products/gemini/gemini-3/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4.1",
      "name": "Grok 4.1",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2025-11-17",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-11-17",
      "catalog_releases": [
        {
          "name": "Grok 4.1",
          "date": "2025-11-17",
          "link": "https://x.ai/news/grok-4-1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-4-1"
      ],
      "benchmarks": {},
      "catalog_link": "https://x.ai/news/grok-4-1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.1 Instant",
      "name": "GPT-5.1 Instant",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-11-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-11-13",
      "catalog_releases": [
        {
          "name": "GPT-5.1 Instant",
          "date": "2025-11-13",
          "link": "https://openai.com/index/gpt-5-1/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/gpt-5-1/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.1",
      "name": "GPT-5.1",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-11-13",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-11-13",
      "catalog_releases": [
        {
          "name": "GPT-5.1",
          "date": "2025-11-13",
          "link": "https://openai.com/index/gpt-5-1/"
        }
      ],
      "eci": 150.97,
      "eci_low": 149.42,
      "eci_high": 152.61,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.1-2025-11-13_high",
        "gpt-5.1-2025-11-13_low",
        "gpt-5.1-2025-11-13_medium",
        "gpt-5.1-2025-11-13_none",
        "gpt-5.1-2025-11-13_unknown"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 48.0,
          "variant": "gpt-5.1-2025-11-13_high",
          "configurations": [
            {
              "variant": "gpt-5.1-2025-11-13_high",
              "score": 48.0
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 19.0,
          "variant": "gpt-5.1-2025-11-13_low",
          "configurations": [
            {
              "variant": "gpt-5.1-2025-11-13_medium",
              "score": 16.0
            },
            {
              "variant": "gpt-5.1-2025-11-13_low",
              "score": 19.0
            },
            {
              "variant": "gpt-5.1-2025-11-13_none",
              "score": 15.0
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/gpt-5-1/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.1-Codex-mini",
      "name": "GPT-5.1-Codex-mini",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-11-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.1-codex-mini"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5.1-Codex",
      "name": "GPT-5.1-Codex",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-11-12",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-11-12",
      "catalog_releases": [
        {
          "name": "GPT-5.1-Codex",
          "date": "2025-11-12",
          "link": ""
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5.1-codex"
      ],
      "benchmarks": {},
      "catalog_link": "",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Meta's Generative Ads Model (GEM)",
      "name": "Meta's Generative Ads Model (GEM)",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2025-11-10",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-11-10",
      "catalog_releases": [
        {
          "name": "Meta's Generative Ads Model (GEM)",
          "date": "2025-11-10",
          "link": "https://engineering.fb.com/2025/11/10/ml-applications/metas-generative-ads-model-gem-the-central-brain-accelerating-ads-recommendation-ai-innovation/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Recommendation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://engineering.fb.com/2025/11/10/ml-applications/metas-generative-ads-model-gem-the-central-brain-accelerating-ads-recommendation-ai-innovation/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Tongyi DeepResearch",
      "name": "Tongyi DeepResearch",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-10-28",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-10-28",
      "catalog_releases": [
        {
          "name": "Tongyi DeepResearch",
          "date": "2025-10-28",
          "link": "https://arxiv.org/abs/2510.24701"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 30500000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2510.24701",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Veo 3.1",
      "name": "Veo 3.1",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-10-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-10-15",
      "catalog_releases": [
        {
          "name": "Veo 3.1",
          "date": "2025-10-15",
          "link": "https://blog.google/technology/ai/veo-updates-flow/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Video,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://blog.google/technology/ai/veo-updates-flow/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Haiku 4.5",
      "name": "Claude Haiku 4.5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-10-15",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-10-15",
      "catalog_releases": [
        {
          "name": "Claude Haiku 4.5",
          "date": "2025-10-15",
          "link": "https://www.anthropic.com/news/claude-haiku-4-5"
        }
      ],
      "eci": 142.76,
      "eci_low": 141.15,
      "eci_high": 144.34,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-haiku-4-5-20251001",
        "claude-haiku-4-5-20251001_16K",
        "claude-haiku-4-5-20251001_1K",
        "claude-haiku-4-5-20251001_32K",
        "claude-haiku-4-5-20251001_59K",
        "claude-haiku-4-5-20251001_8K",
        "claude-haiku-4-5-20251001_low",
        "claude-haiku-4-5-20251001_unknown"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 13.200000000000001,
          "variant": "claude-haiku-4-5-20251001",
          "configurations": [
            {
              "variant": "claude-haiku-4-5-20251001_32K",
              "score": 12.6
            },
            {
              "variant": "claude-haiku-4-5-20251001",
              "score": 13.200000000000001
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/news/claude-haiku-4-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5 Pro",
      "name": "GPT-5 Pro",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-10-07",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-10-07",
      "catalog_releases": [
        {
          "name": "GPT-5 Pro",
          "date": "2025-10-07",
          "link": "https://cdn.openai.com/gpt-5-system-card.pdf"
        }
      ],
      "eci": 151.27,
      "eci_low": 149.32,
      "eci_high": 153.32,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5-pro-2025-10-06_high",
        "gpt-5-pro-2025-10-06_medium",
        "gpt-5-pro-2025-10-06_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 19.51219512195122,
          "variant": "gpt-5-pro-2025-10-06_high",
          "configurations": [
            {
              "variant": "gpt-5-pro-2025-10-06_high",
              "score": 19.51219512195122
            }
          ]
        }
      },
      "catalog_link": "https://cdn.openai.com/gpt-5-system-card.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Sora 2.0",
      "name": "Sora 2.0",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-09-30",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-09-30",
      "catalog_releases": [
        {
          "name": "Sora 2.0",
          "date": "2025-09-30",
          "link": "https://openai.com/index/sora-2/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Video",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/sora-2/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V3.2-Exp",
      "name": "DeepSeek-V3.2-Exp",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-09-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 144.39,
      "eci_low": 142.07,
      "eci_high": 146.73,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 4.18e+24,
      "variants": [
        "DeepSeek-V3.2-Exp",
        "DeepSeek-V3.2-Exp_high",
        "DeepSeek-V3.2-Exp_thinking",
        "DeepSeek-V3.2-Exp_unknown"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Sonnet 4.5",
      "name": "Claude Sonnet 4.5",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-09-29",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-09-29",
      "catalog_releases": [
        {
          "name": "Claude Sonnet 4.5",
          "date": "2025-09-29",
          "link": "https://www.anthropic.com/news/claude-sonnet-4-5"
        }
      ],
      "eci": 147.59,
      "eci_low": 146.14,
      "eci_high": 149.04,
      "access": "Closed weights",
      "domain": "Language,Vision,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-sonnet-4-5-20250929",
        "claude-sonnet-4-5-20250929_12K",
        "claude-sonnet-4-5-20250929_16K",
        "claude-sonnet-4-5-20250929_1K",
        "claude-sonnet-4-5-20250929_27K",
        "claude-sonnet-4-5-20250929_2K",
        "claude-sonnet-4-5-20250929_32K",
        "claude-sonnet-4-5-20250929_48K",
        "claude-sonnet-4-5-20250929_59K",
        "claude-sonnet-4-5-20250929_8K",
        "claude-sonnet-4-5-20250929_low",
        "claude-sonnet-4-5-20250929_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 2.4390243902439024,
          "variant": "claude-sonnet-4-5-20250929_32K",
          "configurations": [
            {
              "variant": "claude-sonnet-4-5-20250929_32K",
              "score": 2.4390243902439024
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 30.7,
          "variant": "claude-sonnet-4-5-20250929_59K",
          "configurations": [
            {
              "variant": "claude-sonnet-4-5-20250929_59K",
              "score": 30.7
            },
            {
              "variant": "claude-sonnet-4-5-20250929",
              "score": 23.7
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 17.0,
          "variant": "claude-sonnet-4-5-20250929_48K",
          "configurations": [
            {
              "variant": "claude-sonnet-4-5-20250929_48K",
              "score": 17.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/news/claude-sonnet-4-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "gemini-robotics-er-1.5-preview",
      "name": "gemini-robotics-er-1.5-preview",
      "lab": "Google DeepMind",
      "organization": null,
      "lab_inferred": true,
      "date": "2025-09-26",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-robotics-er-1.5-preview"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini Robotics-ER 1.5",
      "name": "Gemini Robotics-ER 1.5",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-09-25",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-09-25",
      "catalog_releases": [
        {
          "name": "Gemini Robotics-ER 1.5",
          "date": "2025-09-25",
          "link": "https://deepmind.google/discover/blog/gemini-robotics-15-brings-ai-agents-into-the-physical-world/\n\nhttps://storage.googleapis.com/deepmind-media/gemini-robotics/Gemini-Robotics-1-5-Tech-Report.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Vision,Language,Speech",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://deepmind.google/discover/blog/gemini-robotics-15-brings-ai-agents-into-the-physical-world/\n\nhttps://storage.googleapis.com/deepmind-media/gemini-robotics/Gemini-Robotics-1-5-Tech-Report.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Flash-Lite (Sep 2025)",
      "name": "Gemini 2.5 Flash-Lite (Sep 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-09-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-flash-lite-preview-09-2025"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Flash (Sep 2025)",
      "name": "Gemini 2.5 Flash (Sep 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-09-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 141.94,
      "eci_low": 139.1,
      "eci_high": 144.59,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-flash-preview-09-2025",
        "gemini-2.5-flash-preview-09-2025_16k"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-Max",
      "name": "Qwen3-Max",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-09-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-09-05",
      "catalog_releases": [
        {
          "name": "Qwen3-Max",
          "date": "2025-09-05",
          "link": "https://modelstudio.console.alibabacloud.com/?tab=doc#/doc/?type=model&url=2840914_2&modelId=qwen3-max-preview\n\nhttps://qwen.ai/blog?id=87dc93fc8a590dc718c77e1f6e84c07b474f6c5a&from=home.latest-research-list"
        }
      ],
      "eci": 142.44,
      "eci_low": 140.29,
      "eci_high": 144.54,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": 1000000000000.0,
      "compute_flop": 1.512e+25,
      "variants": [
        "qwen3-max-2025-09-23"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 48.74874874874875,
          "variant": "qwen3-max-2025-09-23",
          "configurations": [
            {
              "variant": "qwen3-max-2025-09-23",
              "score": 48.74874874874875
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 5.0,
          "variant": "qwen3-max-2025-09-23",
          "configurations": [
            {
              "variant": "qwen3-max-2025-09-23",
              "score": 5.0
            }
          ]
        }
      },
      "catalog_link": "https://modelstudio.console.alibabacloud.com/?tab=doc#/doc/?type=model&url=2840914_2&modelId=qwen3-max-preview\n\nhttps://qwen.ai/blog?id=87dc93fc8a590dc718c77e1f6e84c07b474f6c5a&from=home.latest-research-list",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "qwen3-coder-plus",
      "name": "qwen3-coder-plus",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2025-09-23",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3-coder-plus"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-VL-235B-A22B",
      "name": "Qwen3-VL-235B-A22B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-09-23",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-09-23",
      "catalog_releases": [
        {
          "name": "Qwen3-VL-235B-A22B",
          "date": "2025-09-23",
          "link": "https://arxiv.org/abs/2511.21631"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language,Vision,Multimodal,Video",
      "parameters": 235000000000.0,
      "compute_flop": 5.038044e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2511.21631",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-Omni-30B-A3B",
      "name": "Qwen3-Omni-30B-A3B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-09-22",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-09-22",
      "catalog_releases": [
        {
          "name": "Qwen3-Omni-30B-A3B",
          "date": "2025-09-22",
          "link": "https://arxiv.org/abs/2509.17765"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Multimodal,Language,Vision,Speech,Video",
      "parameters": 35300000000.0,
      "compute_flop": 3.6e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2509.17765",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V3.1-Terminus",
      "name": "DeepSeek-V3.1-Terminus",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-09-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.594058e+24,
      "variants": [
        "DeepSeek-V3.1-Terminus"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4 Fast",
      "name": "Grok 4 Fast",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2025-09-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 144.35,
      "eci_low": 142.19,
      "eci_high": 146.46,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-4-fast"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Magistral Small 1.2",
      "name": "Magistral Small 1.2",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "magistral-small-2509"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Magistral Medium 1.2",
      "name": "Magistral Medium 1.2",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "magistral-medium-2509"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AgentFounder-30B",
      "name": "AgentFounder-30B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-09-16",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-09-16",
      "catalog_releases": [
        {
          "name": "AgentFounder-30B",
          "date": "2025-09-16",
          "link": "https://arxiv.org/abs/2509.13310"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 30000000000.0,
      "compute_flop": 6.5367e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2509.13310",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5-Codex",
      "name": "GPT-5-Codex",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-09-15",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5-codex",
        "gpt-5-codex_high"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "grok-code-fast-1",
      "name": "grok-code-fast-1",
      "lab": "xAI",
      "organization": null,
      "lab_inferred": true,
      "date": "2025-08-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-code-fast-1"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Flash Image (Nano Banana)",
      "name": "Gemini 2.5 Flash Image (Nano Banana)",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2025-08-26",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-08-26",
      "catalog_releases": [
        {
          "name": "Gemini 2.5 Flash Image (Nano Banana)",
          "date": "2025-08-26",
          "link": "https://gemini.google/overview/image-generation/\n\nhttps://aistudio.google.com/models/gemini-2-5-flash-image"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://gemini.google/overview/image-generation/\n\nhttps://aistudio.google.com/models/gemini-2-5-flash-image",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V3.1",
      "name": "DeepSeek-V3.1",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-08-21",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 140.8,
      "eci_low": 137.55,
      "eci_high": 143.94,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.594058e+24,
      "variants": [
        "DeepSeek-V3.1",
        "DeepSeek-V3.1_thinking"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 3 270M",
      "name": "Gemma 3 270M",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-08-14",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 9.72e+21,
      "variants": [
        "gemma-3-270m-it"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Medium 3.1",
      "name": "Mistral Medium 3.1",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-08-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-medium-2508"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5 nano",
      "name": "GPT-5 nano",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-08-07",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-08-07",
      "catalog_releases": [
        {
          "name": "GPT-5 nano",
          "date": "2025-08-07",
          "link": "https://cdn.openai.com/gpt-5-system-card.pdf"
        }
      ],
      "eci": 139.09,
      "eci_low": 137.15,
      "eci_high": 140.93,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5-nano-2025-08-07_high",
        "gpt-5-nano-2025-08-07_low",
        "gpt-5-nano-2025-08-07_medium",
        "gpt-5-nano-2025-08-07_minimal",
        "gpt-5-nano-2025-08-07_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 2.4390243902439024,
          "variant": "gpt-5-nano-2025-08-07_high",
          "configurations": [
            {
              "variant": "gpt-5-nano-2025-08-07_high",
              "score": 2.4390243902439024
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 11.700000000000001,
          "variant": "gpt-5-nano-2025-08-07_high",
          "configurations": [
            {
              "variant": "gpt-5-nano-2025-08-07_high",
              "score": 11.700000000000001
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 9.0,
          "variant": "gpt-5-nano-2025-08-07_medium",
          "configurations": [
            {
              "variant": "gpt-5-nano-2025-08-07_high",
              "score": 8.0
            },
            {
              "variant": "gpt-5-nano-2025-08-07_medium",
              "score": 9.0
            },
            {
              "variant": "gpt-5-nano-2025-08-07_low",
              "score": 9.0
            },
            {
              "variant": "gpt-5-nano-2025-08-07_minimal",
              "score": 5.0
            }
          ]
        }
      },
      "catalog_link": "https://cdn.openai.com/gpt-5-system-card.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5 mini",
      "name": "GPT-5 mini",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-08-07",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-08-07",
      "catalog_releases": [
        {
          "name": "GPT-5 mini",
          "date": "2025-08-07",
          "link": "https://cdn.openai.com/gpt-5-system-card.pdf"
        }
      ],
      "eci": 146.01,
      "eci_low": 144.4,
      "eci_high": 147.59,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-5-mini-2025-08-07_high",
        "gpt-5-mini-2025-08-07_low",
        "gpt-5-mini-2025-08-07_medium",
        "gpt-5-mini-2025-08-07_minimal",
        "gpt-5-mini-2025-08-07_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 12.195121951219512,
          "variant": "gpt-5-mini-2025-08-07_high",
          "configurations": [
            {
              "variant": "gpt-5-mini-2025-08-07_high",
              "score": 12.195121951219512
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 21.6,
          "variant": "gpt-5-mini-2025-08-07_high",
          "configurations": [
            {
              "variant": "gpt-5-mini-2025-08-07_high",
              "score": 21.6
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 10.0,
          "variant": "gpt-5-mini-2025-08-07_minimal",
          "configurations": [
            {
              "variant": "gpt-5-mini-2025-08-07_high",
              "score": 5.0
            },
            {
              "variant": "gpt-5-mini-2025-08-07_medium",
              "score": 4.0
            },
            {
              "variant": "gpt-5-mini-2025-08-07_minimal",
              "score": 10.0
            }
          ]
        }
      },
      "catalog_link": "https://cdn.openai.com/gpt-5-system-card.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-5",
      "name": "GPT-5",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-08-07",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-08-07",
      "catalog_releases": [
        {
          "name": "GPT-5",
          "date": "2025-08-07",
          "link": "https://cdn.openai.com/gpt-5-system-card.pdf\nhttps://openai.com/index/introducing-gpt-5/"
        }
      ],
      "eci": 150.0,
      "eci_low": null,
      "eci_high": null,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": 6.6e+25,
      "variants": [
        "gpt-5-2025-08-07_high",
        "gpt-5-2025-08-07_low",
        "gpt-5-2025-08-07_medium",
        "gpt-5-2025-08-07_minimal",
        "gpt-5-2025-08-07_unknown",
        "gpt-5-chat"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 21.951219512195124,
          "variant": "gpt-5-2025-08-07_high",
          "configurations": [
            {
              "variant": "gpt-5-2025-08-07_high",
              "score": 21.951219512195124
            }
          ]
        },
        "mirrorcode.csv": {
          "score": 4.444444444444445,
          "variant": "gpt-5-2025-08-07_high",
          "configurations": [
            {
              "variant": "gpt-5-2025-08-07_high",
              "score": 4.444444444444445
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 50.1,
          "variant": "gpt-5-2025-08-07_high",
          "configurations": [
            {
              "variant": "gpt-5-2025-08-07_high",
              "score": 50.1
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 23.0,
          "variant": "gpt-5-2025-08-07_high",
          "configurations": [
            {
              "variant": "gpt-5-2025-08-07_medium",
              "score": 15.0
            },
            {
              "variant": "gpt-5-2025-08-07_low",
              "score": 16.0
            },
            {
              "variant": "gpt-5-2025-08-07_minimal",
              "score": 14.000000000000002
            },
            {
              "variant": "gpt-5-2025-08-07_high",
              "score": 23.0
            }
          ]
        }
      },
      "catalog_link": "https://cdn.openai.com/gpt-5-system-card.pdf\nhttps://openai.com/index/introducing-gpt-5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ChatGPT Latest",
      "name": "ChatGPT Latest",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-08-07",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "alpine-alpha_high",
        "alpine-alpha_xhigh",
        "galapagos-alpha",
        "galapagos-alpha_xhigh",
        "galapagos-pro-alpha_xhigh"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "qwen3-4b-instruct-2507",
      "name": "qwen3-4b-instruct-2507",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2025-08-06",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen3-4b-instruct-2507"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "gpt-oss-20b",
      "name": "gpt-oss-20b",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-08-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-08-05",
      "catalog_releases": [
        {
          "name": "gpt-oss-20b",
          "date": "2025-08-05",
          "link": "https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf"
        }
      ],
      "eci": 134.88,
      "eci_low": 132.27,
      "eci_high": 137.31,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 20910000000.0,
      "compute_flop": 5.49e+23,
      "variants": [
        "gpt-oss-20b",
        "gpt-oss-20b_high",
        "gpt-oss-20b_low",
        "gpt-oss-20b_medium",
        "gpt-oss-20b_unknown"
      ],
      "benchmarks": {},
      "catalog_link": "https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "gpt-oss-120b",
      "name": "gpt-oss-120b",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-08-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-08-05",
      "catalog_releases": [
        {
          "name": "gpt-oss-120b",
          "date": "2025-08-05",
          "link": "https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf"
        }
      ],
      "eci": 140.23,
      "eci_low": 138.26,
      "eci_high": 142.08,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 116830000000.0,
      "compute_flop": 4.94e+24,
      "variants": [
        "chutes/gpt-oss-120b",
        "chutes/gpt-oss-120b_high",
        "gpt-oss-120b",
        "gpt-oss-120b_high",
        "gpt-oss-120b_low",
        "gpt-oss-120b_medium",
        "gpt-oss-120b_unknown",
        "openai/gpt-oss-120b_high",
        "openai/gpt-oss-120b_low"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 2.0,
          "variant": "gpt-oss-120b_medium",
          "configurations": [
            {
              "variant": "gpt-oss-120b_high",
              "score": 0.0
            },
            {
              "variant": "gpt-oss-120b_medium",
              "score": 2.0
            }
          ]
        }
      },
      "catalog_link": "https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Opus 4.1",
      "name": "Claude Opus 4.1",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-08-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-08-05",
      "catalog_releases": [
        {
          "name": "Claude Opus 4.1",
          "date": "2025-08-05",
          "link": "https://www.anthropic.com/news/claude-opus-4-1"
        }
      ],
      "eci": 144.77,
      "eci_low": 143.0,
      "eci_high": 146.5,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-opus-4-1-20250805",
        "claude-opus-4-1-20250805_16K",
        "claude-opus-4-1-20250805_24K",
        "claude-opus-4-1-20250805_27K",
        "claude-opus-4-1-20250805_32K",
        "claude-opus-4-1-20250805_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 2.4390243902439024,
          "variant": "claude-opus-4-1-20250805_32K",
          "configurations": [
            {
              "variant": "claude-opus-4-1-20250805_32K",
              "score": 2.4390243902439024
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 21.0,
          "variant": "claude-opus-4-1-20250805_24K",
          "configurations": [
            {
              "variant": "claude-opus-4-1-20250805_24K",
              "score": 21.0
            }
          ]
        }
      },
      "catalog_link": "https://www.anthropic.com/news/claude-opus-4-1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen Image",
      "name": "Qwen Image",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-08-04",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-08-04",
      "catalog_releases": [
        {
          "name": "Qwen Image",
          "date": "2025-08-04",
          "link": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/Qwen_Image.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Image generation,Vision",
      "parameters": 27000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-Image/Qwen_Image.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Deep Think",
      "name": "Gemini 2.5 Deep Think",
      "lab": "Google DeepMind",
      "organization": "Google,Google DeepMind",
      "lab_inferred": false,
      "date": "2025-08-01",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-08-01",
      "catalog_releases": [
        {
          "name": "Gemini 2.5 Deep Think",
          "date": "2025-08-01",
          "link": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Deep-Think-Model-Card.pdf\n\nhttps://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Language,Multimodal,Vision,Video,Audio,Mathematics",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-deep-think-2025-08-01-webapp"
      ],
      "benchmarks": {},
      "catalog_link": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Deep-Think-Model-Card.pdf\n\nhttps://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-Coder-480B-A35B",
      "name": "Qwen3-Coder-480B-A35B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-07-31",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-07-22",
      "catalog_releases": [
        {
          "name": "Qwen3-Coder-480B-A35B",
          "date": "2025-07-22",
          "link": "https://qwenlm.github.io/blog/qwen3-coder/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 480000000000.0,
      "compute_flop": 1.575e+24,
      "variants": [
        "Qwen3-Coder-480B-A35B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen3-coder/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-30B-A3B-Thinking (Jul 2025)",
      "name": "Qwen3-30B-A3B-Thinking (Jul 2025)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-07-30",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 138.35,
      "eci_low": 135.77,
      "eci_high": 140.82,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 6.48e+23,
      "variants": [
        "qwen3-30b-a3b-thinking-2507"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-30B-A3B-Instruct (Jul 2025)",
      "name": "Qwen3-30B-A3B-Instruct (Jul 2025)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-07-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 134.83,
      "eci_low": 131.99,
      "eci_high": 137.52,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 6.48e+23,
      "variants": [
        "qwen3-30b-a3b-instruct-2507"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-235B-A22B-Thinking (Jul 2025)",
      "name": "Qwen3-235B-A22B-Thinking (Jul 2025)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-07-25",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-07-25",
      "catalog_releases": [
        {
          "name": "Qwen3-235B-A22B-Thinking (Jul 2025)",
          "date": "2025-07-25",
          "link": "https://qwenlm.github.io/blog/qwen3/"
        }
      ],
      "eci": 143.27,
      "eci_low": 141.4,
      "eci_high": 145.12,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 235000000000.0,
      "compute_flop": 4.752e+24,
      "variants": [
        "Qwen/Qwen3-235B-A22B-Thinking-2507",
        "Qwen3-235B-A22B-Thinking-2507",
        "accounts/fireworks/models/qwen3-235b-a22b-thinking-2507",
        "chutes/Qwen3-235B-A22B-Thinking-2507",
        "qwen/qwen3-235b-a22b-thinking-2507"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 40.44044044044044,
          "variant": "Qwen3-235B-A22B-Thinking-2507",
          "configurations": [
            {
              "variant": "Qwen3-235B-A22B-Thinking-2507",
              "score": 40.44044044044044
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 9.0,
          "variant": "Qwen3-235B-A22B-Thinking-2507",
          "configurations": [
            {
              "variant": "Qwen3-235B-A22B-Thinking-2507",
              "score": 9.0
            }
          ]
        }
      },
      "catalog_link": "https://qwenlm.github.io/blog/qwen3/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-235B-A22B-Instruct (Jul 2025)",
      "name": "Qwen3-235B-A22B-Instruct (Jul 2025)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-07-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 138.07,
      "eci_low": 135.66,
      "eci_high": 140.24,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 4.752e+24,
      "variants": [
        "Qwen3-235B-A22B-Instruct-2507",
        "parasail-qwen3-235b-a22b-instruct-2507"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-235B-A22B (Jul 2025)",
      "name": "Qwen3-235B-A22B (Jul 2025)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-07-25",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-07-25",
      "catalog_releases": [
        {
          "name": "Qwen3-235B-A22B (Jul 2025)",
          "date": "2025-07-25",
          "link": "https://qwenlm.github.io/blog/qwen3/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 235000000000.0,
      "compute_flop": 4.752e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen3/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini Embedding",
      "name": "Gemini Embedding",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-07-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-07-14",
      "catalog_releases": [
        {
          "name": "Gemini Embedding",
          "date": "2025-07-14",
          "link": "https://arxiv.org/abs/2503.07891"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2503.07891",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4 Heavy",
      "name": "Grok 4 Heavy",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2025-07-10",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-07-10",
      "catalog_releases": [
        {
          "name": "Grok 4 Heavy",
          "date": "2025-07-10",
          "link": "https://x.ai/news/grok-4"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-4-heavy-web-app"
      ],
      "benchmarks": {},
      "catalog_link": "https://x.ai/news/grok-4",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 4",
      "name": "Grok 4",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2025-07-09",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-07-09",
      "catalog_releases": [
        {
          "name": "Grok 4",
          "date": "2025-07-09",
          "link": "https://x.ai/news/grok-4"
        }
      ],
      "eci": 146.96,
      "eci_low": 145.48,
      "eci_high": 148.46,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": 3000000000000.0,
      "compute_flop": 5.0000000000001e+26,
      "variants": [
        "grok-4-0709",
        "grok-4-0709_high"
      ],
      "benchmarks": {},
      "catalog_link": "https://x.ai/news/grok-4",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok-3 mini",
      "name": "Grok-3 mini",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2025-06-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 139.95,
      "eci_low": 138.15,
      "eci_high": 141.69,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-3-mini-beta_high",
        "grok-3-mini-beta_low",
        "grok-3-mini-beta_medium",
        "grok-3-mini-beta_unknown",
        "grok-3-mini_high",
        "grok-3-mini_low",
        "grok-3-mini_unknown"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Small 3.2",
      "name": "Mistral Small 3.2",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-06-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-small-2506",
        "mistral-small-3.2-2506"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Flash-Lite (Jun 2025)",
      "name": "Gemini 2.5 Flash-Lite (Jun 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-06-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 133.05,
      "eci_low": 129.37,
      "eci_high": 136.47,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-flash-lite-preview-06-17-thinking",
        "gemini-2.5-flash-lite-preview-06-17_16K"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Flash (Jun 2025)",
      "name": "Gemini 2.5 Flash (Jun 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-06-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 140.84,
      "eci_low": 138.4,
      "eci_high": 143.12,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-flash"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "FGN",
      "name": "FGN",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-06-12",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-06-12",
      "catalog_releases": [
        {
          "name": "FGN",
          "date": "2025-06-12",
          "link": "https://arxiv.org/abs/2506.10772v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Earth science",
      "parameters": 720000000.0,
      "compute_flop": 9.618950880000001e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2506.10772v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "o3-pro",
      "name": "o3-pro",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-06-10",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 147.46,
      "eci_low": 145.58,
      "eci_high": 149.37,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "o3-pro-2025-06-10_high",
        "o3-pro-2025-06-10_low",
        "o3-pro-2025-06-10_medium",
        "o3-pro-2025-06-10_unknown"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Magistral Small 1.0",
      "name": "Magistral Small 1.0",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-06-10",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "magistral-small-2506"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Magistral Medium 1.1",
      "name": "Magistral Medium 1.1",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-06-10",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "magistral-medium-2506"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3 Embedding",
      "name": "Qwen3 Embedding",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-06-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-06-05",
      "catalog_releases": [
        {
          "name": "Qwen3 Embedding",
          "date": "2025-06-05",
          "link": "https://arxiv.org/abs/2506.05176v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 8000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2506.05176v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Pro (Jun 2025)",
      "name": "Gemini 2.5 Pro (Jun 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-06-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-06-05",
      "catalog_releases": [
        {
          "name": "Gemini 2.5 Pro (Jun 2025)",
          "date": "2025-06-05",
          "link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/#gemini-2-5-thinking"
        }
      ],
      "eci": 144.91,
      "eci_low": 143.5,
      "eci_high": 146.3,
      "access": "Closed weights",
      "domain": "Language,Vision,Video,Multimodal,Speech",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-pro",
        "gemini-2.5-pro-preview-06-05",
        "gemini-2.5-pro-preview-06-05_1K",
        "gemini-2.5-pro-preview-06-05_32K",
        "gemini-2.5-pro_16K",
        "gemini-2.5-pro_32K",
        "gemini-2.5-pro_8K"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 0.0,
          "variant": "gemini-2.5-pro",
          "configurations": [
            {
              "variant": "gemini-2.5-pro",
              "score": 0.0
            }
          ]
        }
      },
      "catalog_link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/#gemini-2-5-thinking",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "deepseek-r1-0528-qwen3-8b",
      "name": "deepseek-r1-0528-qwen3-8b",
      "lab": "DeepSeek",
      "organization": null,
      "lab_inferred": true,
      "date": "2025-05-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "deepseek-r1-0528-qwen3-8b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-R1 (May 2025)",
      "name": "DeepSeek-R1 (May 2025)",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-05-28",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-05-28",
      "catalog_releases": [
        {
          "name": "DeepSeek-R1 (May 2025)",
          "date": "2025-05-28",
          "link": "https://api-docs.deepseek.com/news/news250120"
        }
      ],
      "eci": 140.75,
      "eci_low": 138.93,
      "eci_high": 142.45,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 671000000000.0,
      "compute_flop": 3.3e+24,
      "variants": [
        "DeepSeek-R1-0528",
        "chutes/DeepSeek-R1-0528"
      ],
      "benchmarks": {},
      "catalog_link": "https://api-docs.deepseek.com/news/news250120",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Codestral Embed",
      "name": "Codestral Embed",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-05-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "codestral-embed"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "claude-strudel-v4-p_max",
      "name": "claude-strudel-v4-p_max",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-05-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-strudel-v4-p_max"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "claude-strudel-v4-p_high",
      "name": "claude-strudel-v4-p_high",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-05-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-strudel-v4-p_high"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "claude-strudel-v4-p",
      "name": "claude-strudel-v4-p",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-05-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-strudel-v4-p"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral OCR (May 2025)",
      "name": "Mistral OCR (May 2025)",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-05-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-ocr-2505"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Sonnet 4",
      "name": "Claude Sonnet 4",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-05-22",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-05-22",
      "catalog_releases": [
        {
          "name": "Claude Sonnet 4",
          "date": "2025-05-22",
          "link": "https://www.anthropic.com/claude/sonnet\n\nhttps://www.anthropic.com/news/claude-4"
        }
      ],
      "eci": 141.88,
      "eci_low": 140.3,
      "eci_high": 143.41,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-sonnet-4-20250514",
        "claude-sonnet-4-20250514_12K",
        "claude-sonnet-4-20250514_16K",
        "claude-sonnet-4-20250514_1K",
        "claude-sonnet-4-20250514_2K",
        "claude-sonnet-4-20250514_32K",
        "claude-sonnet-4-20250514_59K",
        "claude-sonnet-4-20250514_64K",
        "claude-sonnet-4-20250514_8K",
        "claude-sonnet-4-20250514_unknown",
        "claude-strudel-v8-p",
        "claude-strudel-v8-p_max"
      ],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/claude/sonnet\n\nhttps://www.anthropic.com/news/claude-4",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Opus 4",
      "name": "Claude Opus 4",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-05-22",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-05-22",
      "catalog_releases": [
        {
          "name": "Claude Opus 4",
          "date": "2025-05-22",
          "link": "https://www.anthropic.com/claude/opus\n\nhttps://www.anthropic.com/news/claude-4"
        }
      ],
      "eci": 142.88,
      "eci_low": 141.27,
      "eci_high": 144.47,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-opus-4-20250514",
        "claude-opus-4-20250514_12K",
        "claude-opus-4-20250514_16K",
        "claude-opus-4-20250514_1K",
        "claude-opus-4-20250514_27K",
        "claude-opus-4-20250514_2K",
        "claude-opus-4-20250514_32K",
        "claude-opus-4-20250514_8K",
        "claude-opus-4-20250514_unknown"
      ],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/claude/opus\n\nhttps://www.anthropic.com/news/claude-4",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Veo 3",
      "name": "Veo 3",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-05-21",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-05-21",
      "catalog_releases": [
        {
          "name": "Veo 3",
          "date": "2025-05-21",
          "link": "https://deepmind.google/models/veo/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Video,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://deepmind.google/models/veo/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Devstral Small",
      "name": "Devstral Small",
      "lab": "Mistral",
      "organization": "Mistral AI,All Hands AI",
      "lab_inferred": false,
      "date": "2025-05-21",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "devstral-small-2505"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Imagen 4",
      "name": "Imagen 4",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2025-05-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-05-20",
      "catalog_releases": [
        {
          "name": "Imagen 4",
          "date": "2025-05-20",
          "link": "https://storage.googleapis.com/deepmind-media/Model-Cards/Imagen-4-Model-Card.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://storage.googleapis.com/deepmind-media/Model-Cards/Imagen-4-Model-Card.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Flash (May 2025)",
      "name": "Gemini 2.5 Flash (May 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-05-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 141.58,
      "eci_low": 139.44,
      "eci_high": 143.64,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-flash-preview-05-20",
        "gemini-2.5-flash-preview-05-20_16K",
        "gemini-2.5-flash-preview-05-20_1K",
        "gemini-2.5-flash-preview-05-20_23K",
        "gemini-2.5-flash-preview-05-20_8K"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "codex-mini-2025-05-16",
      "name": "codex-mini-2025-05-16",
      "lab": "OpenAI",
      "organization": null,
      "lab_inferred": true,
      "date": "2025-05-16",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "codex-mini-2025-05-16"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Medium 3",
      "name": "Mistral Medium 3",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-05-07",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 132.81,
      "eci_low": 130.17,
      "eci_high": 135.2,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-medium-2505"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Pro (May 2025)",
      "name": "Gemini 2.5 Pro (May 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-05-06",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-05-06",
      "catalog_releases": [
        {
          "name": "Gemini 2.5 Pro (May 2025)",
          "date": "2025-05-06",
          "link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/#gemini-2-5-thinking"
        }
      ],
      "eci": 140.64,
      "eci_low": 138.31,
      "eci_high": 142.95,
      "access": "Closed weights",
      "domain": "Language,Vision,Video,Multimodal,Speech",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-pro-preview-05-06"
      ],
      "benchmarks": {},
      "catalog_link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/#gemini-2-5-thinking",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-Prover-V2-671B",
      "name": "DeepSeek-Prover-V2-671B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-04-30",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "deepseek-prover-v2"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-4B",
      "name": "Qwen3-4B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 8.64e+23,
      "variants": [
        "qwen3-4b",
        "qwen3-4b_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-32B",
      "name": "Qwen3-32B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 136.84,
      "eci_low": 134.41,
      "eci_high": 139.22,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 7.0848e+24,
      "variants": [
        "Qwen3-32B",
        "Qwen3-32B_none",
        "chutes/Qwen3-32B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-30B-A3B",
      "name": "Qwen3-30B-A3B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 134.18,
      "eci_low": 131.45,
      "eci_high": 136.76,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 6.48e+23,
      "variants": [
        "Qwen3-30B-A3B",
        "Qwen3-30B-A3B_none",
        "chutes/Qwen3-30B-A3B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-14B",
      "name": "Qwen3-14B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 136.08,
      "eci_low": 133.42,
      "eci_high": 138.55,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.1968e+24,
      "variants": [
        "chutes/Qwen3-14B",
        "qwen3-14b",
        "qwen3-14b_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-1.7B",
      "name": "Qwen3-1.7B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.672e+23,
      "variants": [
        "qwen3-1.7b",
        "qwen3-1.7b_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-0.6B",
      "name": "Qwen3-0.6B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.296e+23,
      "variants": [
        "qwen3-0.6b",
        "qwen3-0.6b_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-8B",
      "name": "Qwen3-8B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 132.08,
      "eci_low": 129.06,
      "eci_high": 134.78,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.7712e+24,
      "variants": [
        "chutes/Qwen3-8B",
        "qwen3-8b",
        "qwen3-8b_none"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen3-235B-A22B",
      "name": "Qwen3-235B-A22B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-28",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-04-29",
      "catalog_releases": [
        {
          "name": "Qwen3-235B-A22B",
          "date": "2025-04-29",
          "link": "https://qwenlm.github.io/blog/qwen3/"
        }
      ],
      "eci": 138.43,
      "eci_low": 136.11,
      "eci_high": 140.67,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 235000000000.0,
      "compute_flop": 4.752e+24,
      "variants": [
        "Qwen3-235B-A22B",
        "chutes/Qwen3-235B-A22B",
        "qwen3-235b-a22b"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen3/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen Plus (Apr 2025)",
      "name": "Qwen Plus (Apr 2025)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen-plus-2025-04-28"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "gpt-image-1",
      "name": "gpt-image-1",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-04-23",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-04-23",
      "catalog_releases": [
        {
          "name": "gpt-image-1",
          "date": "2025-04-23",
          "link": "https://openai.com/index/image-generation-api/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/image-generation-api/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Flash (Apr 2025)",
      "name": "Gemini 2.5 Flash (Apr 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-04-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 140.73,
      "eci_low": 138.35,
      "eci_high": 142.96,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-flash-preview-04-17",
        "gemini-2.5-flash-preview-04-17 (16K thinking)",
        "gemini-2.5-flash-preview-04-17 (24K thinking)"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "o4-mini",
      "name": "o4-mini",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-04-16",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 145.22,
      "eci_low": 143.68,
      "eci_high": 146.7,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "o4-mini-2025-04-16_high",
        "o4-mini-2025-04-16_low",
        "o4-mini-2025-04-16_medium",
        "o4-mini-2025-04-16_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 4.878048780487805,
          "variant": "o4-mini-2025-04-16_high",
          "configurations": [
            {
              "variant": "o4-mini-2025-04-16_high",
              "score": 4.878048780487805
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 19.6,
          "variant": "o4-mini-2025-04-16_low",
          "configurations": [
            {
              "variant": "o4-mini-2025-04-16_low",
              "score": 19.6
            },
            {
              "variant": "o4-mini-2025-04-16_high",
              "score": 18.8
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 5.0,
          "variant": "o4-mini-2025-04-16_high",
          "configurations": [
            {
              "variant": "o4-mini-2025-04-16_high",
              "score": 5.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "o3",
      "name": "o3",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-04-16",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-12-20",
      "catalog_releases": [
        {
          "name": "o3",
          "date": "2024-12-20",
          "link": "https://openai.com/index/introducing-o3-and-o4-mini/"
        }
      ],
      "eci": 147.05,
      "eci_low": 145.6,
      "eci_high": 148.55,
      "access": "Closed weights",
      "domain": "Language,Vision,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "o3-2025-04-16_high",
        "o3-2025-04-16_low",
        "o3-2025-04-16_medium",
        "o3-2025-04-16_unknown"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 49.4,
          "variant": "o3-2025-04-16_high",
          "configurations": [
            {
              "variant": "o3-2025-04-16_high",
              "score": 49.4
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 28.999999999999996,
          "variant": "o3-2025-04-16_high",
          "configurations": [
            {
              "variant": "o3-2025-04-16_high",
              "score": 28.999999999999996
            },
            {
              "variant": "o3-2025-04-16_medium",
              "score": 23.0
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-o3-and-o4-mini/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4.1 nano",
      "name": "GPT-4.1 nano",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-04-14",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 126.81,
      "eci_low": 123.79,
      "eci_high": 129.59,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4.1-nano-2025-04-14"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 6.0,
          "variant": "gpt-4.1-nano-2025-04-14",
          "configurations": [
            {
              "variant": "gpt-4.1-nano-2025-04-14",
              "score": 6.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4.1 mini",
      "name": "GPT-4.1 mini",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-04-14",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 134.27,
      "eci_low": 132.13,
      "eci_high": 136.21,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4.1-mini-2025-04-14"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 12.7,
          "variant": "gpt-4.1-mini-2025-04-14",
          "configurations": [
            {
              "variant": "gpt-4.1-mini-2025-04-14",
              "score": 12.7
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 7.000000000000001,
          "variant": "gpt-4.1-mini-2025-04-14",
          "configurations": [
            {
              "variant": "gpt-4.1-mini-2025-04-14",
              "score": 7.000000000000001
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4.1",
      "name": "GPT-4.1",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-04-14",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 136.18,
      "eci_low": 134.14,
      "eci_high": 138.06,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4.1-2025-04-14"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 31.1,
          "variant": "gpt-4.1-2025-04-14",
          "configurations": [
            {
              "variant": "gpt-4.1-2025-04-14",
              "score": 31.1
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok 3",
      "name": "Grok 3",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2025-04-09",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-02-17",
      "catalog_releases": [
        {
          "name": "Grok 3",
          "date": "2025-02-17",
          "link": "https://x.ai/blog/grok-3"
        }
      ],
      "eci": 137.59,
      "eci_low": 135.63,
      "eci_high": 139.42,
      "access": "Closed weights",
      "domain": "Language,Vision,Multimodal",
      "parameters": 3000000000000.0,
      "compute_flop": 3.5e+26,
      "variants": [
        "grok-3",
        "grok-3-beta"
      ],
      "benchmarks": {},
      "catalog_link": "https://x.ai/blog/grok-3",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "QWQ-Plus",
      "name": "QWQ-Plus",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-04-08",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwq-plus"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 4 Maverick",
      "name": "Llama 4 Maverick",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2025-04-06",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-04-05",
      "catalog_releases": [
        {
          "name": "Llama 4 Maverick",
          "date": "2025-04-05",
          "link": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/"
        }
      ],
      "eci": 132.24,
      "eci_low": 130.0,
      "eci_high": 134.26,
      "access": "Open weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": 400000000000.0,
      "compute_flop": 2.244000000001e+24,
      "variants": [
        "Llama-4-Maverick-17B-128E-Instruct",
        "Llama-4-Maverick-17B-128E-Instruct-FP8",
        "chutes/Llama-4-Maverick-17B-128E-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 4 Scout",
      "name": "Llama 4 Scout",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2025-04-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-04-05",
      "catalog_releases": [
        {
          "name": "Llama 4 Scout",
          "date": "2025-04-05",
          "link": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/"
        }
      ],
      "eci": 127.77,
      "eci_low": 124.88,
      "eci_high": 130.4,
      "access": "Open weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": 109000000000.0,
      "compute_flop": 4.08e+24,
      "variants": [
        "Llama-4-Scout-17B-16E-Instruct",
        "chutes/Llama-4-Scout-17B-16E Instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 4 Behemoth (preview)",
      "name": "Llama 4 Behemoth (preview)",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2025-04-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-04-05",
      "catalog_releases": [
        {
          "name": "Llama 4 Behemoth (preview)",
          "date": "2025-04-05",
          "link": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Language,Vision",
      "parameters": 2000000000000.0,
      "compute_flop": 5.18400000000001e+25,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.5 Pro (Mar 2025)",
      "name": "Gemini 2.5 Pro (Mar 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-03-31",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-03-25",
      "catalog_releases": [
        {
          "name": "Gemini 2.5 Pro (Mar 2025)",
          "date": "2025-03-25",
          "link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/#gemini-2-5-thinking"
        }
      ],
      "eci": 144.05,
      "eci_low": 142.08,
      "eci_high": 146.0,
      "access": "Closed weights",
      "domain": "Language,Vision,Video,Multimodal,Speech",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.5-pro-exp-03-25",
        "gemini-2.5-pro-preview-03-25"
      ],
      "benchmarks": {},
      "catalog_link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/#gemini-2-5-thinking",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4o (Mar 2025)",
      "name": "GPT-4o (Mar 2025)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-03-27",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-03-27",
      "catalog_releases": [
        {
          "name": "GPT-4o (Mar 2025)",
          "date": "2025-03-27",
          "link": "https://help.openai.com/en/articles/6825453-chatgpt-release-notes"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Audio,Speech,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "chatgpt-4o-03-27-2025"
      ],
      "benchmarks": {},
      "catalog_link": "https://help.openai.com/en/articles/6825453-chatgpt-release-notes",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "QVQ-Max",
      "name": "QVQ-Max",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-03-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qvq-max-2025-03-25"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V3 (Mar 2025)",
      "name": "DeepSeek-V3 (Mar 2025)",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-03-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-03-24",
      "catalog_releases": [
        {
          "name": "DeepSeek-V3 (Mar 2025)",
          "date": "2025-03-24",
          "link": "https://arxiv.org/abs/2412.19437"
        }
      ],
      "eci": 135.28,
      "eci_low": 133.12,
      "eci_high": 137.34,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 671000000000.0,
      "compute_flop": 3.3e+24,
      "variants": [
        "DeepSeek-V3-0324",
        "chutes/DeepSeek-V3-0324"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2412.19437",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "o1-pro",
      "name": "o1-pro",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-03-19",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-03-19",
      "catalog_releases": [
        {
          "name": "o1-pro",
          "date": "2025-03-19",
          "link": "https://openai.com/index/introducing-chatgpt-pro/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Mathematics,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "o1-pro-2025-03-19",
        "o1-pro-2025-03-19_low"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-chatgpt-pro/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Small 3.1",
      "name": "Mistral Small 3.1",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-03-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-small-2503",
        "mistral-small-3.1-2503"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 3 4B",
      "name": "Gemma 3 4B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-03-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 9.6e+22,
      "variants": [
        "gemma-3-4b-it"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 3 27B",
      "name": "Gemma 3 27B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-03-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 127.5,
      "eci_low": 124.41,
      "eci_high": 130.28,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.268e+24,
      "variants": [
        "chutes/Gemma-3-27b-It",
        "gemma-3-27b-it"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 3 1B",
      "name": "Gemma 3 1B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-03-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.2e+22,
      "variants": [
        "gemma-3-1b-it"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 3 12B",
      "name": "Gemma 3 12B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-03-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 8.64e+23,
      "variants": [
        "gemma-3-12b-it"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral OCR (Mar 2025)",
      "name": "Mistral OCR (Mar 2025)",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-03-06",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-ocr-2503"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral OCR",
      "name": "Mistral OCR",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-03-06",
      "date_basis": "Catalog publication date",
      "catalog_date": "2025-03-06",
      "catalog_releases": [
        {
          "name": "Mistral OCR",
          "date": "2025-03-06",
          "link": "https://mistral.ai/news/mistral-ocr"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Vision,Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://mistral.ai/news/mistral-ocr",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "QwQ-32B",
      "name": "QwQ-32B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-03-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-03-06",
      "catalog_releases": [
        {
          "name": "QwQ-32B",
          "date": "2025-03-06",
          "link": "https://qwenlm.github.io/blog/qwq-32b/"
        }
      ],
      "eci": 136.48,
      "eci_low": 133.72,
      "eci_high": 139.15,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 32500000000.0,
      "compute_flop": 3.51e+24,
      "variants": [
        "QwQ-32B",
        "QwQ-32B (16K thinking)",
        "chutes/QwQ-32B"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwq-32b/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4.5",
      "name": "GPT-4.5",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-02-27",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-02-27",
      "catalog_releases": [
        {
          "name": "GPT-4.5",
          "date": "2025-02-27",
          "link": "https://openai.com/index/introducing-gpt-4-5/"
        }
      ],
      "eci": 136.5,
      "eci_low": 134.41,
      "eci_high": 138.45,
      "access": "Closed weights",
      "domain": "Language,Vision,Multimodal",
      "parameters": null,
      "compute_flop": 3.8e+26,
      "variants": [
        "gpt-4.5-preview-2025-02-27"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-gpt-4-5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 3.7 Sonnet",
      "name": "Claude 3.7 Sonnet",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2025-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-02-24",
      "catalog_releases": [
        {
          "name": "Claude 3.7 Sonnet",
          "date": "2025-02-24",
          "link": "https://www.anthropic.com/news/claude-3-7-sonnet"
        }
      ],
      "eci": 141.05,
      "eci_low": 139.41,
      "eci_high": 142.58,
      "access": "Closed weights",
      "domain": "Language,Vision,Multimodal",
      "parameters": null,
      "compute_flop": 3.35e+25,
      "variants": [
        "claude-3-7-sonnet-20250219",
        "claude-3-7-sonnet-20250219_12K",
        "claude-3-7-sonnet-20250219_15K",
        "claude-3-7-sonnet-20250219_16K",
        "claude-3-7-sonnet-20250219_1K",
        "claude-3-7-sonnet-20250219_2K",
        "claude-3-7-sonnet-20250219_32K",
        "claude-3-7-sonnet-20250219_64K",
        "claude-3-7-sonnet-20250219_8K",
        "claude-3-7-sonnet-20250219_unknown"
      ],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/news/claude-3-7-sonnet",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Saba",
      "name": "Mistral Saba",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-02-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-saba-2502"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.0 Pro",
      "name": "Gemini 2.0 Pro",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-02-05",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-12-11",
      "catalog_releases": [
        {
          "name": "Gemini 2.0 Pro",
          "date": "2024-12-11",
          "link": "https://deepmind.google/technologies/gemini/pro/"
        }
      ],
      "eci": 135.41,
      "eci_low": 132.66,
      "eci_high": 138.06,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision,Video,Audio",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.0-pro-exp-02-05"
      ],
      "benchmarks": {},
      "catalog_link": "https://deepmind.google/technologies/gemini/pro/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.0 Flash-Lite",
      "name": "Gemini 2.0 Flash-Lite",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2025-02-05",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.0-flash-lite",
        "gemini-2.0-flash-lite-001",
        "gemini-2.0-flash-lite-preview-02-05"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.0 Flash (Feb 2025)",
      "name": "Gemini 2.0 Flash (Feb 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,Google",
      "lab_inferred": false,
      "date": "2025-02-05",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 134.02,
      "eci_low": 131.74,
      "eci_high": 136.18,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.0-flash-001",
        "gemini-2.0-flash-02-05"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "o3-mini",
      "name": "o3-mini",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-01-31",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-01-31",
      "catalog_releases": [
        {
          "name": "o3-mini",
          "date": "2025-01-31",
          "link": "https://openai.com/index/openai-o3-mini/"
        }
      ],
      "eci": 140.65,
      "eci_low": 138.96,
      "eci_high": 142.25,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "o3-mini-2025-01-31_high",
        "o3-mini-2025-01-31_low",
        "o3-mini-2025-01-31_medium",
        "o3-mini-2025-01-31_unknown"
      ],
      "benchmarks": {
        "frontiermath_tier_4_v2.csv": {
          "score": 0.0,
          "variant": "o3-mini-2025-01-31_high",
          "configurations": [
            {
              "variant": "o3-mini-2025-01-31_high",
              "score": 0.0
            }
          ]
        },
        "simpleqa_verified.csv": {
          "score": 15.299999999999999,
          "variant": "o3-mini-2025-01-31_high",
          "configurations": [
            {
              "variant": "o3-mini-2025-01-31_high",
              "score": 15.299999999999999
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 7.000000000000001,
          "variant": "o3-mini-2025-01-31_high",
          "configurations": [
            {
              "variant": "o3-mini-2025-01-31_high",
              "score": 7.000000000000001
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/openai-o3-mini/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4o (Jan 2025)",
      "name": "GPT-4o (Jan 2025)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-01-29",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-01-29",
      "catalog_releases": [
        {
          "name": "GPT-4o (Jan 2025)",
          "date": "2025-01-29",
          "link": "https://help.openai.com/en/articles/9624314-model-release-notes"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Audio,Speech,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "chatgpt-4o-01-29-2025"
      ],
      "benchmarks": {},
      "catalog_link": "https://help.openai.com/en/articles/9624314-model-release-notes",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Max",
      "name": "Qwen2.5-Max",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-01-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 131.17,
      "eci_low": 128.42,
      "eci_high": 133.62,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen-max-2025-01-25",
        "qwen2.5-max"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen Plus (Jan 2025)",
      "name": "Qwen Plus (Jan 2025)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2025-01-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen-plus-2025-01-25"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Small 3",
      "name": "Mistral Small 3",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-01-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.152e+24,
      "variants": [
        "mistral-small-2501",
        "mistral-small-3-2501"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Computer-Using Agent (CUA)",
      "name": "Computer-Using Agent (CUA)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2025-01-23",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-01-23",
      "catalog_releases": [
        {
          "name": "Computer-Using Agent (CUA)",
          "date": "2025-01-23",
          "link": "https://openai.com/index/computer-using-agent/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Vision,Language,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "CUA"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/computer-using-agent/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.0 Flash Thinking (Jan 2025)",
      "name": "Gemini 2.0 Flash Thinking (Jan 2025)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,Google",
      "lab_inferred": false,
      "date": "2025-01-21",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 133.61,
      "eci_low": 130.54,
      "eci_high": 136.4,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.0-flash-thinking-exp-01-21"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-R1-Distill-Qwen-7B",
      "name": "DeepSeek-R1-Distill-Qwen-7B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-01-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-R1-Distill-Qwen-7B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-R1-Distill-Qwen-32B",
      "name": "DeepSeek-R1-Distill-Qwen-32B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-01-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-R1-Distill-Qwen-32B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-R1-Distill-Qwen-14B",
      "name": "DeepSeek-R1-Distill-Qwen-14B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-01-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-R1-Distill-Qwen-14B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-R1-Distill-Qwen-1.5B",
      "name": "DeepSeek-R1-Distill-Qwen-1.5B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-01-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-R1-Distill-Qwen-1.5B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-R1-Distill-Llama-8B",
      "name": "DeepSeek-R1-Distill-Llama-8B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-01-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-R1-Distill-Llama-8B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-R1-Distill-Llama-70B",
      "name": "DeepSeek-R1-Distill-Llama-70B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-01-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-R1-Distill-Llama-70B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-R1",
      "name": "DeepSeek-R1",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2025-01-20",
      "date_basis": "Benchmark release date",
      "catalog_date": "2025-01-20",
      "catalog_releases": [
        {
          "name": "DeepSeek-R1",
          "date": "2025-01-20",
          "link": "https://api-docs.deepseek.com/news/news250120"
        }
      ],
      "eci": 138.63,
      "eci_low": 136.7,
      "eci_high": 140.4,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 671000000000.0,
      "compute_flop": 3.3e+24,
      "variants": [
        "DeepSeek-R1"
      ],
      "benchmarks": {},
      "catalog_link": "https://api-docs.deepseek.com/news/news250120",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Codestral",
      "name": "Codestral",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2025-01-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "codestral-2501",
        "codestral-2508"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V3",
      "name": "DeepSeek-V3",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2024-12-26",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-12-24",
      "catalog_releases": [
        {
          "name": "DeepSeek-V3",
          "date": "2024-12-24",
          "link": "https://arxiv.org/abs/2412.19437"
        }
      ],
      "eci": 131.71,
      "eci_low": 129.14,
      "eci_high": 134.01,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 671000000000.0,
      "compute_flop": 3.3e+24,
      "variants": [
        "DeepSeek-V3",
        "DeepSeek-V3-Base"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2412.19437",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.0 Flash Thinking (Dec 2024)",
      "name": "Gemini 2.0 Flash Thinking (Dec 2024)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,Google",
      "lab_inferred": false,
      "date": "2024-12-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.0-flash-thinking-exp-1219"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama-3.1-Nemotron-70B-Instruct",
      "name": "Llama-3.1-Nemotron-70B-Instruct",
      "lab": "Meta",
      "organization": "Nvidia,Meta AI",
      "lab_inferred": false,
      "date": "2024-12-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-10-12",
      "catalog_releases": [
        {
          "name": "Llama-3.1-Nemotron-70B-Instruct",
          "date": "2024-10-12",
          "link": "https://catalog.ngc.nvidia.com/orgs/nim/teams/nvidia/containers/llama-3.1-nemotron-70b-instruct"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": null,
      "compute_flop": 7.929e+24,
      "variants": [
        "llama-3.1-nemotron-70b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://catalog.ngc.nvidia.com/orgs/nim/teams/nvidia/containers/llama-3.1-nemotron-70b-instruct",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "o1",
      "name": "o1",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-12-17",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-12-05",
      "catalog_releases": [
        {
          "name": "o1",
          "date": "2024-12-05",
          "link": "https://openai.com/index/introducing-chatgpt-pro/"
        }
      ],
      "eci": 141.18,
      "eci_low": 139.48,
      "eci_high": 142.79,
      "access": "Closed weights",
      "domain": "Language,Mathematics,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "o1-2024-12-17_high",
        "o1-2024-12-17_low",
        "o1-2024-12-17_medium",
        "o1-2024-12-17_unknown"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 41.099999999999994,
          "variant": "o1-2024-12-17_high",
          "configurations": [
            {
              "variant": "o1-2024-12-17_high",
              "score": 41.099999999999994
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/introducing-chatgpt-pro/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Veo 2",
      "name": "Veo 2",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-12-16",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-12-16",
      "catalog_releases": [
        {
          "name": "Veo 2",
          "date": "2024-12-16",
          "link": "https://deepmind.google/technologies/veo/veo-2/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Video,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://deepmind.google/technologies/veo/veo-2/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Apollo 7B",
      "name": "Apollo 7B",
      "lab": "Meta",
      "organization": "Meta AI,Stanford University",
      "lab_inferred": false,
      "date": "2024-12-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-12-13",
      "catalog_releases": [
        {
          "name": "Apollo 7B",
          "date": "2024-12-13",
          "link": "https://arxiv.org/abs/2412.10360"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Video,Language",
      "parameters": 7000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2412.10360",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok-2 (Dec 2024)",
      "name": "Grok-2 (Dec 2024)",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2024-12-12",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 129.75,
      "eci_low": 127.01,
      "eci_high": 132.25,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.96e+25,
      "variants": [
        "grok-2-1212",
        "grok-2-vision-1212"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V2.5 (Dec 2024)",
      "name": "DeepSeek-V2.5 (Dec 2024)",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2024-12-10",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.7892e+24,
      "variants": [
        "DeepSeek-V2.5-1210"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Sora Turbo",
      "name": "Sora Turbo",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-12-09",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-12-09",
      "catalog_releases": [
        {
          "name": "Sora Turbo",
          "date": "2024-12-09",
          "link": "https://openai.com/index/sora-is-here/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Video,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/sora-is-here/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3.3 70B",
      "name": "Llama 3.3 70B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-12-06",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-12-06",
      "catalog_releases": [
        {
          "name": "Llama 3.3 70B",
          "date": "2024-12-06",
          "link": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama3_3/"
        }
      ],
      "eci": 126.16,
      "eci_low": 123.1,
      "eci_high": 128.98,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 70000000000.0,
      "compute_flop": 6.8649768e+24,
      "variants": [
        "Llama-3.3-70B-Instruct",
        "Llama-3.3-70B-Instruct-Q8_0-GGUF-draft",
        "llama3.3:70b-instruct-q8_0"
      ],
      "benchmarks": {},
      "catalog_link": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama3_3/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 2.0 Flash (Dec 2024)",
      "name": "Gemini 2.0 Flash (Dec 2024)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,Google",
      "lab_inferred": false,
      "date": "2024-12-06",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-2.0-flash-exp",
        "gemini-exp-1206"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2-7B",
      "name": "Qwen2-7B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-11-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.9400000000001e+23,
      "variants": [
        "qwen2-7b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2-57B-A14B",
      "name": "Qwen2-57B-A14B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-11-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.7800000000001e+23,
      "variants": [
        "qwen2-57b-a14b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "QwQ-32B-Preview",
      "name": "QwQ-32B-Preview",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-11-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.51e+24,
      "variants": [
        "QwQ-32B-Preview",
        "QwQ-32B-Preview-Q8_0-GGUF"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder-32B-Instruct",
      "name": "Qwen2.5-Coder-32B-Instruct",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-11-21",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.0725e+24,
      "variants": [
        "Qwen2.5-Coder-32B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4o (Nov 2024)",
      "name": "GPT-4o (Nov 2024)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-11-20",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-11-20",
      "catalog_releases": [
        {
          "name": "GPT-4o (Nov 2024)",
          "date": "2024-11-20",
          "link": "https://help.openai.com/en/articles/9624314-model-release-notes"
        }
      ],
      "eci": 128.78,
      "eci_low": 126.12,
      "eci_high": 131.24,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Audio,Speech,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "chatgpt-4o-11-20-2024",
        "gpt-4o-2024-11-20"
      ],
      "benchmarks": {},
      "catalog_link": "https://help.openai.com/en/articles/9624314-model-release-notes",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Pixtral Large",
      "name": "Pixtral Large",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-11-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-11-18",
      "catalog_releases": [
        {
          "name": "Pixtral Large",
          "date": "2024-11-18",
          "link": "https://mistral.ai/news/pixtral-large/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Multimodal,Language,Vision",
      "parameters": 124000000000.0,
      "compute_flop": null,
      "variants": [
        "pixtral-large-2411"
      ],
      "benchmarks": {},
      "catalog_link": "https://mistral.ai/news/pixtral-large/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Large 2 (Nov 2024)",
      "name": "Mistral Large 2 (Nov 2024)",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-11-15",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Mistral-Large-Instruct-2411",
        "mistral-large-2411"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini-Exp-1114",
      "name": "Gemini-Exp-1114",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-11-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-11-15",
      "catalog_releases": [
        {
          "name": "Gemini-Exp-1114",
          "date": "2024-11-15",
          "link": "https://www.tomsguide.com/ai/google-gemini/google-drops-new-gemini-model-and-it-goes-straight-to-the-top-of-the-llm-leaderboard"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.tomsguide.com/ai/google-gemini/google-drops-new-gemini-model-and-it-goes-straight-to-the-top-of-the-llm-leaderboard",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder-3B-Instruct",
      "name": "Qwen2.5-Coder-3B-Instruct",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-11-06",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen2.5-Coder-3B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder-14B-Instruct",
      "name": "Qwen2.5-Coder-14B-Instruct",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-11-06",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen2.5-Coder-14B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-Turbo",
      "name": "Qwen-Turbo",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-11-01",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen-turbo-2024-11-01"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 3.5 Sonnet (October 2024)",
      "name": "Claude 3.5 Sonnet (October 2024)",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2024-10-22",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-10-22",
      "catalog_releases": [
        {
          "name": "Claude 3.5 Sonnet (October 2024)",
          "date": "2024-10-22",
          "link": "https://www.anthropic.com/news/3-5-models-and-computer-use"
        }
      ],
      "eci": 133.34,
      "eci_low": 131.04,
      "eci_high": 135.54,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-3-5-sonnet-20241022"
      ],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/news/3-5-models-and-computer-use",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 3.5 Haiku",
      "name": "Claude 3.5 Haiku",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2024-10-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 124.75,
      "eci_low": 121.49,
      "eci_high": 127.65,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-3-5-haiku-20241022"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Ministral 8B",
      "name": "Ministral 8B",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-10-16",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Ministral-8B-Instruct-2410",
        "ministral-8b-2410"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Ministral 3B",
      "name": "Ministral 3B",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-10-16",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Ministral-3B",
        "ministral-3b-2410"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Movie Gen Video",
      "name": "Movie Gen Video",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-10-04",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-10-04",
      "catalog_releases": [
        {
          "name": "Movie Gen Video",
          "date": "2024-10-04",
          "link": "https://ai.meta.com/static-resource/movie-gen-research-paper"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Video,Vision",
      "parameters": 30000000000.0,
      "compute_flop": 1.65e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/static-resource/movie-gen-research-paper",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.5 Flash 8B",
      "name": "Gemini 1.5 Flash 8B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-10-03",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-1.5-flash-8b-001"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3.2 90B",
      "name": "Llama 3.2 90B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-09-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 122.89,
      "eci_low": 119.04,
      "eci_high": 126.23,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Llama-3.2-90B-Vision-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3.2 3B",
      "name": "Llama 3.2 3B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-09-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.7334e+23,
      "variants": [
        "Llama-3.2-3B-Instruct",
        "llama-3.2-3b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3.2 1B",
      "name": "Llama 3.2 1B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-09-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 6.642e+22,
      "variants": [
        "Llama-3.2-1B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3.2 11B",
      "name": "Llama 3.2 11B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-09-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-09-24",
      "catalog_releases": [
        {
          "name": "Llama 3.2 11B",
          "date": "2024-09-24",
          "link": "https://ai.meta.com/blog/llama-3-2-connect-2024-vision-edge-mobile-devices/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Multimodal,Vision,Language",
      "parameters": 10600000000.0,
      "compute_flop": 5.79e+23,
      "variants": [
        "Llama-3.2-11B-Vision-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/blog/llama-3-2-connect-2024-vision-edge-mobile-devices/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.5 Pro (Sept 2024)",
      "name": "Gemini 1.5 Pro (Sept 2024)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-09-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 130.45,
      "eci_low": 127.81,
      "eci_high": 132.84,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-1.5-pro-002"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.5 Flash (Sep 2024)",
      "name": "Gemini 1.5 Flash (Sep 2024)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-09-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 126.96,
      "eci_low": 123.87,
      "eci_high": 129.8,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-1.5-flash-002",
        "gemini-1.5-flash-exp-0827"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "qwen2.5-0.5b-instruct",
      "name": "qwen2.5-0.5b-instruct",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-09-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen2.5-0.5b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-7B",
      "name": "Qwen2.5-7B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 8.2188e+23,
      "variants": [
        "qwen2.5-7b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-72B",
      "name": "Qwen2.5-72B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-19",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-09-19",
      "catalog_releases": [
        {
          "name": "Qwen2.5-72B",
          "date": "2024-09-19",
          "link": "https://qwenlm.github.io/blog/qwen2.5/"
        }
      ],
      "eci": 127.21,
      "eci_low": 124.47,
      "eci_high": 129.8,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 72700000000.0,
      "compute_flop": 7.8e+24,
      "variants": [
        "Qwen2.5-72B",
        "Qwen2.5-VL-72B-Instruct",
        "qwen2.5-72b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen2.5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-3B",
      "name": "Qwen2.5-3B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.3372e+23,
      "variants": [
        "qwen2.5-3b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-14B",
      "name": "Qwen2.5-14B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.58760000000001e+24,
      "variants": [
        "qwen2.5-14b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-1.5B",
      "name": "Qwen2.5-1.5B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-19",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.6632e+23,
      "variants": [
        "qwen2.5-1.5b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5 Instruct (72B)",
      "name": "Qwen2.5 Instruct (72B)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-19",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-09-19",
      "catalog_releases": [
        {
          "name": "Qwen2.5 Instruct (72B)",
          "date": "2024-09-19",
          "link": "https://qwenlm.github.io/blog/qwen2.5/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": 72700000000.0,
      "compute_flop": 7.8516e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen2.5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder-3B",
      "name": "Qwen2.5-Coder-3B",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 96.63,
      "eci_low": 82.05,
      "eci_high": 108.97,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen2.5-Coder-3B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder-32B",
      "name": "Qwen2.5-Coder-32B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 118.74,
      "eci_low": 110.04,
      "eci_high": 127.38,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.0725e+24,
      "variants": [
        "Qwen2.5-Coder-32B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder-14B",
      "name": "Qwen2.5-Coder-14B",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 113.72,
      "eci_low": 104.82,
      "eci_high": 122.09,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen2.5-Coder-14B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder-0.5B",
      "name": "Qwen2.5-Coder-0.5B",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 67.95,
      "eci_low": 49.04,
      "eci_high": 83.62,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen2.5-Coder-0.5B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder (7B)",
      "name": "Qwen2.5-Coder (7B)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 107.45,
      "eci_low": 98.13,
      "eci_high": 115.86,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.5113e+23,
      "variants": [
        "Qwen2.5-Coder-7B",
        "Qwen2.5-Coder-7B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-Coder (1.5B)",
      "name": "Qwen2.5-Coder (1.5B)",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 89.01,
      "eci_low": 76.06,
      "eci_high": 99.99,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 5.082e+22,
      "variants": [
        "Qwen2.5-Coder-1.5B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2.5-32B",
      "name": "Qwen2.5-32B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-09-17",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-09-17",
      "catalog_releases": [
        {
          "name": "Qwen2.5-32B",
          "date": "2024-09-17",
          "link": "https://qwenlm.github.io/blog/qwen2.5/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 32500000000.0,
      "compute_flop": 3.51e+24,
      "variants": [
        "Qwen2.5-Coder-32B-Instruct-Q8_0-GGUF",
        "qwen2.5-32b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen2.5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Pixtral 12B",
      "name": "Pixtral 12B",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-09-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Pixtral-12B-2409"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Small v24.09",
      "name": "Mistral Small v24.09",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-09-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-small-2409"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "o1-preview",
      "name": "o1-preview",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-09-12",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-09-12",
      "catalog_releases": [
        {
          "name": "o1-preview",
          "date": "2024-09-12",
          "link": "https://openai.com/index/introducing-openai-o1-preview/"
        }
      ],
      "eci": 133.54,
      "eci_low": 130.97,
      "eci_high": 135.94,
      "access": "Closed weights",
      "domain": "Language,Mathematics,Biology",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "o1-preview-2024-09-12"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/introducing-openai-o1-preview/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "o1-mini",
      "name": "o1-mini",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-09-12",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-09-12",
      "catalog_releases": [
        {
          "name": "o1-mini",
          "date": "2024-09-12",
          "link": "https://openai.com/index/learning-to-reason-with-llms/"
        }
      ],
      "eci": 135.28,
      "eci_low": 133.1,
      "eci_high": 137.29,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "o1-mini-2024-09-12_high",
        "o1-mini-2024-09-12_low",
        "o1-mini-2024-09-12_medium",
        "o1-mini-2024-09-12_unknown"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/learning-to-reason-with-llms/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V2.5 (Sep 2024)",
      "name": "DeepSeek-V2.5 (Sep 2024)",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2024-09-06",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-09-06",
      "catalog_releases": [
        {
          "name": "DeepSeek-V2.5",
          "date": "2024-09-06",
          "link": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": 236000000000.0,
      "compute_flop": 1.7892e+24,
      "variants": [
        "DeepSeek-V2.5"
      ],
      "benchmarks": {},
      "catalog_link": "https://huggingface.co/deepseek-ai/DeepSeek-V2.5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaProteo",
      "name": "AlphaProteo",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-09-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-09-05",
      "catalog_releases": [
        {
          "name": "AlphaProteo",
          "date": "2024-09-05",
          "link": "https://storage.googleapis.com/deepmind-media/DeepMind.com/Blog/alphaproteo-generates-novel-proteins-for-biology-and-health-research/AlphaProteo2024.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Biology",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://storage.googleapis.com/deepmind-media/DeepMind.com/Blog/alphaproteo-generates-novel-proteins-for-biology-and-health-research/AlphaProteo2024.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2-VL-7B-Instruct",
      "name": "Qwen2-VL-7B-Instruct",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-08-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen2-VL-7B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2-VL-72B-Instruct",
      "name": "Qwen2-VL-72B-Instruct",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-08-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen2-VL-72B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-Prover-V1.5",
      "name": "DeepSeek-Prover-V1.5",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2024-08-15",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-Prover-V1.5-Base",
        "DeepSeek-Prover-V1.5-RL",
        "DeepSeek-Prover-V1.5-SFT"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok-2 mini",
      "name": "Grok-2 mini",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2024-08-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "grok-2-mini-0813"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok-2 (Aug 2024)",
      "name": "Grok-2 (Aug 2024)",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2024-08-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.96e+25,
      "variants": [
        "grok-2-0813"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok-2",
      "name": "Grok-2",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2024-08-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-08-13",
      "catalog_releases": [
        {
          "name": "Grok-2",
          "date": "2024-08-13",
          "link": "https://x.ai/blog/grok-2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Vision,Multimodal",
      "parameters": null,
      "compute_flop": 2.96e+25,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://x.ai/blog/grok-2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Table Tennis Agent",
      "name": "Table Tennis Agent",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-08-07",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-08-07",
      "catalog_releases": [
        {
          "name": "Table Tennis Agent",
          "date": "2024-08-07",
          "link": "https://deepmind.google/research/publications/107741/\nhttps://arxiv.org/abs/2408.03906"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Robotics",
      "parameters": 185000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://deepmind.google/research/publications/107741/\nhttps://arxiv.org/abs/2408.03906",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4o (Aug 2024)",
      "name": "GPT-4o (Aug 2024)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-08-06",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-08-06",
      "catalog_releases": [
        {
          "name": "GPT-4o (Aug 2024)",
          "date": "2024-08-06",
          "link": "https://help.openai.com/en/articles/9624314-model-release-notes"
        }
      ],
      "eci": 127.12,
      "eci_low": 124.07,
      "eci_high": 129.86,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Audio,Speech,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4o-2024-08-06"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 26.0,
          "variant": "gpt-4o-2024-08-06",
          "configurations": [
            {
              "variant": "gpt-4o-2024-08-06",
              "score": 26.0
            }
          ]
        }
      },
      "catalog_link": "https://help.openai.com/en/articles/9624314-model-release-notes",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Large 2 (Jul 2024)",
      "name": "Mistral Large 2 (Jul 2024)",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-07-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 126.42,
      "eci_low": 123.24,
      "eci_high": 129.4,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Mistral-Large-Instruct-2407",
        "mistral-large-2407"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Large 2",
      "name": "Mistral Large 2",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-07-24",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-07-24",
      "catalog_releases": [
        {
          "name": "Mistral Large 2",
          "date": "2024-07-24",
          "link": "https://mistral.ai/news/mistral-large-2407/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Language",
      "parameters": 123000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://mistral.ai/news/mistral-large-2407/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3.1-8B",
      "name": "Llama 3.1-8B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-07-23",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 110.78,
      "eci_low": 105.32,
      "eci_high": 115.82,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.224e+24,
      "variants": [
        "Llama-3.1-8B-Instruct",
        "llama3.1:8b-instruct-q8_0"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3.1-70B",
      "name": "Llama 3.1-70B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-07-23",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 124.72,
      "eci_low": 121.58,
      "eci_high": 127.63,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 7.929e+24,
      "variants": [
        "Llama-3.1-70B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3.1-405B",
      "name": "Llama 3.1-405B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-07-23",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-07-23",
      "catalog_releases": [
        {
          "name": "Llama 3.1-405B",
          "date": "2024-07-23",
          "link": "https://ai.meta.com/research/publications/the-llama-3-herd-of-models/"
        }
      ],
      "eci": 127.87,
      "eci_low": 124.95,
      "eci_high": 130.51,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 405000000000.0,
      "compute_flop": 3.8e+25,
      "variants": [
        "Llama-3.1-405B",
        "Llama-3.1-405B-Instruct",
        "llama3.1:405b-instruct-q4_K_M"
      ],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/research/publications/the-llama-3-herd-of-models/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral NeMo",
      "name": "Mistral NeMo",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-07-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 113.81,
      "eci_low": 108.8,
      "eci_high": 118.19,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Mistral-Nemo-Base-2407",
        "Mistral-Nemo-Instruct-2407",
        "open-mistral-nemo-2407"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4o mini",
      "name": "GPT-4o mini",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-07-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-07-18",
      "catalog_releases": [
        {
          "name": "GPT-4o mini",
          "date": "2024-07-18",
          "link": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/"
        }
      ],
      "eci": 123.75,
      "eci_low": 120.5,
      "eci_high": 126.65,
      "access": "Closed weights",
      "domain": "Language,Multimodal,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4o-mini-2024-07-18"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 8.3,
          "variant": "gpt-4o-mini-2024-07-18",
          "configurations": [
            {
              "variant": "gpt-4o-mini-2024-07-18",
              "score": 8.3
            }
          ]
        },
        "mystery_game_puzzles.csv": {
          "score": 12.0,
          "variant": "gpt-4o-mini-2024-07-18",
          "configurations": [
            {
              "variant": "gpt-4o-mini-2024-07-18",
              "score": 12.0
            }
          ]
        }
      },
      "catalog_link": "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "mathstral",
      "name": "mathstral",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-07-16",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-07-16",
      "catalog_releases": [
        {
          "name": "Mathstral",
          "date": "2024-07-16",
          "link": "https://huggingface.co/mistralai/Mathstral-7B-v0.1\nhttps://mistral.ai/news/mathstral"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Mathematics,Language",
      "parameters": 7000000000.0,
      "compute_flop": null,
      "variants": [
        "mathstral"
      ],
      "benchmarks": {},
      "catalog_link": "https://huggingface.co/mistralai/Mathstral-7B-v0.1\nhttps://mistral.ai/news/mathstral",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Codestral Mamba",
      "name": "Codestral Mamba",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-07-16",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "open-codestral-mamba"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V2 (MoE-236B, Jun 2024)",
      "name": "DeepSeek-V2 (MoE-236B, Jun 2024)",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2024-06-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.02e+24,
      "variants": [
        "DeepSeek-V2-Chat-0628"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 2 9B",
      "name": "Gemma 2 9B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-06-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 112.28,
      "eci_low": 106.66,
      "eci_high": 117.21,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 4.32e+23,
      "variants": [
        "gemma-2-9b",
        "gemma-2-9b-it"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 2 27B",
      "name": "Gemma 2 27B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-06-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 118.61,
      "eci_low": 114.48,
      "eci_high": 122.35,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.106e+24,
      "variants": [
        "gemma-2-27b",
        "gemma-2-27b-it",
        "gemma2:27b-instruct-q8_0"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 3.5 Sonnet",
      "name": "Claude 3.5 Sonnet",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2024-06-20",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-06-20",
      "catalog_releases": [
        {
          "name": "Claude 3.5 Sonnet",
          "date": "2024-06-20",
          "link": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf"
        }
      ],
      "eci": 130.0,
      "eci_low": null,
      "eci_high": null,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": 2.700000000000001e+25,
      "variants": [
        "claude-3-5-sonnet-20240620"
      ],
      "benchmarks": {},
      "catalog_link": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-Coder-V2 236B",
      "name": "DeepSeek-Coder-V2 236B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2024-06-17",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-06-17",
      "catalog_releases": [
        {
          "name": "DeepSeek-Coder-V2 236B",
          "date": "2024-06-17",
          "link": "https://github.com/deepseek-ai/DeepSeek-Coder-V2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": 236000000000.0,
      "compute_flop": 1.2852e+24,
      "variants": [
        "DeepSeek-Coder-V2-Base",
        "DeepSeek-Coder-V2-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://github.com/deepseek-ai/DeepSeek-Coder-V2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-Coder-V2-Lite-Instruct",
      "name": "DeepSeek-Coder-V2-Lite-Instruct",
      "lab": "DeepSeek",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-06-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-Coder-V2-Lite-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-Coder-V2-Lite-Base",
      "name": "DeepSeek-Coder-V2-Lite-Base",
      "lab": "DeepSeek",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-06-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 99.31,
      "eci_low": 88.78,
      "eci_high": 108.58,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "DeepSeek-Coder-V2-Lite-Base"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen2-72B",
      "name": "Qwen2-72B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-06-07",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-06-07",
      "catalog_releases": [
        {
          "name": "Qwen2-72B",
          "date": "2024-06-07",
          "link": "https://qwenlm.github.io/blog/qwen2/"
        }
      ],
      "eci": 122.33,
      "eci_low": 118.73,
      "eci_high": 125.52,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 72710000000.0,
      "compute_flop": 3.02e+24,
      "variants": [
        "qwen2-72b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen2/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.5 Flash (May 2024)",
      "name": "Gemini 1.5 Flash (May 2024)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-05-23",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 120.55,
      "eci_low": 116.51,
      "eci_high": 124.09,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-1.5-flash-001",
        "gemini-1.5-flash-0514"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.5 Pro (May 2024)",
      "name": "Gemini 1.5 Pro (May 2024)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-05-14",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 125.37,
      "eci_low": 122.09,
      "eci_high": 128.37,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-1.5-pro-001"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4o (May 2024)",
      "name": "GPT-4o (May 2024)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-05-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 128.13,
      "eci_low": 125.37,
      "eci_high": 130.68,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4o-2024-05-13"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4o",
      "name": "GPT-4o",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-05-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-05-13",
      "catalog_releases": [
        {
          "name": "GPT-4o",
          "date": "2024-05-13",
          "link": "https://openai.com/index/hello-gpt-4o/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Audio,Speech,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/hello-gpt-4o/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaFold 3",
      "name": "AlphaFold 3",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,Isomorphic Labs",
      "lab_inferred": false,
      "date": "2024-05-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-05-08",
      "catalog_releases": [
        {
          "name": "AlphaFold 3",
          "date": "2024-05-08",
          "link": "https://www.nature.com/articles/s41586-024-07487-w"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Biology",
      "parameters": null,
      "compute_flop": 4.1405645e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.nature.com/articles/s41586-024-07487-w",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek-V2 (MoE-236B, May 2024)",
      "name": "DeepSeek-V2 (MoE-236B, May 2024)",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2024-05-07",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 120.76,
      "eci_low": 113.88,
      "eci_high": 127.62,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.02e+24,
      "variants": [
        "DeepSeek-V2"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GenCast",
      "name": "GenCast",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-05-01",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-05-01",
      "catalog_releases": [
        {
          "name": "GenCast",
          "date": "2024-05-01",
          "link": "https://arxiv.org/abs/2312.15796"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Earth science",
      "parameters": null,
      "compute_flop": 8.169984e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2312.15796",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen1.5-110B",
      "name": "Qwen1.5-110B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-04-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen1.5-110b-chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "llama3-llava-next-8b",
      "name": "llama3-llava-next-8b",
      "lab": "Meta",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-04-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "llama3-llava-next-8b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3-8B",
      "name": "Llama 3-8B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-04-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 109.0,
      "eci_low": 103.32,
      "eci_high": 113.95,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 7.2e+23,
      "variants": [
        "Meta-Llama-3-8B",
        "Meta-Llama-3-8B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 3-70B",
      "name": "Llama 3-70B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2024-04-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-04-18",
      "catalog_releases": [
        {
          "name": "Llama 3-70B",
          "date": "2024-04-18",
          "link": "https://ai.meta.com/blog/meta-llama-3/"
        }
      ],
      "eci": 121.34,
      "eci_low": 117.67,
      "eci_high": 124.75,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 70000000000.0,
      "compute_flop": 7.861e+24,
      "variants": [
        "Meta-Llama-3-70B",
        "Meta-Llama-3-70B-Instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/blog/meta-llama-3/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mixtral 8x22B",
      "name": "Mixtral 8x22B",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-04-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 118.66,
      "eci_low": 114.34,
      "eci_high": 122.54,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.34e+24,
      "variants": [
        "Mixtral-8x22B-Instruct-v0.1",
        "Mixtral-8x22B-v0.1",
        "open-mixtral-8x22b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "CodeQwen1.5-7B",
      "name": "CodeQwen1.5-7B",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-04-15",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 74.53,
      "eci_low": 58.12,
      "eci_high": 88.3,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "CodeQwen1.5-7B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4 Turbo (Apr 2024)",
      "name": "GPT-4 Turbo (Apr 2024)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-04-09",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-04-09",
      "catalog_releases": [
        {
          "name": "GPT-4 Turbo (Apr 2024)",
          "date": "2024-04-09",
          "link": "https://x.com/OpenAI/status/1778574613813006610"
        }
      ],
      "eci": 125.82,
      "eci_low": 122.84,
      "eci_high": 128.62,
      "access": "Closed weights",
      "domain": "Multimodal,Vision,Language,Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4-turbo-2024-04-09"
      ],
      "benchmarks": {},
      "catalog_link": "https://x.com/OpenAI/status/1778574613813006610",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 3 Haiku",
      "name": "Claude 3 Haiku",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2024-03-07",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 117.79,
      "eci_low": 113.6,
      "eci_high": 121.58,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-3-haiku-20240307"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 3 Sonnet",
      "name": "Claude 3 Sonnet",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2024-02-29",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-03-04",
      "catalog_releases": [
        {
          "name": "Claude 3 Sonnet",
          "date": "2024-03-04",
          "link": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf"
        }
      ],
      "eci": 119.65,
      "eci_low": 115.71,
      "eci_high": 123.23,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-3-sonnet-20240229"
      ],
      "benchmarks": {},
      "catalog_link": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 3 Opus",
      "name": "Claude 3 Opus",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2024-02-29",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-03-04",
      "catalog_releases": [
        {
          "name": "Claude 3 Opus",
          "date": "2024-03-04",
          "link": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf"
        }
      ],
      "eci": 125.58,
      "eci_low": 122.65,
      "eci_high": 128.17,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-3-opus-20240229"
      ],
      "benchmarks": {
        "simpleqa_verified.csv": {
          "score": 12.6,
          "variant": "claude-3-opus-20240229",
          "configurations": [
            {
              "variant": "claude-3-opus-20240229",
              "score": 12.6
            }
          ]
        }
      },
      "catalog_link": "https://www-cdn.anthropic.com/de8ba9b01c9ab7cbabf5c33b80b7bbc618857627/Model_Card_Claude_3.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "mistral-small-2402",
      "name": "mistral-small-2402",
      "lab": "Mistral",
      "organization": null,
      "lab_inferred": true,
      "date": "2024-02-26",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-small-2402"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Large",
      "name": "Mistral Large",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2024-02-26",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-02-26",
      "catalog_releases": [
        {
          "name": "Mistral Large",
          "date": "2024-02-26",
          "link": "https://mistral.ai/news/mistral-large/"
        }
      ],
      "eci": 119.96,
      "eci_low": 115.99,
      "eci_high": 123.45,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-large-2402"
      ],
      "benchmarks": {},
      "catalog_link": "https://mistral.ai/news/mistral-large/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 7B",
      "name": "Gemma 7B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-02-21",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 102.87,
      "eci_low": 94.96,
      "eci_high": 109.59,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.07e+23,
      "variants": [
        "gemma-7b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemma 2B",
      "name": "Gemma 2B",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-02-21",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 75.36,
      "eci_low": 60.81,
      "eci_high": 87.85,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 4.5115822e+22,
      "variants": [
        "gemma-2b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Sora",
      "name": "Sora",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-02-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-02-15",
      "catalog_releases": [
        {
          "name": "Sora",
          "date": "2024-02-15",
          "link": "https://openai.com/index/video-generation-models-as-world-simulators/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Video,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/index/video-generation-models-as-world-simulators/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.5 Pro (Feb 2024)",
      "name": "Gemini 1.5 Pro (Feb 2024)",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-02-15",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-1.5-pro-001-feb24"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.5 Pro",
      "name": "Gemini 1.5 Pro",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-02-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-02-15",
      "catalog_releases": [
        {
          "name": "Gemini 1.5 Pro",
          "date": "2024-02-15",
          "link": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Multimodal",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeekMath 7B",
      "name": "DeepSeekMath 7B",
      "lab": "DeepSeek",
      "organization": "DeepSeek,Tsinghua University,Peking University",
      "lab_inferred": false,
      "date": "2024-02-05",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.014e+23,
      "variants": [
        "deepseek-math-7b-base"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen1.5-7B",
      "name": "Qwen1.5-7B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-02-04",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.68e+23,
      "variants": [
        "Qwen1.5-7B",
        "qwen1.5-7b-chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen1.5-72B",
      "name": "Qwen1.5-72B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-02-04",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-02-04",
      "catalog_releases": [
        {
          "name": "Qwen1.5-72B",
          "date": "2024-02-04",
          "link": "https://qwenlm.github.io/blog/qwen1.5/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": 72000000000.0,
      "compute_flop": 1.3e+24,
      "variants": [
        "qwen1.5-72b-chat"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen1.5/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen1.5-32B",
      "name": "Qwen1.5-32B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-02-04",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "qwen1.5-32B",
        "qwen1.5-32b-chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen1.5-14B",
      "name": "Qwen1.5-14B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-02-04",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.36e+23,
      "variants": [
        "qwen1.5-14B",
        "qwen1.5-14b-chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "text-embedding-3-small",
      "name": "text-embedding-3-small",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-01-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "text-embedding-3-small"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "text-embedding-3-large",
      "name": "text-embedding-3-large",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-01-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "text-embedding-3-large"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4 Turbo (Nov 2023)",
      "name": "GPT-4 Turbo (Nov 2023)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-01-25",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-11-06",
      "catalog_releases": [
        {
          "name": "GPT-4 Turbo (Nov 2023)",
          "date": "2023-11-06",
          "link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday"
        }
      ],
      "eci": 124.75,
      "eci_low": 121.47,
      "eci_high": 127.68,
      "access": "Closed weights",
      "domain": "Multimodal,Vision,Language,Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4-0125-preview",
        "gpt-4-1106-preview",
        "gpt-4-turbo"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/blog/new-models-and-developer-products-announced-at-devday",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3.5 Turbo (Jan 2024)",
      "name": "GPT-3.5 Turbo (Jan 2024)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2024-01-25",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 112.77,
      "eci_low": 107.84,
      "eci_high": 117.15,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-3.5-turbo-0125"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 3.0,
          "variant": "gpt-3.5-turbo-0125",
          "configurations": [
            {
              "variant": "gpt-3.5-turbo-0125",
              "score": 3.0
            }
          ]
        }
      },
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-VL-Max",
      "name": "Qwen-VL-Max",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2024-01-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2024-01-25",
      "catalog_releases": [
        {
          "name": "Qwen-VL-Max",
          "date": "2024-01-25",
          "link": "https://qwenlm.github.io/blog/qwen-vl/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Vision",
      "parameters": 7000000000.0,
      "compute_flop": null,
      "variants": [
        "Qwen-VL-Max"
      ],
      "benchmarks": {},
      "catalog_link": "https://qwenlm.github.io/blog/qwen-vl/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaGeometry",
      "name": "AlphaGeometry",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,New York University (NYU)",
      "lab_inferred": false,
      "date": "2024-01-17",
      "date_basis": "Catalog publication date",
      "catalog_date": "2024-01-17",
      "catalog_releases": [
        {
          "name": "AlphaGeometry",
          "date": "2024-01-17",
          "link": "https://www.nature.com/articles/s41586-023-06747-5"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Mathematics",
      "parameters": 151000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.nature.com/articles/s41586-023-06747-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.0 Pro Vision",
      "name": "Gemini 1.0 Pro Vision",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2024-01-04",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-1.0-pro-vision"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GQA-8-XXL",
      "name": "GQA-8-XXL",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2023-12-23",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-12-23",
      "catalog_releases": [
        {
          "name": "GQA-8-XXL",
          "date": "2023-12-23",
          "link": "https://arxiv.org/abs/2305.13245"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 11000000000.0,
      "compute_flop": 3.4912896e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2305.13245",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini Nano-2",
      "name": "Gemini Nano-2",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-12-19",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-12-19",
      "catalog_releases": [
        {
          "name": "Gemini Nano-2",
          "date": "2023-12-19",
          "link": "https://arxiv.org/abs/2312.11805"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Language,Vision,Audio",
      "parameters": 3250000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2312.11805",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini Nano-1",
      "name": "Gemini Nano-1",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-12-19",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-12-19",
      "catalog_releases": [
        {
          "name": "Gemini Nano-1",
          "date": "2023-12-19",
          "link": "https://arxiv.org/abs/2312.11805"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Vision,Audio",
      "parameters": 1800000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2312.11805",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "FunSearch",
      "name": "FunSearch",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-12-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-12-14",
      "catalog_releases": [
        {
          "name": "FunSearch",
          "date": "2023-12-14",
          "link": "https://www.nature.com/articles/s41586-023-06924-6\nhttps://deepmind.google/discover/blog/funsearch-making-new-discoveries-in-mathematical-sciences-using-large-language-models/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language,Search",
      "parameters": 15000000000.0,
      "compute_flop": 3.87e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.nature.com/articles/s41586-023-06924-6\nhttps://deepmind.google/discover/blog/funsearch-making-new-discoveries-in-mathematical-sciences-using-large-language-models/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.0 Pro",
      "name": "Gemini 1.0 Pro",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-12-13",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-12-06",
      "catalog_releases": [
        {
          "name": "Gemini 1.0 Pro",
          "date": "2023-12-06",
          "link": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gemini-1.0-pro-001"
      ],
      "benchmarks": {},
      "catalog_link": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mixtral 8x7B",
      "name": "Mixtral 8x7B",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2023-12-11",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-12-11",
      "catalog_releases": [
        {
          "name": "Mixtral 8x7B",
          "date": "2023-12-11",
          "link": "https://mistral.ai/news/mixtral-of-experts/,"
        }
      ],
      "eci": 113.1,
      "eci_low": 108.13,
      "eci_high": 117.48,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 46700000000.0,
      "compute_flop": 7.74e+23,
      "variants": [
        "Mixtral-8x7B-Instruct-v0.1",
        "Mixtral-8x7B-v0.1",
        "open-mixtral-8x7b"
      ],
      "benchmarks": {},
      "catalog_link": "https://mistral.ai/news/mixtral-of-experts/,",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral Medium",
      "name": "Mistral Medium",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2023-12-11",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "mistral-medium-2312"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral 7B v0.2",
      "name": "Mistral 7B v0.2",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2023-12-11",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Mistral-7B-Instruct-v0.2"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama Guard",
      "name": "Llama Guard",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-12-07",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-12-07",
      "catalog_releases": [
        {
          "name": "Llama Guard",
          "date": "2023-12-07",
          "link": "https://arxiv.org/abs/2312.06674"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": 7000000000.0,
      "compute_flop": 1.6e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2312.06674",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gemini 1.0 Ultra",
      "name": "Gemini 1.0 Ultra",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-12-06",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-12-06",
      "catalog_releases": [
        {
          "name": "Gemini 1.0 Ultra",
          "date": "2023-12-06",
          "link": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Language,Vision",
      "parameters": null,
      "compute_flop": 5.0000000001e+25,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-72B",
      "name": "Qwen-72B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2023-11-30",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-11-30",
      "catalog_releases": [
        {
          "name": "Qwen-72B",
          "date": "2023-11-30",
          "link": "https://huggingface.co/Qwen/Qwen-72B"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language",
      "parameters": 72000000000.0,
      "compute_flop": 1.3e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://huggingface.co/Qwen/Qwen-72B",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-1_8B",
      "name": "Qwen-1_8B",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2023-11-30",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 59.23,
      "eci_low": 40.43,
      "eci_high": 75.43,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen-1_8B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GNoME for crystal discovery",
      "name": "GNoME for crystal discovery",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-11-29",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-11-29",
      "catalog_releases": [
        {
          "name": "GNoME for crystal discovery",
          "date": "2023-11-29",
          "link": "https://www.nature.com/articles/s41586-023-06735-9"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Materials science",
      "parameters": 16240000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.nature.com/articles/s41586-023-06735-9",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek LLM 7B",
      "name": "DeepSeek LLM 7B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2023-11-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 8.4e+22,
      "variants": [
        "deepseek-llm-7b-base",
        "deepseek-llm-7b-chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek LLM 67B",
      "name": "DeepSeek LLM 67B",
      "lab": "DeepSeek",
      "organization": "DeepSeek",
      "lab_inferred": false,
      "date": "2023-11-29",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 8.04e+23,
      "variants": [
        "deepseek-llm-67b-base",
        "deepseek-llm-67b-chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 2.1",
      "name": "Claude 2.1",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2023-11-21",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-11-21",
      "catalog_releases": [
        {
          "name": "Claude 2.1",
          "date": "2023-11-21",
          "link": "https://www.anthropic.com/index/claude-2-1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-2.1"
      ],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/index/claude-2-1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-Audio-Chat",
      "name": "Qwen-Audio-Chat",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2023-11-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-11-14",
      "catalog_releases": [
        {
          "name": "Qwen-Audio-Chat",
          "date": "2023-11-14",
          "link": "https://arxiv.org/abs/2311.07919"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Language,Speech,Audio",
      "parameters": 8460000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2311.07919",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GraphCast",
      "name": "GraphCast",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-11-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-11-14",
      "catalog_releases": [
        {
          "name": "GraphCast",
          "date": "2023-11-14",
          "link": "https://www.science.org/doi/epdf/10.1126/science.adi2336\n\nhttps://arxiv.org/abs/2212.12794"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Earth science",
      "parameters": null,
      "compute_flop": 2.1000000000000002e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.science.org/doi/epdf/10.1126/science.adi2336\n\nhttps://arxiv.org/abs/2212.12794",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "MultiBand Diffusion",
      "name": "MultiBand Diffusion",
      "lab": "Meta",
      "organization": "Meta AI,Hebrew University of Jerusalem,LORIA",
      "lab_inferred": false,
      "date": "2023-11-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-11-08",
      "catalog_releases": [
        {
          "name": "MultiBand Diffusion",
          "date": "2023-11-08",
          "link": "https://arxiv.org/abs/2308.02560"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Audio,Speech",
      "parameters": null,
      "compute_flop": 2.6e+19,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2308.02560",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "mPLUG-Owl2",
      "name": "mPLUG-Owl2",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2023-11-07",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-11-07",
      "catalog_releases": [
        {
          "name": "mPLUG-Owl2",
          "date": "2023-11-07",
          "link": "https://arxiv.org/abs/2311.04257"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision,Language,Multimodal",
      "parameters": 7120000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2311.04257",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4V",
      "name": "GPT-4V",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-11-06",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-09-25",
      "catalog_releases": [
        {
          "name": "GPT-4V",
          "date": "2023-09-25",
          "link": "https://cdn.openai.com/papers/GPTV_System_Card.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Multimodal,Vision,Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-4-1106-vision-preview"
      ],
      "benchmarks": {},
      "catalog_link": "https://cdn.openai.com/papers/GPTV_System_Card.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3.5 Turbo (Nov 2023)",
      "name": "GPT-3.5 Turbo (Nov 2023)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-11-06",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 112.57,
      "eci_low": 107.25,
      "eci_high": 117.26,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-3.5-turbo-1106"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Grok-1",
      "name": "Grok-1",
      "lab": "xAI",
      "organization": "xAI",
      "lab_inferred": false,
      "date": "2023-11-04",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-11-04",
      "catalog_releases": [
        {
          "name": "Grok-1",
          "date": "2023-11-04",
          "link": "https://x.ai/model-card/,"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 314000000000.0,
      "compute_flop": 2.90000000001e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://x.ai/model-card/,",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "RT-Trajectory",
      "name": "RT-Trajectory",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,University of California San Diego,Stanford University",
      "lab_inferred": false,
      "date": "2023-11-03",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-11-03",
      "catalog_releases": [
        {
          "name": "RT-Trajectory",
          "date": "2023-11-03",
          "link": "https://arxiv.org/abs/2311.01977"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Robotics",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2311.01977",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek Coder 6.7B",
      "name": "DeepSeek Coder 6.7B",
      "lab": "DeepSeek",
      "organization": "DeepSeek,Peking University",
      "lab_inferred": false,
      "date": "2023-11-02",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 67.02,
      "eci_low": 48.31,
      "eci_high": 82.73,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 8.04e+22,
      "variants": [
        "deepseek-coder-6.7b-base",
        "deepseek-coder-6.7b-instruct",
        "deepseek-coder-7b-base-v1.5"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek Coder 33B",
      "name": "DeepSeek Coder 33B",
      "lab": "DeepSeek",
      "organization": "DeepSeek,Peking University",
      "lab_inferred": false,
      "date": "2023-11-02",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 75.93,
      "eci_low": 59.8,
      "eci_high": 89.43,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.96e+23,
      "variants": [
        "deepseek-coder-33b-base",
        "deepseek-coder-33b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepSeek Coder 1.3B",
      "name": "DeepSeek Coder 1.3B",
      "lab": "DeepSeek",
      "organization": "DeepSeek,Peking University",
      "lab_inferred": false,
      "date": "2023-11-02",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.56e+22,
      "variants": [
        "deepseek-coder-1.3b-base",
        "deepseek-coder-1.3b-instruct"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DALL·E 3",
      "name": "DALL·E 3",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-10-19",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-10-19",
      "catalog_releases": [
        {
          "name": "DALL·E 3",
          "date": "2023-10-19",
          "link": "https://cdn.openai.com/papers/dall-e-3.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://cdn.openai.com/papers/dall-e-3.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLI-3",
      "name": "PaLI-3",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,Google Research,Google Cloud",
      "lab_inferred": false,
      "date": "2023-10-17",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-10-17",
      "catalog_releases": [
        {
          "name": "PaLI-3",
          "date": "2023-10-17",
          "link": "https://arxiv.org/abs/2310.09199"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Language,Vision",
      "parameters": 5000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2310.09199",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "RT-2-X",
      "name": "RT-2-X",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-10-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-10-13",
      "catalog_releases": [
        {
          "name": "RT-2-X",
          "date": "2023-10-13",
          "link": "https://arxiv.org/abs/2310.08864"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Robotics",
      "parameters": 55000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2310.08864",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-7B",
      "name": "Qwen-7B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2023-09-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 88.87,
      "eci_low": 77.54,
      "eci_high": 98.62,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.01e+23,
      "variants": [
        "Qwen-7B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral 7B v0.3",
      "name": "Mistral 7B v0.3",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2023-09-27",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Mistral-7B-Instruct-v0.3",
        "Mistral-7B-v0.3",
        "open-mistral-7b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mistral 7B v0.1",
      "name": "Mistral 7B v0.1",
      "lab": "Mistral",
      "organization": "Mistral AI",
      "lab_inferred": false,
      "date": "2023-09-27",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 102.56,
      "eci_low": 94.84,
      "eci_high": 109.26,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Mistral-7B-Instruct-v0.1",
        "Mistral-7B-v0.1"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-14B",
      "name": "Qwen-14B",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2023-09-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 101.61,
      "eci_low": 92.75,
      "eci_high": 109.54,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.5e+23,
      "variants": [
        "Qwen-14B",
        "Qwen-14B-Chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaMissense",
      "name": "AlphaMissense",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-09-22",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-09-22",
      "catalog_releases": [
        {
          "name": "AlphaMissense",
          "date": "2023-09-22",
          "link": "https://www.science.org/doi/10.1126/science.adg7492"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Biology",
      "parameters": 93000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.science.org/doi/10.1126/science.adg7492",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3.5 Turbo Instruct",
      "name": "GPT-3.5 Turbo Instruct",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-09-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-09-28",
      "catalog_releases": [
        {
          "name": "GPT-3.5 Turbo Instruct",
          "date": "2023-09-28",
          "link": "https://platform.openai.com/docs/models/gpt-3.5-turbo"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-3.5-turbo-instruct"
      ],
      "benchmarks": {},
      "catalog_link": "https://platform.openai.com/docs/models/gpt-3.5-turbo",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-VL",
      "name": "Qwen-VL",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2023-08-24",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-08-24",
      "catalog_releases": [
        {
          "name": "Qwen-VL",
          "date": "2023-08-24",
          "link": "https://arxiv.org/abs/2308.12966"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (restricted use)",
      "domain": "Multimodal,Language,Vision",
      "parameters": 9600000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2308.12966",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3 XL (babbage-002)",
      "name": "GPT-3 XL (babbage-002)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-08-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.38e+21,
      "variants": [
        "babbage-002"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Qwen-VL-Chat",
      "name": "Qwen-VL-Chat",
      "lab": "Alibaba",
      "organization": null,
      "lab_inferred": true,
      "date": "2023-08-20",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "Qwen-VL-Chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude Instant",
      "name": "Claude Instant",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2023-08-09",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 114.0,
      "eci_low": 107.19,
      "eci_high": 120.14,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-instant-1.1",
        "claude-instant-1.2"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "RT-2",
      "name": "RT-2",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-07-28",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-07-28",
      "catalog_releases": [
        {
          "name": "RT-2",
          "date": "2023-07-28",
          "link": "https://arxiv.org/abs/2307.15818"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Robotics",
      "parameters": 55000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2307.15818",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AudioLM",
      "name": "AudioLM",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2023-07-26",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-07-26",
      "catalog_releases": [
        {
          "name": "AudioLM",
          "date": "2023-07-26",
          "link": "https://arxiv.org/abs/2209.03143"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Audio",
      "parameters": 1500000000.0,
      "compute_flop": 3.9e+18,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2209.03143",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 2-7B",
      "name": "Llama 2-7B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-07-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-07-18",
      "catalog_releases": [
        {
          "name": "Llama 2-7B",
          "date": "2023-07-18",
          "link": "https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/"
        }
      ],
      "eci": 82.8,
      "eci_low": 70.34,
      "eci_high": 93.04,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 7000000000.0,
      "compute_flop": 8.4e+22,
      "variants": [
        "Llama-2-7b",
        "llama-2-7b-chat"
      ],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 2-70B",
      "name": "Llama 2-70B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-07-18",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-07-18",
      "catalog_releases": [
        {
          "name": "Llama 2-70B",
          "date": "2023-07-18",
          "link": "https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/\nhttps://arxiv.org/abs/2307.09288"
        }
      ],
      "eci": 105.64,
      "eci_low": 99.62,
      "eci_high": 110.86,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 70000000000.0,
      "compute_flop": 8.1e+23,
      "variants": [
        "/HFEndpoint/Llama 2-70B",
        "Llama-2-70b-chat",
        "Llama-2-70b-chat-hf",
        "Llama-2-70b-hf "
      ],
      "benchmarks": {},
      "catalog_link": "https://ai.meta.com/research/publications/llama-2-open-foundation-and-fine-tuned-chat-models/\nhttps://arxiv.org/abs/2307.09288",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 2-34B",
      "name": "Llama 2-34B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-07-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 93.69,
      "eci_low": 83.73,
      "eci_high": 102.35,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 4.08e+23,
      "variants": [
        "Llama-2-34b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Llama 2-13B",
      "name": "Llama 2-13B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-07-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 96.28,
      "eci_low": 87.19,
      "eci_high": 104.31,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.6e+23,
      "variants": [
        "Llama-2-13b",
        "Llama-2-13b-chat"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 2",
      "name": "Claude 2",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2023-07-11",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-07-11",
      "catalog_releases": [
        {
          "name": "Claude 2",
          "date": "2023-07-11",
          "link": "https://www.anthropic.com/index/claude-2,"
        }
      ],
      "eci": 117.12,
      "eci_low": 112.61,
      "eci_high": 121.05,
      "access": "Closed weights",
      "domain": "Language",
      "parameters": null,
      "compute_flop": 3.866e+24,
      "variants": [
        "claude-2.0"
      ],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/index/claude-2,",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4 (Jun 2023)",
      "name": "GPT-4 (Jun 2023)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-06-13",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-06-13",
      "catalog_releases": [
        {
          "name": "GPT-4 (Jun 2023)",
          "date": "2023-06-13",
          "link": "https://arxiv.org/abs/2303.08774"
        }
      ],
      "eci": 119.82,
      "eci_low": 115.94,
      "eci_high": 123.33,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": 1800000000000.0,
      "compute_flop": 2.1e+25,
      "variants": [
        "gpt-4-0613",
        "gpt-4-32k-0613"
      ],
      "benchmarks": {
        "mystery_game_puzzles.csv": {
          "score": 12.0,
          "variant": "gpt-4-0613",
          "configurations": [
            {
              "variant": "gpt-4-0613",
              "score": 12.0
            }
          ]
        }
      },
      "catalog_link": "https://arxiv.org/abs/2303.08774",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3.5 Turbo (Jun 2023)",
      "name": "GPT-3.5 Turbo (Jun 2023)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-06-13",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 105.84,
      "eci_low": 97.72,
      "eci_high": 113.26,
      "access": "Closed weights",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-3.5-turbo-0613",
        "gpt-3.5-turbo-16k-0613"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3.5 Turbo",
      "name": "GPT-3.5 Turbo",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-06-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-06-13",
      "catalog_releases": [
        {
          "name": "GPT-3.5 Turbo",
          "date": "2023-06-13",
          "link": "https://platform.openai.com/docs/models/gpt-3.5-turbo"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://platform.openai.com/docs/models/gpt-3.5-turbo",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "MusicGen",
      "name": "MusicGen",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-06-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-06-08",
      "catalog_releases": [
        {
          "name": "MusicGen",
          "date": "2023-06-08",
          "link": "https://arxiv.org/abs/2306.05284"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Audio",
      "parameters": 3359000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2306.05284",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLI-X",
      "name": "PaLI-X",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2023-05-29",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-05-29",
      "catalog_releases": [
        {
          "name": "PaLI-X",
          "date": "2023-05-29",
          "link": "https://arxiv.org/abs/2305.18565"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Language,Vision,Video",
      "parameters": 55000000000.0,
      "compute_flop": 5.6e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2305.18565",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ONE-PEACE",
      "name": "ONE-PEACE",
      "lab": "Alibaba",
      "organization": "Alibaba,Huazhong University of Science and Technology",
      "lab_inferred": false,
      "date": "2023-05-18",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-05-18",
      "catalog_releases": [
        {
          "name": "ONE-PEACE",
          "date": "2023-05-18",
          "link": "https://arxiv.org/abs/2305.11172v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Multimodal,Vision,Speech,Language",
      "parameters": 4000000000.0,
      "compute_flop": 1.8e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2305.11172v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLM 2-S",
      "name": "PaLM 2-S",
      "lab": "Google DeepMind",
      "organization": null,
      "lab_inferred": true,
      "date": "2023-05-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 93.62,
      "eci_low": 80.24,
      "eci_high": 105.75,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "PaLM 2-S"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLM 2-M",
      "name": "PaLM 2-M",
      "lab": "Google DeepMind",
      "organization": null,
      "lab_inferred": true,
      "date": "2023-05-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 97.77,
      "eci_low": 84.79,
      "eci_high": 109.73,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "PaLM 2-M"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLM 2-L",
      "name": "PaLM 2-L",
      "lab": "Google DeepMind",
      "organization": null,
      "lab_inferred": true,
      "date": "2023-05-17",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 113.71,
      "eci_low": 98.69,
      "eci_high": 129.5,
      "access": "Other",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "PaLM 2-L"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Med-PaLM 2",
      "name": "Med-PaLM 2",
      "lab": "Google DeepMind",
      "organization": "Google Research,DeepMind",
      "lab_inferred": false,
      "date": "2023-05-16",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-05-16",
      "catalog_releases": [
        {
          "name": "Med-PaLM 2",
          "date": "2023-05-16",
          "link": "https://arxiv.org/abs/2305.09617"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Medicine,Language",
      "parameters": 340000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2305.09617",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLM 2",
      "name": "PaLM 2",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2023-05-10",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-05-10",
      "catalog_releases": [
        {
          "name": "PaLM 2",
          "date": "2023-05-10",
          "link": "https://arxiv.org/abs/2305.10403"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": 340000000000.0,
      "compute_flop": 7.34e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2305.10403",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ImageBind",
      "name": "ImageBind",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-05-09",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-05-09",
      "catalog_releases": [
        {
          "name": "ImageBind",
          "date": "2023-05-09",
          "link": "https://arxiv.org/abs/2305.05665,"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Multimodal,Vision,Audio,Language,Image generation,Speech",
      "parameters": 932000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2305.05665,",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Agile Soccer Robot",
      "name": "Agile Soccer Robot",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-04-26",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-04-26",
      "catalog_releases": [
        {
          "name": "Agile Soccer Robot",
          "date": "2023-04-26",
          "link": "https://arxiv.org/abs/2304.13653"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Robotics",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2304.13653",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude 1.3",
      "name": "Claude 1.3",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2023-04-18",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "claude-1.3"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DINOv2",
      "name": "DINOv2",
      "lab": "Meta",
      "organization": "Facebook AI Research,INRIA",
      "lab_inferred": false,
      "date": "2023-04-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-04-14",
      "catalog_releases": [
        {
          "name": "DINOv2",
          "date": "2023-04-14",
          "link": "https://arxiv.org/abs/2304.07193"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 1140000000.0,
      "compute_flop": 7.41851136e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2304.07193",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Incoder-6.7B",
      "name": "Incoder-6.7B",
      "lab": "Meta",
      "organization": "Facebook AI Research,University of Washington,University of California (UC) Berkeley,Carnegie Mellon University (CMU),Toyota Technological Institute at Chicago",
      "lab_inferred": false,
      "date": "2023-04-09",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-04-09",
      "catalog_releases": [
        {
          "name": "Incoder-6.7B",
          "date": "2023-04-09",
          "link": "https://arxiv.org/abs/2204.05999"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Language",
      "parameters": 6700000000.0,
      "compute_flop": 3.00001e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2204.05999",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Segment Anything Model",
      "name": "Segment Anything Model",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-04-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-04-05",
      "catalog_releases": [
        {
          "name": "Segment Anything Model",
          "date": "2023-04-05",
          "link": "https://arxiv.org/abs/2304.02643"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 636000000.0,
      "compute_flop": 7.8e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2304.02643",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "SigLIP 400M",
      "name": "SigLIP 400M",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind",
      "lab_inferred": false,
      "date": "2023-03-27",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-03-27",
      "catalog_releases": [
        {
          "name": "SigLIP 400M",
          "date": "2023-03-27",
          "link": "https://arxiv.org/abs/2303.15343"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 400000000.0,
      "compute_flop": 4.9467301e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2303.15343",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-4 (Mar 2023)",
      "name": "GPT-4 (Mar 2023)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-03-14",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-03-15",
      "catalog_releases": [
        {
          "name": "GPT-4 (Mar 2023)",
          "date": "2023-03-15",
          "link": "https://arxiv.org/abs/2303.08774"
        }
      ],
      "eci": 121.58,
      "eci_low": 117.72,
      "eci_high": 125.02,
      "access": "Closed weights",
      "domain": "Multimodal,Language,Vision",
      "parameters": 1800000000000.0,
      "compute_flop": 2.1e+25,
      "variants": [
        "gpt-4-0314",
        "gpt-4-32k-0314"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2303.08774",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Claude",
      "name": "Claude",
      "lab": "Anthropic",
      "organization": "Anthropic",
      "lab_inferred": false,
      "date": "2023-03-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-03-14",
      "catalog_releases": [
        {
          "name": "Claude",
          "date": "2023-03-14",
          "link": "https://www.anthropic.com/index/introducing-claude"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.anthropic.com/index/introducing-claude",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLM-E",
      "name": "PaLM-E",
      "lab": "Google DeepMind",
      "organization": "Google,TU Berlin",
      "lab_inferred": false,
      "date": "2023-03-06",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-03-06",
      "catalog_releases": [
        {
          "name": "PaLM-E",
          "date": "2023-03-06",
          "link": "https://arxiv.org/abs/2303.03378"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Robotics,Vision,Language",
      "parameters": 562000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2303.03378",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AudioGen",
      "name": "AudioGen",
      "lab": "Meta",
      "organization": "Meta AI,Hebrew University of Jerusalem",
      "lab_inferred": false,
      "date": "2023-03-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-03-05",
      "catalog_releases": [
        {
          "name": "AudioGen",
          "date": "2023-03-05",
          "link": "https://arxiv.org/abs/2209.15352"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Audio",
      "parameters": 1000000000.0,
      "compute_flop": 9.5e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2209.15352",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3.5 Turbo (Mar 2023)",
      "name": "GPT-3.5 Turbo (Mar 2023)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2023-03-01",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "gpt-3.5-turbo-0301"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "LLaMA-7B",
      "name": "LLaMA-7B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 76.96,
      "eci_low": 62.98,
      "eci_high": 88.78,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 4.00000001e+22,
      "variants": [
        "LLaMA-7B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "LLaMA-65B",
      "name": "LLaMA-65B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": "2023-02-24",
      "catalog_releases": [
        {
          "name": "LLaMA-65B",
          "date": "2023-02-24",
          "link": "https://arxiv.org/abs/2302.13971"
        }
      ],
      "eci": 101.38,
      "eci_low": 92.98,
      "eci_high": 108.66,
      "access": "Open weights",
      "domain": "Language",
      "parameters": 65200000000.0,
      "compute_flop": 5.5e+23,
      "variants": [
        "LLaMA-65B"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2302.13971",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "LLaMA-33B",
      "name": "LLaMA-33B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 96.37,
      "eci_low": 87.05,
      "eci_high": 104.65,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.7300000000001e+23,
      "variants": [
        "LLaMA-33B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "LLaMA-13B",
      "name": "LLaMA-13B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2023-02-24",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": 84.95,
      "eci_low": 72.99,
      "eci_high": 95.23,
      "access": "Open weights",
      "domain": null,
      "parameters": null,
      "compute_flop": 7.8e+22,
      "variants": [
        "LLaMA-13B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "BASIC-L + Lion",
      "name": "BASIC-L + Lion",
      "lab": "Google DeepMind",
      "organization": "Google,University of California Los Angeles (UCLA)",
      "lab_inferred": false,
      "date": "2023-02-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-02-13",
      "catalog_releases": [
        {
          "name": "BASIC-L + Lion",
          "date": "2023-02-13",
          "link": "https://arxiv.org/abs/2302.06675v4"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Vision",
      "parameters": 3070000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2302.06675v4",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ViT-22B",
      "name": "ViT-22B",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2023-02-10",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-02-10",
      "catalog_releases": [
        {
          "name": "ViT-22B",
          "date": "2023-02-10",
          "link": "https://arxiv.org/abs/2302.05442v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Vision",
      "parameters": 21743000000.0,
      "compute_flop": 1.93248e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2302.05442v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "MusicLM",
      "name": "MusicLM",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2023-01-26",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-01-26",
      "catalog_releases": [
        {
          "name": "MusicLM",
          "date": "2023-01-26",
          "link": "https://arxiv.org/abs/2301.11325"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Audio",
      "parameters": 860000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2301.11325",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DreamerV3",
      "name": "DreamerV3",
      "lab": "Google DeepMind",
      "organization": "DeepMind,University of Toronto",
      "lab_inferred": false,
      "date": "2023-01-10",
      "date_basis": "Catalog publication date",
      "catalog_date": "2023-01-10",
      "catalog_releases": [
        {
          "name": "DreamerV3",
          "date": "2023-01-10",
          "link": "https://arxiv.org/abs/2301.04104v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Games",
      "parameters": 200000000.0,
      "compute_flop": 2.2032e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2301.04104v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "RT-1",
      "name": "RT-1",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2022-12-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-12-13",
      "catalog_releases": [
        {
          "name": "RT-1",
          "date": "2022-12-13",
          "link": "https://arxiv.org/abs/2212.06817"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Robotics",
      "parameters": 35000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2212.06817",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeepNash",
      "name": "DeepNash",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2022-12-01",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-12-01",
      "catalog_releases": [
        {
          "name": "DeepNash",
          "date": "2022-12-01",
          "link": "https://www.science.org/stoken/author-tokens/ST-887/full"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Games",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.science.org/stoken/author-tokens/ST-887/full",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "davinci-003",
      "name": "davinci-003",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-11-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "text-davinci-003"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "CICERO",
      "name": "CICERO",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-11-22",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-11-22",
      "catalog_releases": [
        {
          "name": "CICERO",
          "date": "2022-11-22",
          "link": "https://www.science.org/doi/10.1126/science.ade9097"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Games",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.science.org/doi/10.1126/science.ade9097",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AR-LDM",
      "name": "AR-LDM",
      "lab": "Alibaba",
      "organization": "Alibaba,University of Waterloo,Vector Institute",
      "lab_inferred": false,
      "date": "2022-11-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-11-20",
      "catalog_releases": [
        {
          "name": "AR-LDM",
          "date": "2022-11-20",
          "link": "https://arxiv.org/abs/2211.10950"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Image generation",
      "parameters": 1500000000.0,
      "compute_flop": 5.1e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2211.10950",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Galactica",
      "name": "Galactica",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-11-16",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-11-16",
      "catalog_releases": [
        {
          "name": "Galactica",
          "date": "2022-11-16",
          "link": "https://arxiv.org/abs/2211.09085"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Language,Biology",
      "parameters": 120000000000.0,
      "compute_flop": 3.24e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2211.09085",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Mogrifier RLSTM (WT2)",
      "name": "Mogrifier RLSTM (WT2)",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2022-11-03",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-11-03",
      "catalog_releases": [
        {
          "name": "Mogrifier RLSTM (WT2)",
          "date": "2022-11-03",
          "link": "https://arxiv.org/abs/2211.01848"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 35000000.0,
      "compute_flop": 1.4e+17,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2211.01848",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "EnCodec",
      "name": "EnCodec",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-10-24",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-10-24",
      "catalog_releases": [
        {
          "name": "EnCodec",
          "date": "2022-10-24",
          "link": "https://arxiv.org/abs/2210.13438,"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Audio",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2210.13438,",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "U-PaLM (540B)",
      "name": "U-PaLM (540B)",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2022-10-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-10-20",
      "catalog_releases": [
        {
          "name": "U-PaLM (540B)",
          "date": "2022-10-20",
          "link": "https://arxiv.org/abs/2210.11399"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 540000000000.0,
      "compute_flop": 2.53e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2210.11399",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "LMSI-Palm",
      "name": "LMSI-Palm",
      "lab": "Google DeepMind",
      "organization": "Google,University of Illinois Urbana-Champaign (UIUC)",
      "lab_inferred": false,
      "date": "2022-10-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-10-20",
      "catalog_releases": [
        {
          "name": "LMSI-Palm",
          "date": "2022-10-20",
          "link": "https://arxiv.org/abs/2210.11610"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 540000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2210.11610",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Flan-PaLM 540B",
      "name": "Flan-PaLM 540B",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2022-10-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-10-20",
      "catalog_releases": [
        {
          "name": "Flan-PaLM 540B",
          "date": "2022-10-20",
          "link": "https://arxiv.org/abs/2210.11416"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 540000000000.0,
      "compute_flop": 2.540000000001e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2210.11416",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Diplodocus",
      "name": "Diplodocus",
      "lab": "Meta",
      "organization": "Meta AI,Massachusetts Institute of Technology (MIT)",
      "lab_inferred": false,
      "date": "2022-10-11",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-10-11",
      "catalog_releases": [
        {
          "name": "Diplodocus",
          "date": "2022-10-11",
          "link": "https://arxiv.org/abs/2210.05492"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Games",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2210.05492",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaTensor",
      "name": "AlphaTensor",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2022-10-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-10-05",
      "catalog_releases": [
        {
          "name": "AlphaTensor",
          "date": "2022-10-05",
          "link": "https://www.nature.com/articles/s41586-022-05172-4"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Other,Games,Mathematics",
      "parameters": null,
      "compute_flop": 7.1414784e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.nature.com/articles/s41586-022-05172-4",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Make-A-Video",
      "name": "Make-A-Video",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-09-29",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-09-29",
      "catalog_releases": [
        {
          "name": "Make-A-Video",
          "date": "2022-09-29",
          "link": "https://arxiv.org/abs/2209.14792"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Video",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2209.14792",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Whisper",
      "name": "Whisper",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-09-21",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-09-21",
      "catalog_releases": [
        {
          "name": "Whisper",
          "date": "2022-09-21",
          "link": "https://cdn.openai.com/papers/whisper.pdf\n\nhttps://arxiv.org/abs/2212.04356"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Speech",
      "parameters": 1550000000.0,
      "compute_flop": 4.2072663e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://cdn.openai.com/papers/whisper.pdf\n\nhttps://arxiv.org/abs/2212.04356",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLI",
      "name": "PaLI",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2022-09-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-09-14",
      "catalog_releases": [
        {
          "name": "PaLI",
          "date": "2022-09-14",
          "link": "https://arxiv.org/abs/2209.06794v4"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language,Vision,Multimodal",
      "parameters": 16900000000.0,
      "compute_flop": 1.69e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2209.06794v4",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ESM2-15B",
      "name": "ESM2-15B",
      "lab": "Meta",
      "organization": "Meta AI,New York University (NYU),Stanford University,Massachusetts Institute of Technology (MIT)",
      "lab_inferred": false,
      "date": "2022-07-21",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-07-21",
      "catalog_releases": [
        {
          "name": "ESM2-15B",
          "date": "2022-07-21",
          "link": "https://www.science.org/doi/abs/10.1126/science.ade2574\nhttps://www.biorxiv.org/content/10.1101/2022.07.20.500902v2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Biology",
      "parameters": 15000000000.0,
      "compute_flop": 7.35000000001e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.science.org/doi/abs/10.1126/science.ade2574\nhttps://www.biorxiv.org/content/10.1101/2022.07.20.500902v2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "NLLB",
      "name": "NLLB",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-07-06",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-07-06",
      "catalog_releases": [
        {
          "name": "NLLB",
          "date": "2022-07-06",
          "link": "https://research.facebook.com/publications/no-language-left-behind/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 54500000000.0,
      "compute_flop": 1.751113728e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://research.facebook.com/publications/no-language-left-behind/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Minerva (540B)",
      "name": "Minerva (540B)",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2022-06-29",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-06-29",
      "catalog_releases": [
        {
          "name": "Minerva (540B)",
          "date": "2022-06-29",
          "link": "https://arxiv.org/abs/2206.14858"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 540350000000.0,
      "compute_flop": 2.7415e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2206.14858",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Parti",
      "name": "Parti",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2022-06-22",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-06-22",
      "catalog_releases": [
        {
          "name": "Parti",
          "date": "2022-06-22",
          "link": "https://arxiv.org/abs/2206.10789v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Image generation",
      "parameters": 20000000000.0,
      "compute_flop": 5.09607936e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2206.10789v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "CoCa",
      "name": "CoCa",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2022-06-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-06-14",
      "catalog_releases": [
        {
          "name": "CoCa",
          "date": "2022-06-14",
          "link": "https://arxiv.org/abs/2205.01917v2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Vision",
      "parameters": 2100000000.0,
      "compute_flop": 7.3e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2205.01917v2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Imagen",
      "name": "Imagen",
      "lab": "Google DeepMind",
      "organization": "Google Brain",
      "lab_inferred": false,
      "date": "2022-05-23",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-05-23",
      "catalog_releases": [
        {
          "name": "Imagen",
          "date": "2022-05-23",
          "link": "https://arxiv.org/abs/2205.11487"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": 7762000000.0,
      "compute_flop": 1.46e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2205.11487",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gato",
      "name": "Gato",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2022-05-12",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-05-12",
      "catalog_releases": [
        {
          "name": "Gato",
          "date": "2022-05-12",
          "link": "https://arxiv.org/abs/2205.06175"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Robotics,Games,Language",
      "parameters": 1180000000.0,
      "compute_flop": 4.02e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2205.06175",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "opt-13b",
      "name": "opt-13b",
      "lab": "Meta",
      "organization": null,
      "lab_inferred": true,
      "date": "2022-05-11",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "opt-13b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "OPT-1.3B",
      "name": "OPT-1.3B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-05-11",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "opt-1.3b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "UL2",
      "name": "UL2",
      "lab": "Google DeepMind",
      "organization": "Google Research,Google Brain",
      "lab_inferred": false,
      "date": "2022-05-10",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-05-10",
      "catalog_releases": [
        {
          "name": "UL2",
          "date": "2022-05-10",
          "link": "https://arxiv.org/abs/2205.05131v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 20000000000.0,
      "compute_flop": 1.2e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2205.05131v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "OPT-66B",
      "name": "OPT-66B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-05-03",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.100000000001e+23,
      "variants": [
        "opt-66b"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "OPT-175B",
      "name": "OPT-175B",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-05-02",
      "date_basis": "Benchmark release date",
      "catalog_date": "2022-05-02",
      "catalog_releases": [
        {
          "name": "OPT-175B",
          "date": "2022-05-02",
          "link": "https://arxiv.org/abs/2205.01068"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Language",
      "parameters": 175000000000.0,
      "compute_flop": 4.3e+23,
      "variants": [
        "opt-175b"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2205.01068",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Flamingo",
      "name": "Flamingo",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2022-04-29",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-04-29",
      "catalog_releases": [
        {
          "name": "Flamingo",
          "date": "2022-04-29",
          "link": "https://arxiv.org/abs/2204.14198"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Vision,Language,Video",
      "parameters": 80000000000.0,
      "compute_flop": 2.18972000000001e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2204.14198",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "XMC-GAN",
      "name": "XMC-GAN",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2022-04-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-04-14",
      "catalog_releases": [
        {
          "name": "XMC-GAN",
          "date": "2022-04-14",
          "link": "https://arxiv.org/abs/2101.04702"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Image generation",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2101.04702",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Sparse all-MLP",
      "name": "Sparse all-MLP",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-04-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-04-14",
      "catalog_releases": [
        {
          "name": "Sparse all-MLP",
          "date": "2022-04-14",
          "link": "https://arxiv.org/abs/2203.06850"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 9410000000.0,
      "compute_flop": 5.32224e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2203.06850",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DALL·E 2",
      "name": "DALL·E 2",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-04-06",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-04-06",
      "catalog_releases": [
        {
          "name": "DALL·E 2",
          "date": "2022-04-06",
          "link": "https://cdn.openai.com/papers/dall-e-2.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": 3500000000.0,
      "compute_flop": 3.3695784e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://cdn.openai.com/papers/dall-e-2.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLM 62B",
      "name": "PaLM 62B",
      "lab": "Google DeepMind",
      "organization": null,
      "lab_inferred": true,
      "date": "2022-04-04",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "PaLM 62B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PaLM (540B)",
      "name": "PaLM (540B)",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2022-04-04",
      "date_basis": "Benchmark release date",
      "catalog_date": "2022-04-04",
      "catalog_releases": [
        {
          "name": "PaLM (540B)",
          "date": "2022-04-04",
          "link": "https://arxiv.org/abs/2204.02311"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 540350000000.0,
      "compute_flop": 2.5272e+24,
      "variants": [
        "PaLM 540B"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2204.02311",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Chinchilla",
      "name": "Chinchilla",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2022-03-29",
      "date_basis": "Benchmark release date",
      "catalog_date": "2022-03-29",
      "catalog_releases": [
        {
          "name": "Chinchilla",
          "date": "2022-03-29",
          "link": "https://arxiv.org/abs/2203.15556"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 70000000000.0,
      "compute_flop": 5.76e+23,
      "variants": [
        "Chinchilla (70B)"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2203.15556",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Make-A-Scene",
      "name": "Make-A-Scene",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-03-24",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-03-24",
      "catalog_releases": [
        {
          "name": "Make-A-Scene",
          "date": "2022-03-24",
          "link": "https://arxiv.org/abs/2203.13131"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Image generation",
      "parameters": 4000000000.0,
      "compute_flop": 6.4172851e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2203.13131",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "davinci-002",
      "name": "davinci-002",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-03-15",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.578e+24,
      "variants": [
        "code-davinci-002",
        "text-davinci-002"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3.5 (davinci-002)",
      "name": "GPT-3.5 (davinci-002)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-03-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-03-15",
      "catalog_releases": [
        {
          "name": "GPT-3.5 (davinci-002)",
          "date": "2022-03-15",
          "link": "https://platform.openai.com/docs/models/gpt-3-5"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": null,
      "compute_flop": 2.578e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://platform.openai.com/docs/models/gpt-3-5",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Statement Curriculum Learning",
      "name": "Statement Curriculum Learning",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-03-02",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-03-02",
      "catalog_releases": [
        {
          "name": "Statement Curriculum Learning",
          "date": "2022-03-02",
          "link": "https://arxiv.org/abs/2202.01344"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language,Mathematics",
      "parameters": 774000000.0,
      "compute_flop": 1.7901648e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2202.01344",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ST-MoE",
      "name": "ST-MoE",
      "lab": "Google DeepMind",
      "organization": "Google,Google Brain,Google Research",
      "lab_inferred": false,
      "date": "2022-02-17",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-02-17",
      "catalog_releases": [
        {
          "name": "ST-MoE",
          "date": "2022-02-17",
          "link": "https://arxiv.org/abs/2202.08906v2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 269000000000.0,
      "compute_flop": 2.9e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2202.08906v2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "LaMDA",
      "name": "LaMDA",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2022-02-10",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-02-10",
      "catalog_releases": [
        {
          "name": "LaMDA",
          "date": "2022-02-10",
          "link": "https://arxiv.org/abs/2201.08239"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 137000000000.0,
      "compute_flop": 3.55e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2201.08239",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "MaskGIT (ImageNet)",
      "name": "MaskGIT (ImageNet)",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2022-02-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-02-08",
      "catalog_releases": [
        {
          "name": "MaskGIT (ImageNet)",
          "date": "2022-02-08",
          "link": "https://arxiv.org/abs/2202.04200"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "",
      "domain": "Image generation",
      "parameters": 227000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2202.04200",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "RETRO-7B",
      "name": "RETRO-7B",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2022-02-07",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-02-07",
      "catalog_releases": [
        {
          "name": "RETRO-7B",
          "date": "2022-02-07",
          "link": "https://arxiv.org/abs/2112.04426"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 7500000000.0,
      "compute_flop": 1.68e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2112.04426",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaCode",
      "name": "AlphaCode",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2022-02-02",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-02-02",
      "catalog_releases": [
        {
          "name": "AlphaCode",
          "date": "2022-02-02",
          "link": "https://arxiv.org/abs/2203.07814"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 41100000000.0,
      "compute_flop": 2.38010000000001e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2203.07814",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "InstructGPT 6B",
      "name": "InstructGPT 6B",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-01-27",
      "date_basis": "Benchmark release date",
      "catalog_date": "2022-01-27",
      "catalog_releases": [
        {
          "name": "InstructGPT 6B",
          "date": "2022-01-27",
          "link": "https://arxiv.org/abs/2203.02155"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": 6000000000.0,
      "compute_flop": null,
      "variants": [
        "text-curie-001"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2203.02155",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "InstructGPT 350M",
      "name": "InstructGPT 350M",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-01-27",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": null,
      "variants": [
        "text-ada-001"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "InstructGPT 175B",
      "name": "InstructGPT 175B",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-01-27",
      "date_basis": "Benchmark release date",
      "catalog_date": "2022-01-27",
      "catalog_releases": [
        {
          "name": "InstructGPT 175B",
          "date": "2022-01-27",
          "link": "https://arxiv.org/pdf/2203.02155"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": 175000000000.0,
      "compute_flop": 3.19181e+23,
      "variants": [
        "text-davinci-001"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/pdf/2203.02155",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "InstructGPT 1.3B",
      "name": "InstructGPT 1.3B",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2022-01-27",
      "date_basis": "Benchmark release date",
      "catalog_date": "2022-01-27",
      "catalog_releases": [
        {
          "name": "InstructGPT 1.3B",
          "date": "2022-01-27",
          "link": "https://arxiv.org/abs/2203.02155"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": 1300000000.0,
      "compute_flop": null,
      "variants": [
        "text-babbage-001"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2203.02155",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "data2vec (vision)",
      "name": "data2vec (vision)",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-01-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-01-20",
      "catalog_releases": [
        {
          "name": "data2vec (vision)",
          "date": "2022-01-20",
          "link": "https://ai.facebook.com/research/data2vec-a-general-framework-for-self-supervised-learning-in-speech-vision-and-language/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 705134592.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://ai.facebook.com/research/data2vec-a-general-framework-for-self-supervised-learning-in-speech-vision-and-language/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "data2vec (speech)",
      "name": "data2vec (speech)",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-01-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-01-20",
      "catalog_releases": [
        {
          "name": "data2vec (speech)",
          "date": "2022-01-20",
          "link": "https://ai.facebook.com/research/data2vec-a-general-framework-for-self-supervised-learning-in-speech-vision-and-language/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Speech",
      "parameters": 705134592.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://ai.facebook.com/research/data2vec-a-general-framework-for-self-supervised-learning-in-speech-vision-and-language/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "data2vec (language)",
      "name": "data2vec (language)",
      "lab": "Meta",
      "organization": "Meta AI",
      "lab_inferred": false,
      "date": "2022-01-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-01-20",
      "catalog_releases": [
        {
          "name": "data2vec (language)",
          "date": "2022-01-20",
          "link": "https://ai.facebook.com/research/data2vec-a-general-framework-for-self-supervised-learning-in-speech-vision-and-language/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 705134592.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://ai.facebook.com/research/data2vec-a-general-framework-for-self-supervised-learning-in-speech-vision-and-language/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Detic",
      "name": "Detic",
      "lab": "Meta",
      "organization": "Meta AI,University of Texas at Austin",
      "lab_inferred": false,
      "date": "2022-01-07",
      "date_basis": "Catalog publication date",
      "catalog_date": "2022-01-07",
      "catalog_releases": [
        {
          "name": "Detic",
          "date": "2022-01-07",
          "link": "https://arxiv.org/abs/2201.02605"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 88000000.0,
      "compute_flop": 2.34399744e+19,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2201.02605",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "XGLM-7.5B",
      "name": "XGLM-7.5B",
      "lab": "Meta",
      "organization": "Meta AI,Facebook AI Research",
      "lab_inferred": false,
      "date": "2021-12-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-12-20",
      "catalog_releases": [
        {
          "name": "XGLM-7.5B",
          "date": "2021-12-20",
          "link": "https://arxiv.org/abs/2112.10668"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Language",
      "parameters": 7500000000.0,
      "compute_flop": 2.25e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2112.10668",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Contriever",
      "name": "Contriever",
      "lab": "Meta",
      "organization": "Meta AI,University College London (UCL),PSL University,Université Grenoble Alpes",
      "lab_inferred": false,
      "date": "2021-12-16",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-12-16",
      "catalog_releases": [
        {
          "name": "Contriever",
          "date": "2021-12-16",
          "link": "https://arxiv.org/abs/2112.09118"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Language",
      "parameters": 110000000.0,
      "compute_flop": 1.57e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2112.09118",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "LongT5",
      "name": "LongT5",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2021-12-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-12-15",
      "catalog_releases": [
        {
          "name": "LongT5",
          "date": "2021-12-15",
          "link": "https://arxiv.org/abs/2112.07916"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 3000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2112.07916",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GLaM",
      "name": "GLaM",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2021-12-13",
      "date_basis": "Benchmark release date",
      "catalog_date": "2021-12-13",
      "catalog_releases": [
        {
          "name": "GLaM",
          "date": "2021-12-13",
          "link": "https://arxiv.org/abs/2112.06905"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 1200000000000.0,
      "compute_flop": 3.6363112434e+23,
      "variants": [
        "GLaM (MoE)"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2112.06905",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Gopher (280B)",
      "name": "Gopher (280B)",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2021-12-08",
      "date_basis": "Benchmark release date",
      "catalog_date": "2021-12-08",
      "catalog_releases": [
        {
          "name": "Gopher (280B)",
          "date": "2021-12-08",
          "link": "https://arxiv.org/abs/2112.11446"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 280000000000.0,
      "compute_flop": 6.31e+23,
      "variants": [
        "Gopher (280B)"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2112.11446",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Student of Games",
      "name": "Student of Games",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2021-12-06",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-12-06",
      "catalog_releases": [
        {
          "name": "Student of Games",
          "date": "2021-12-06",
          "link": "https://arxiv.org/abs/2112.03178"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Games",
      "parameters": null,
      "compute_flop": 3.6679273004682866e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2112.03178",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "BASIC-L",
      "name": "BASIC-L",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2021-11-19",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-11-19",
      "catalog_releases": [
        {
          "name": "BASIC-L",
          "date": "2021-11-19",
          "link": "https://arxiv.org/abs/2111.10050"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Vision",
      "parameters": 3070000000.0,
      "compute_flop": 4.12e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2111.10050",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AliceMind-MMU",
      "name": "AliceMind-MMU",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2021-11-17",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-11-17",
      "catalog_releases": [
        {
          "name": "AliceMind-MMU",
          "date": "2021-11-17",
          "link": "https://arxiv.org/abs/2111.08896"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2111.08896",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ViT-G/14 (LiT)",
      "name": "ViT-G/14 (LiT)",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2021-11-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-11-15",
      "catalog_releases": [
        {
          "name": "ViT-G/14 (LiT)",
          "date": "2021-11-15",
          "link": "https://arxiv.org/abs/2111.07991v3"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Vision",
      "parameters": 3005000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2111.07991v3",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Masked Autoencoders ViT-H",
      "name": "Masked Autoencoders ViT-H",
      "lab": "Meta",
      "organization": "Facebook AI Research",
      "lab_inferred": false,
      "date": "2021-11-11",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-11-11",
      "catalog_releases": [
        {
          "name": "Masked Autoencoders ViT-H",
          "date": "2021-11-11",
          "link": "https://arxiv.org/abs/2111.06377"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Vision",
      "parameters": 632000000.0,
      "compute_flop": 4.6e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2111.06377",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaFold-Multimer",
      "name": "AlphaFold-Multimer",
      "lab": "Google DeepMind",
      "organization": "Google DeepMind,DeepMind",
      "lab_inferred": false,
      "date": "2021-10-04",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-10-04",
      "catalog_releases": [
        {
          "name": "AlphaFold-Multimer",
          "date": "2021-10-04",
          "link": "https://www.biorxiv.org/content/10.1101/2021.10.04.463034v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Biology",
      "parameters": null,
      "compute_flop": 4.35e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.biorxiv.org/content/10.1101/2021.10.04.463034v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "FLAN 137B",
      "name": "FLAN 137B",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2021-09-03",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-09-03",
      "catalog_releases": [
        {
          "name": "FLAN 137B",
          "date": "2021-09-03",
          "link": "https://arxiv.org/abs/2109.01652"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 137000000000.0,
      "compute_flop": 2.047e+24,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2109.01652",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "XLMR-XXL",
      "name": "XLMR-XXL",
      "lab": "Meta",
      "organization": "Facebook AI Research",
      "lab_inferred": false,
      "date": "2021-08-17",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-08-17",
      "catalog_releases": [
        {
          "name": "XLMR-XXL",
          "date": "2021-08-17",
          "link": "https://arxiv.org/abs/2105.00572"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 10700000000.0,
      "compute_flop": 3.366e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2105.00572",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "W2v-BERT",
      "name": "W2v-BERT",
      "lab": "Google DeepMind",
      "organization": "Google Brain,Massachusetts Institute of Technology (MIT)",
      "lab_inferred": false,
      "date": "2021-08-07",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-08-07",
      "catalog_releases": [
        {
          "name": "W2v-BERT",
          "date": "2021-08-07",
          "link": "https://arxiv.org/abs/2108.06209v2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Speech",
      "parameters": 1000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2108.06209v2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "6-Act Tether",
      "name": "6-Act Tether",
      "lab": "Meta",
      "organization": "Facebook AI Research,Georgia Institute of Technology",
      "lab_inferred": false,
      "date": "2021-08-03",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-08-03",
      "catalog_releases": [
        {
          "name": "6-Act Tether",
          "date": "2021-08-03",
          "link": "https://openaccess.thecvf.com/content/ICCV2021/html/Ye_Auxiliary_Tasks_and_Exploration_Enable_ObjectGoal_Navigation_ICCV_2021_paper.html\nhttps://arxiv.org/abs/2104.04112"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Robotics",
      "parameters": 5000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openaccess.thecvf.com/content/ICCV2021/html/Ye_Auxiliary_Tasks_and_Exploration_Enable_ObjectGoal_Navigation_ICCV_2021_paper.html\nhttps://arxiv.org/abs/2104.04112",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "SEER",
      "name": "SEER",
      "lab": "Meta",
      "organization": "Facebook AI Research,INRIA",
      "lab_inferred": false,
      "date": "2021-07-29",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-07-29",
      "catalog_releases": [
        {
          "name": "SEER",
          "date": "2021-07-29",
          "link": "https://arxiv.org/abs/2103.01988"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Vision",
      "parameters": 1300000000.0,
      "compute_flop": 1.8e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2103.01988",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "HuBERT",
      "name": "HuBERT",
      "lab": "Meta",
      "organization": "Facebook AI Research",
      "lab_inferred": false,
      "date": "2021-07-27",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-07-27",
      "catalog_releases": [
        {
          "name": "HuBERT",
          "date": "2021-07-27",
          "link": "https://arxiv.org/abs/2106.07447"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Speech",
      "parameters": 1000000000.0,
      "compute_flop": 5.54e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2106.07447",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GOAT",
      "name": "GOAT",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2021-07-27",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-07-27",
      "catalog_releases": [
        {
          "name": "GOAT",
          "date": "2021-07-27",
          "link": "https://deepmind.com/blog/article/generally-capable-agents-emerge-from-open-ended-play\n\nhttps://arxiv.org/abs/2107.12808"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Games",
      "parameters": 3472816.0,
      "compute_flop": 2.412e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://deepmind.com/blog/article/generally-capable-agents-emerge-from-open-ended-play\n\nhttps://arxiv.org/abs/2107.12808",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Codex",
      "name": "Codex",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2021-07-07",
      "date_basis": "Benchmark release date",
      "catalog_date": "2021-07-07",
      "catalog_releases": [
        {
          "name": "Codex",
          "date": "2021-07-07",
          "link": "https://openai.com/blog/openai-codex/\nhttps://arxiv.org/abs/2107.03374"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": 12000000000.0,
      "compute_flop": 7.344e+22,
      "variants": [
        "code-cushman-002",
        "o"
      ],
      "benchmarks": {},
      "catalog_link": "https://openai.com/blog/openai-codex/\nhttps://arxiv.org/abs/2107.03374",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "EfficientNetV2-XL",
      "name": "EfficientNetV2-XL",
      "lab": "Google DeepMind",
      "organization": "Google,Google Brain",
      "lab_inferred": false,
      "date": "2021-06-23",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-06-23",
      "catalog_releases": [
        {
          "name": "EfficientNetV2-XL",
          "date": "2021-06-23",
          "link": "https://arxiv.org/abs/2104.00298"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 208000000.0,
      "compute_flop": 9.56e+19,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2104.00298",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ALIGN",
      "name": "ALIGN",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2021-06-11",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-06-11",
      "catalog_releases": [
        {
          "name": "ALIGN",
          "date": "2021-06-11",
          "link": "https://arxiv.org/abs/2102.05918"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Vision,Language",
      "parameters": 820000000.0,
      "compute_flop": 2.598670000001e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2102.05918",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "CoAtNet",
      "name": "CoAtNet",
      "lab": "Google DeepMind",
      "organization": "Google,Google Research,Google Brain",
      "lab_inferred": false,
      "date": "2021-06-09",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-06-09",
      "catalog_releases": [
        {
          "name": "CoAtNet",
          "date": "2021-06-09",
          "link": "https://arxiv.org/abs/2106.04803v2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Vision",
      "parameters": 2440000000.0,
      "compute_flop": 4.27e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2106.04803v2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ViT-G/14",
      "name": "ViT-G/14",
      "lab": "Google DeepMind",
      "organization": "Google Brain,Google Research",
      "lab_inferred": false,
      "date": "2021-06-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-06-08",
      "catalog_releases": [
        {
          "name": "ViT-G/14",
          "date": "2021-06-08",
          "link": "https://arxiv.org/abs/2106.04560"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Vision",
      "parameters": 1843000000.0,
      "compute_flop": 5.85e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2106.04560",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ByT5-XXL",
      "name": "ByT5-XXL",
      "lab": "Google DeepMind",
      "organization": "Google,Google Research",
      "lab_inferred": false,
      "date": "2021-05-28",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-05-28",
      "catalog_releases": [
        {
          "name": "ByT5-XXL",
          "date": "2021-05-28",
          "link": "https://arxiv.org/abs/2105.13626"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 12900000000.0,
      "compute_flop": 8.1e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2105.13626",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Transformer local-attention (NesT-B)",
      "name": "Transformer local-attention (NesT-B)",
      "lab": "Google DeepMind",
      "organization": "Google Cloud,Google Research",
      "lab_inferred": false,
      "date": "2021-05-26",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-05-26",
      "catalog_releases": [
        {
          "name": "Transformer local-attention (NesT-B)",
          "date": "2021-05-26",
          "link": "https://arxiv.org/abs/2105.12723v4"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 90100000.0,
      "compute_flop": 2.40576e+19,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2105.12723v4",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ADM",
      "name": "ADM",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2021-05-11",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-05-11",
      "catalog_releases": [
        {
          "name": "ADM",
          "date": "2021-05-11",
          "link": "https://arxiv.org/abs/2105.05233"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (non-commercial)",
      "domain": "Image generation",
      "parameters": 559000000.0,
      "compute_flop": 6.2e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2105.05233",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "PLUG",
      "name": "PLUG",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2021-04-19",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-04-19",
      "catalog_releases": [
        {
          "name": "PLUG",
          "date": "2021-04-19",
          "link": "https://mp.weixin.qq.com/s/DAQomIkDa52Sef-ruyH5qg"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Hosted access (no API)",
      "domain": "Language",
      "parameters": 27000000000.0,
      "compute_flop": 3.5997696e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://mp.weixin.qq.com/s/DAQomIkDa52Sef-ruyH5qg",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "M6-T",
      "name": "M6-T",
      "lab": "Alibaba",
      "organization": "Alibaba",
      "lab_inferred": false,
      "date": "2021-03-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-03-05",
      "catalog_releases": [
        {
          "name": "M6-T",
          "date": "2021-03-05",
          "link": "https://arxiv.org/abs/2105.15082"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Language,Vision",
      "parameters": 1002700000000.0,
      "compute_flop": 5.5e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2105.15082",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Generative BST",
      "name": "Generative BST",
      "lab": "Meta",
      "organization": "Facebook AI Research",
      "lab_inferred": false,
      "date": "2021-03-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-03-05",
      "catalog_releases": [
        {
          "name": "Generative BST",
          "date": "2021-03-05",
          "link": "https://arxiv.org/abs/2004.13637"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 9431810048.0,
      "compute_flop": 1.449e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2004.13637",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Meta Pseudo Labels",
      "name": "Meta Pseudo Labels",
      "lab": "Google DeepMind",
      "organization": "Google Brain,Google AI",
      "lab_inferred": false,
      "date": "2021-03-01",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-03-01",
      "catalog_releases": [
        {
          "name": "Meta Pseudo Labels",
          "date": "2021-03-01",
          "link": "https://arxiv.org/abs/2003.10580"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Vision",
      "parameters": 480000000.0,
      "compute_flop": 4.79e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2003.10580",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "MSA Transformer",
      "name": "MSA Transformer",
      "lab": "Meta",
      "organization": "Facebook AI Research,University of California (UC) Berkeley,New York University (NYU)",
      "lab_inferred": false,
      "date": "2021-02-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-02-13",
      "catalog_releases": [
        {
          "name": "MSA Transformer",
          "date": "2021-02-13",
          "link": "https://proceedings.mlr.press/v139/rao21a/rao21a.pdf"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Biology",
      "parameters": 100000000.0,
      "compute_flop": 5.49e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://proceedings.mlr.press/v139/rao21a/rao21a.pdf",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DeiT-B",
      "name": "DeiT-B",
      "lab": "Meta",
      "organization": "Meta AI,Sorbonne University",
      "lab_inferred": false,
      "date": "2021-01-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-01-15",
      "catalog_releases": [
        {
          "name": "DeiT-B",
          "date": "2021-01-15",
          "link": "https://arxiv.org/abs/2012.12877"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 86000000.0,
      "compute_flop": 7.884e+19,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2012.12877",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Switch",
      "name": "Switch",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2021-01-11",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-01-11",
      "catalog_releases": [
        {
          "name": "Switch",
          "date": "2021-01-11",
          "link": "https://arxiv.org/abs/2101.03961"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 1571000000000.0,
      "compute_flop": 8.22e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2101.03961",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "BigSSL",
      "name": "BigSSL",
      "lab": "Google DeepMind",
      "organization": "Google,Apple",
      "lab_inferred": false,
      "date": "2021-01-10",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-01-10",
      "catalog_releases": [
        {
          "name": "BigSSL",
          "date": "2021-01-10",
          "link": "https://arxiv.org/abs/2109.13226"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Speech",
      "parameters": 8000000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2109.13226",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "DALL-E",
      "name": "DALL-E",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2021-01-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-01-05",
      "catalog_releases": [
        {
          "name": "DALL-E",
          "date": "2021-01-05",
          "link": "https://openai.com/blog/dall-e/\n\nhttps://arxiv.org/abs/2102.12092"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Image generation",
      "parameters": 12000000000.0,
      "compute_flop": 4.7e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://openai.com/blog/dall-e/\n\nhttps://arxiv.org/abs/2102.12092",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "CLIP (ViT L/14@336px)",
      "name": "CLIP (ViT L/14@336px)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2021-01-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-01-05",
      "catalog_releases": [
        {
          "name": "CLIP (ViT L/14@336px)",
          "date": "2021-01-05",
          "link": "https://arxiv.org/abs/2103.00020\nhttps://huggingface.co/openai/clip-vit-large-patch14-336"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Multimodal,Vision,Language,Video",
      "parameters": 370000000.0,
      "compute_flop": 1.05e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2103.00020\nhttps://huggingface.co/openai/clip-vit-large-patch14-336",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "CLIP (ResNet-50)",
      "name": "CLIP (ResNet-50)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2021-01-05",
      "date_basis": "Catalog publication date",
      "catalog_date": "2021-01-05",
      "catalog_releases": [
        {
          "name": "CLIP (ResNet-50)",
          "date": "2021-01-05",
          "link": "https://arxiv.org/abs/2103.00020"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Multimodal,Vision,Language,Video",
      "parameters": 88600000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2103.00020",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "CT-MoS (WT2)",
      "name": "CT-MoS (WT2)",
      "lab": "Google DeepMind",
      "organization": "Google,National Tsing Hua University",
      "lab_inferred": false,
      "date": "2020-12-25",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-12-25",
      "catalog_releases": [
        {
          "name": "CT-MoS (WT2)",
          "date": "2020-12-25",
          "link": "https://arxiv.org/abs/2012.13575"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 45000000.0,
      "compute_flop": 5.4e+17,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2012.13575",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ESM1b",
      "name": "ESM1b",
      "lab": "Meta",
      "organization": "Facebook AI Research,New York University (NYU)",
      "lab_inferred": false,
      "date": "2020-12-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-12-15",
      "catalog_releases": [
        {
          "name": "ESM1b",
          "date": "2020-12-15",
          "link": "https://www.pnas.org/doi/abs/10.1073/pnas.2016239118"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Biology",
      "parameters": 652400000.0,
      "compute_flop": 5.1e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.pnas.org/doi/abs/10.1073/pnas.2016239118",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaFold 2",
      "name": "AlphaFold 2",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2020-11-30",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-11-30",
      "catalog_releases": [
        {
          "name": "AlphaFold 2",
          "date": "2020-11-30",
          "link": "https://www.nature.com/articles/s41586-021-03819-2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Biology",
      "parameters": 93000000.0,
      "compute_flop": 2.99e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.nature.com/articles/s41586-021-03819-2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ViT-Huge/14",
      "name": "ViT-Huge/14",
      "lab": "Google DeepMind",
      "organization": "Google Brain,Google Research",
      "lab_inferred": false,
      "date": "2020-10-22",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-10-22",
      "catalog_releases": [
        {
          "name": "ViT-Huge/14",
          "date": "2020-10-22",
          "link": "https://arxiv.org/abs/2010.11929"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 632000000.0,
      "compute_flop": 4.262e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2010.11929",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ViT-Base/32",
      "name": "ViT-Base/32",
      "lab": "Google DeepMind",
      "organization": "Google Brain",
      "lab_inferred": false,
      "date": "2020-10-22",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-10-22",
      "catalog_releases": [
        {
          "name": "ViT-Base/32",
          "date": "2020-10-22",
          "link": "https://arxiv.org/abs/2010.11929"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 86000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2010.11929",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "mT5-XXL",
      "name": "mT5-XXL",
      "lab": "Google DeepMind",
      "organization": "Google,Google Research",
      "lab_inferred": false,
      "date": "2020-10-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-10-20",
      "catalog_releases": [
        {
          "name": "mT5-XXL",
          "date": "2020-10-20",
          "link": "https://aclanthology.org/2021.naacl-main.41/"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 13000000000.0,
      "compute_flop": 8.2e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://aclanthology.org/2021.naacl-main.41/",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Conformer + Wav2vec 2.0 + Noisy Student",
      "name": "Conformer + Wav2vec 2.0 + Noisy Student",
      "lab": "Google DeepMind",
      "organization": "Google,Google Research,Google Brain",
      "lab_inferred": false,
      "date": "2020-10-20",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-10-20",
      "catalog_releases": [
        {
          "name": "Conformer + Wav2vec 2.0 + Noisy Student",
          "date": "2020-10-20",
          "link": "https://arxiv.org/abs/2010.10504v2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Speech",
      "parameters": 1000000000.0,
      "compute_flop": 7.6e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2010.10504v2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "EfficientDet",
      "name": "EfficientDet",
      "lab": "Google DeepMind",
      "organization": "Google Brain",
      "lab_inferred": false,
      "date": "2020-07-27",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-07-27",
      "catalog_releases": [
        {
          "name": "EfficientDet",
          "date": "2020-07-27",
          "link": "https://arxiv.org/abs/1911.09070"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 77000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/1911.09070",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GShard (dense)",
      "name": "GShard (dense)",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2020-06-30",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-06-30",
      "catalog_releases": [
        {
          "name": "GShard (dense)",
          "date": "2020-06-30",
          "link": "https://arxiv.org/abs/2006.16668"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 2300000000.0,
      "compute_flop": 4.765e+22,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2006.16668",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3 XL (babbage)",
      "name": "GPT-3 XL (babbage)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2020-06-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": 2.38e+21,
      "variants": [
        "babbage"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3 Medium",
      "name": "GPT-3 Medium",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2020-06-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": 6.41e+20,
      "variants": [
        "ada"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3 6.7B",
      "name": "GPT-3 6.7B",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2020-06-22",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Not stated",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.2e+22,
      "variants": [
        "curie"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "davinci-001",
      "name": "davinci-001",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2020-05-28",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.14e+23,
      "variants": [
        "code-davinci-edit-001",
        "text-davinci-edit-001"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-3 175B (davinci)",
      "name": "GPT-3 175B (davinci)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2020-05-28",
      "date_basis": "Benchmark release date",
      "catalog_date": "2020-05-28",
      "catalog_releases": [
        {
          "name": "GPT-3 175B (davinci)",
          "date": "2020-05-28",
          "link": "https://arxiv.org/abs/2005.14165"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "API access",
      "domain": "Language",
      "parameters": 174600000000.0,
      "compute_flop": 3.14e+23,
      "variants": [
        "davinci",
        "davinci-002"
      ],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2005.14165",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Conformer",
      "name": "Conformer",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2020-05-16",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-05-16",
      "catalog_releases": [
        {
          "name": "Conformer",
          "date": "2020-05-16",
          "link": "https://arxiv.org/abs/2005.08100v1"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Speech",
      "parameters": 118800000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2005.08100v1",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ContextNet",
      "name": "ContextNet",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2020-05-07",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-05-07",
      "catalog_releases": [
        {
          "name": "ContextNet",
          "date": "2020-05-07",
          "link": "https://arxiv.org/abs/2005.03191v3"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Speech",
      "parameters": 112700000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2005.03191v3",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Agent57",
      "name": "Agent57",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2020-03-30",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-03-30",
      "catalog_releases": [
        {
          "name": "Agent57",
          "date": "2020-03-30",
          "link": "https://arxiv.org/abs/2003.13350"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Games",
      "parameters": null,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2003.13350",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "MetNet",
      "name": "MetNet",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2020-03-24",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-03-24",
      "catalog_releases": [
        {
          "name": "MetNet",
          "date": "2020-03-24",
          "link": "https://arxiv.org/abs/2003.12140"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Earth science",
      "parameters": 225000000.0,
      "compute_flop": 9.510912e+18,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2003.12140",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Routing Transformer (WT-103)",
      "name": "Routing Transformer (WT-103)",
      "lab": "Google DeepMind",
      "organization": "Google Research",
      "lab_inferred": false,
      "date": "2020-03-12",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-03-12",
      "catalog_releases": [
        {
          "name": "Routing Transformer (WT-103)",
          "date": "2020-03-12",
          "link": "https://arxiv.org/abs/2003.05997"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Language",
      "parameters": 79500000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2003.05997",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "FFN SwiGLU",
      "name": "FFN SwiGLU",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2020-02-14",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-02-14",
      "catalog_releases": [
        {
          "name": "FFN SwiGLU",
          "date": "2020-02-14",
          "link": "https://arxiv.org/abs/2002.05202"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 220000000.0,
      "compute_flop": 3.3687317e+19,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2002.05202",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "SimCLR",
      "name": "SimCLR",
      "lab": "Google DeepMind",
      "organization": "Google Brain",
      "lab_inferred": false,
      "date": "2020-02-13",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-02-13",
      "catalog_releases": [
        {
          "name": "SimCLR",
          "date": "2020-02-13",
          "link": "https://arxiv.org/abs/2002.05709"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": "Vision",
      "parameters": 375000000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2002.05709",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Perceiver IO (optical flow)",
      "name": "Perceiver IO (optical flow)",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2020-02-08",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-02-08",
      "catalog_releases": [
        {
          "name": "Perceiver IO (optical flow)",
          "date": "2020-02-08",
          "link": "https://arxiv.org/abs/2107.14795"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Multimodal,Language,Vision",
      "parameters": 27900000.0,
      "compute_flop": null,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2107.14795",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "Meena",
      "name": "Meena",
      "lab": "Google DeepMind",
      "organization": "Google Brain",
      "lab_inferred": false,
      "date": "2020-01-28",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-01-28",
      "catalog_releases": [
        {
          "name": "Meena",
          "date": "2020-01-28",
          "link": "https://arxiv.org/abs/2001.09977"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Language",
      "parameters": 2600000000.0,
      "compute_flop": 1.12e+23,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2001.09977",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "ContextNet + Noisy Student",
      "name": "ContextNet + Noisy Student",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2020-01-19",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-01-19",
      "catalog_releases": [
        {
          "name": "ContextNet + Noisy Student",
          "date": "2020-01-19",
          "link": "https://arxiv.org/abs/2005.09629v2"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Speech",
      "parameters": null,
      "compute_flop": 8.16e+21,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://arxiv.org/abs/2005.09629v2",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "AlphaFold",
      "name": "AlphaFold",
      "lab": "Google DeepMind",
      "organization": "DeepMind",
      "lab_inferred": false,
      "date": "2020-01-15",
      "date_basis": "Catalog publication date",
      "catalog_date": "2020-01-15",
      "catalog_releases": [
        {
          "name": "AlphaFold",
          "date": "2020-01-15",
          "link": "https://www.nature.com/articles/s41586-019-1923-7"
        }
      ],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Unreleased",
      "domain": "Biology",
      "parameters": 16340840.0,
      "compute_flop": 1e+20,
      "variants": [],
      "benchmarks": {},
      "catalog_link": "https://www.nature.com/articles/s41586-019-1923-7",
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "GPT-2 (1.5B)",
      "name": "GPT-2 (1.5B)",
      "lab": "OpenAI",
      "organization": "OpenAI",
      "lab_inferred": false,
      "date": "2019-11-05",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 1.920000000001e+21,
      "variants": [
        "gpt2-xl"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "T5-3B",
      "name": "T5-3B",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2019-10-23",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 9.0000000001e+21,
      "variants": [
        "T5-3B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    },
    {
      "id": "T5-11B",
      "name": "T5-11B",
      "lab": "Google DeepMind",
      "organization": "Google",
      "lab_inferred": false,
      "date": "2019-10-23",
      "date_basis": "Benchmark release date",
      "catalog_date": null,
      "catalog_releases": [],
      "eci": null,
      "eci_low": null,
      "eci_high": null,
      "access": "Open weights (unrestricted)",
      "domain": null,
      "parameters": null,
      "compute_flop": 3.3e+22,
      "variants": [
        "T5-11B"
      ],
      "benchmarks": {},
      "catalog_link": null,
      "source": "https://epoch.ai/benchmarks"
    }
  ]
}
