{ "meta": { "api_version": "v1", "endpoint": "/api/v1/models", "updated": "2026-07-22", "verified_date": "2026-07-22", "official_record_verified_date": "2026-07-22", "total": 27, "offset": 0, "limit": 50, "returned": 27, "provenance": "Pricing, context, release, and license fields are factual records with official_evidence when matched. Capabilities and latency are explicitly labelled benchr editorial data." }, "data": [ { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "provider": "Anthropic", "api_name": "claude-sonnet-5", "type": "frontier", "license": "proprietary", "released": "2026-07-01", "deprecated": false, "pricing": { "input_per_million": 2, "output_per_million": 10, "cache_input_per_million": 0.2, "batch_discount": 0.5, "batch_input": 1, "batch_output": 5, "intro_through": "2026-08-31", "standard_input_per_million_from_2026_09_01": 3, "standard_output_per_million_from_2026_09_01": 15, "standard_cache_input_per_million_from_2026_09_01": 0.3 }, "context": { "max_tokens": 1000000, "effective_tokens": 800000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 89.4, "lmsys_arena": 1435, "mmlu": 93.8, "humaneval": 96, "math": 93.5, "gpqa_diamond": 92, "arc_agi_2": 20 }, "estimated_fields": [ "lmsys_arena" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 96, "reasoning": 95, "writing": 93, "vision": 88, "long_context": 93, "multilingual": 89 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 620, "tokens_per_second": 75, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Frontier coding at Sonnet-tier pricing", "128K max output — well above Sonnet 4.6's 64K", "Teams upgrading off Sonnet 4.6 without Opus pricing", "Production agents that need Mythos-class quality" ], "skip_if": [ "Offensive-security or bio work — classifiers fall back to Opus 4.8", "Absolute cheapest volume ? Haiku 4.5 remains cheaper", "You need Fable 5's higher ceiling on the hardest tasks" ], "official_evidence": { "record_id": "claude-sonnet-5", "api_model_id": "claude-sonnet-5", "verified_date": "2026-07-08", "sources": { "release": "https://www.anthropic.com/news/claude-sonnet-5", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/claude-sonnet-5-system-card" }, "pricing": { "inputPerM": 2, "outputPerM": 10, "cachedInputPerM": 0.2, "batchInputPerM": 1, "batchOutputPerM": 5, "introThrough": "2026-08-31", "standardInputPerMFrom2026_09_01": 3, "standardOutputPerMFrom2026_09_01": 15, "standardCachedInputPerMFrom2026_09_01": 0.3, "standardBatchInputPerMFrom2026_09_01": 1.5, "standardBatchOutputPerMFrom2026_09_01": 7.5, "note": "Official Anthropic pricing verified 2026-07-03: introductory $2/$10 through August 31, 2026; standard $3/$15 from September 1, 2026." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": { "SWE-bench Verified": 89.4, "SWE-bench Pro": 71.8, "GPQA Diamond": 92, "Terminal-Bench 2.1": 85.6, "ARC-AGI-2": 20, "Humanity's Last Exam (no tools)": 42.5 }, "notes": "CORRECTED 2026-07-08: max output is 128,000 tokens, not 200,000 as originally recorded on launch. Re-confirmed against platform.claude.com's Models Overview table and the official 'What's New in Sonnet 5' page (both checked live 2026-07-08). Anthropic's second Mythos-class-architecture model after Claude Fable 5. Pricing corrected July 3, 2026 from the official Anthropic pricing page: launch/introductory API rate is $2/$10 per 1M through August 31, 2026, then $3/$15 from September 1, 2026; cache hits and batch pricing follow Anthropic's normal multipliers. Context remains 1,000,000 tokens with a 200,000-token max-output ceiling. Same safety-classifier fallback to Claude Opus 4.8 on offensive-cyber, most biology/chemistry, and distillation requests as Fable 5; adaptive thinking always on, no extended-thinking toggle. SWE-bench Verified (89.4%) edges out Claude Opus 4.8 (88.6%), though Opus 4.8 still leads on GPQA Diamond (93.6% vs 92.0%) and remains Anthropic's top recommendation for the hardest reasoning work. Shipped the same day Anthropic restored Claude Fable 5 to all customers." } }, { "id": "gpt-5-6", "name": "GPT-5.6", "provider": "OpenAI", "api_name": "gpt-5.6-sol", "type": "frontier", "license": "proprietary", "released": "2026-07-09", "deprecated": false, "pricing": { "input_per_million": 5, "output_per_million": 30, "cache_input_per_million": 0.5, "batch_discount": 0.5, "tier_prices_per_million": [ 2.5, 15, 1, 6 ] }, "context": { "max_tokens": 1050000, "effective_tokens": 750000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 89.8, "lmsys_arena": 1440, "mmlu": 93.8, "humaneval": 95.5, "math": 96, "gpqa_diamond": 91.2, "arc_agi_2": 23.5 }, "estimated_fields": [ "lmsys_arena", "mmlu", "math" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 95, "reasoning": 96, "writing": 89, "vision": 90, "long_context": 87, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 500, "tokens_per_second": 85, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Generally available via OpenAI API since July 9, 2026", "New SOTA agentic command-line work (Terminal-Bench)", "Frontier math and reasoning", "Computer use and multi-step agents", "Teams that need OpenAI's absolute best" ], "skip_if": [ "Cost-sensitive workloads — Terra is about half the price at close quality", "You need the deepest single-call reasoning ceiling — GPQA still trails Opus 4.8/Fable 5" ], "official_evidence": { "record_id": "gpt-5-6-sol", "api_model_id": "gpt-5.6-sol", "verified_date": "2026-07-13", "sources": { "release": "https://developers.openai.com/api/docs/changelog", "pricing": "https://developers.openai.com/api/docs/pricing", "benchmarks": "https://deploymentsafety.openai.com/gpt-5-6-preview", "context": "https://developers.openai.com/api/docs/models" }, "pricing": { "inputPerM": 5, "outputPerM": 30, "cachedInputPerM": 0.5, "batchInputPerM": 2.5, "batchOutputPerM": 15, "priceStatus": "GA pricing listed on developers.openai.com/api/docs/pricing as of 2026-07-13.", "note": "Standard short-context API price from OpenAI pricing page, re-read 2026-07-13. Long-context pricing applies separately above the short-context tier; batch/flex are 50% of standard, and priority is 2x standard.", "cacheWritePerM": 6.25, "priorityInputPerM": 10, "priorityOutputPerM": 60 }, "context": { "windowTokens": 1050000, "maxOutputTokens": 128000 }, "benchmarks": { "Terminal-Bench 2.1": 88.8, "Terminal-Bench 2.1 (ultra mode)": 91.9, "SWE-bench Verified": 89.8, "GPQA Diamond": 91.2, "note": "Terminal-Bench 2.1 figures unchanged from the June 26 preview system card. SWE-bench Verified (89.8%) and GPQA Diamond (91.2%) were added at GA (2026-07-01) on the same system-card page — both are OpenAI's strongest release yet but trail Claude Fable 5 (91.0% SWE-bench Verified) and Claude Opus 4.8 (93.6% GPQA Diamond). GeneBench v1 and ExploitGym efficiency-curve figures unchanged from preview." }, "notes": "UPDATED 2026-07-13: GPT-5.6 is now officially generally available, per OpenAI's July 9 API changelog and live models/pricing pages. The prior July 8 correction was accurate at that time but is now superseded. OpenAI's live models page lists a 1.05M context window and 128K max output for this tier; pricing is listed on the official pricing page." } }, { "id": "claude-fable-5", "name": "Claude Fable 5", "provider": "Anthropic", "api_name": "claude-fable-5", "type": "frontier", "license": "proprietary", "released": "2026-06-09", "deprecated": false, "pricing": { "input_per_million": 10, "output_per_million": 50, "cache_input_per_million": 1 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 91, "swe_bench_pro": 80.3, "lmsys_arena": 1455, "mmlu": 94.5, "humaneval": 97, "math": 95.5, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [ "swe_bench_verified", "lmsys_arena", "mmlu", "humaneval", "math" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 99, "reasoning": 98, "writing": 95, "vision": 94, "long_context": 95, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 800, "tokens_per_second": 58, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "The hardest long-horizon agentic coding", "Large-codebase migrations", "Frontier research and finance work", "Vision-driven agent loops" ], "skip_if": [ "Price matters — Opus 4.8 is half the cost", "Offensive-security or bio work — classifiers fall back to Opus 4.8", "Routine chat and drafting" ], "official_evidence": { "record_id": "claude-fable-5", "api_model_id": "claude-fable-5", "verified_date": "2026-07-01", "sources": { "release": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "pricing": "https://platform.claude.com/docs/en/about-claude/models/overview", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/news/claude-fable-5-mythos-5" }, "pricing": { "inputPerM": 10, "outputPerM": 50, "cachedInputPerM": 1, "note": "Prompt caching = 90% input discount per the launch announcement. Included free on Pro/Max/Team/seat-based Enterprise June 9-22, 2026; usage credits from June 23." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": { "SWE-bench Pro": 80.3, "SWE-bench Verified": null, "GPQA Diamond": null, "note": "Anthropic's launch table reports the higher of Mythos 5 / Fable 5 per row (within 1-3 points of each other). Starred rows (cybersecurity, biology, some reasoning) reflect Mythos 5; real-world Fable 5 results there are closer to Opus 4.8 because of the safety classifiers. SWE-bench Verified and GPQA Diamond were not published at launch." }, "notes": "Mythos-class model, generally available June 9, 2026 on the Claude API, AWS, Bedrock, Vertex AI, and Microsoft Foundry. Same underlying model as Claude Mythos 5 (claude-mythos-5, Glasswing-only) but with safety classifiers: offensive-cyber, most biology/chemistry, and distillation requests fall back to Claude Opus 4.8. Anthropic says more than 95% of Fable sessions involve no fallback. Adaptive thinking always on; no extended-thinking toggle. Uses the Opus 4.7 tokenizer (~30% more tokens than pre-4.7 models for the same text). 30-day data retention on all Mythos-class traffic. Availability update, June 12, 2026: Anthropic suspended Fable 5 (and Mythos 5) for all customers to comply with a U.S. Commerce Department export-control directive barring Mythos-class use by any foreign national; AWS revoked Bedrock access the same day. All other Anthropic models are unaffected; Anthropic disputes the recall and says it is working to restore access. Update, June 26-28, 2026: Fable 5 (the GA model) REMAINS fully suspended for all customers — the June 26 U.S. Commerce clearance covered Claude Mythos 5 only, NOT Fable 5. RESTORED, July 1, 2026: the export-control review concluded and the U.S. Commerce Department lifted the suspension on Fable 5 for all customers, the same day Anthropic launched Claude Sonnet 5; AWS restored Bedrock access in step. Mythos 5 remains Project-Glasswing-only as before (unaffected by this restoration). Listed price, context, and benchmark figures are unchanged throughout. Sources: AWS News Blog (Claude Fable 5 on AWS) June 12 + July 1 updates; Anthropic statement + Commerce Secretary's letter, June 26 (CNN/NBC/Fortune/Axios reporting); Anthropic's July 1 restoration statement." } }, { "id": "claude-opus-4-8", "name": "Claude Opus 4.8", "provider": "Anthropic", "api_name": "claude-opus-4-8", "type": "frontier", "license": "proprietary", "released": "2026-05-28", "deprecated": false, "pricing": { "input_per_million": 5, "output_per_million": 25, "cache_input_per_million": 0.5, "batch_discount": 0.5, "fast_mode_input": 10, "fast_mode_output": 50 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 88.6, "lmsys_arena": 1425, "mmlu": 93.5, "humaneval": 96, "math": 93, "gpqa_diamond": 93.6, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 97, "reasoning": 96, "writing": 91, "vision": 86, "long_context": 94, "multilingual": 90 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 700, "tokens_per_second": 68, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Highest-stakes coding tasks", "Complex multi-step agents", "Architecture decisions", "Production SWE-bench-level work" ], "skip_if": [ "You need cheap volume", "Sonnet handles your workload", "Speed is the priority — use Fast Mode instead" ], "official_evidence": { "record_id": "claude-opus-4-8", "api_model_id": "claude-opus-4-8", "verified_date": "2026-06-12", "sources": { "release": "https://www.anthropic.com/news/claude-opus-4-8", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/claude-opus-4-8-system-card" }, "pricing": { "inputPerM": 5, "outputPerM": 25, "fastModeInputPerM": 10, "fastModeOutputPerM": 50, "batchInputPerM": 2.5, "batchOutputPerM": 12.5, "cachedInputPerM": 0.5 }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000, "maxOutputTokensBeta": 300000 }, "benchmarks": { "SWE-bench Verified": 88.6, "SWE-bench Pro": 69.2, "SWE-bench Multilingual": 84.4, "SWE-bench Multimodal": 38.4, "Terminal-Bench 2.1": 74.6, "GPQA Diamond": 93.6, "OSWorld-Verified": 83.4, "BrowseComp (single-agent)": 84.3, "Humanity's Last Exam (no tools)": 49.8, "GDPval-AA (Elo)": 1890, "ARC-AGI-2": null }, "notes": "Standard price unchanged from Opus 4.7. The $10/$50 figure is the optional fast-mode rate (~2.5x output speed), NOT the base price; base is $5/$25. Cache-hit input $0.50 (0.1x base) now explicit on the official pricing page (confirmed June 12, 2026). Official tentative retirement floor: not sooner than May 28, 2027. ARC-AGI-2 is not in the official Opus 4.8 headline summary table, so it is null (do not state one). New Opus 4.7+ tokenizer can use up to ~35% more tokens per the pricing page." } }, { "id": "claude-opus-4-7", "name": "Claude Opus 4.7", "provider": "Anthropic", "api_name": "claude-opus-4-7", "type": "frontier", "license": "proprietary", "released": "2026-04", "deprecated": false, "pricing": { "input_per_million": 5, "output_per_million": 25, "cache_input_per_million": 0.5, "batch_discount": 0.5 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 87.6, "lmsys_arena": 1410, "mmlu": 92.5, "humaneval": 95.2, "math": 91.8, "gpqa_diamond": 94.2, "arc_agi_2": 24 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 96, "reasoning": 96, "writing": 90, "vision": 85, "long_context": 94, "multilingual": 89 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 720, "tokens_per_second": 65, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Complex coding tasks", "Long-document analysis", "Production agent loops", "Architecture decisions" ], "skip_if": [ "You need cheap volume", "Simple summarization", "Sonnet covers your workload" ], "official_evidence": { "record_id": "claude-opus-4-7", "api_model_id": "claude-opus-4-7", "verified_date": "2026-06-12", "sources": { "release": "https://www.anthropic.com/news/claude-opus-4-7", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/claude-opus-4-7-system-card" }, "pricing": { "inputPerM": 5, "outputPerM": 25, "fastModeInputPerM": 30, "fastModeOutputPerM": 150, "batchInputPerM": 2.5, "batchOutputPerM": 12.5, "cachedInputPerM": 0.5 }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000, "maxOutputTokensBeta": 300000 }, "benchmarks": { "SWE-bench Verified": 87.6, "SWE-bench Pro": 64.3, "SWE-bench Multilingual": 80.5, "SWE-bench Multimodal": 34.5, "Terminal-Bench 2.0": 69.4, "GPQA Diamond": 94.2 }, "notes": "Fast mode on 4.7 is $30/$150 (3x more expensive than Opus 4.8's $10/$50 fast mode). Cache-hit input $0.50 confirmed on the official pricing page June 12, 2026; official tentative retirement floor: not sooner than April 16, 2027. Opus 4.8 later restated Opus 4.7's OSWorld score upward (~82.x) after a test-harness fix." } }, { "id": "claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "provider": "Anthropic", "api_name": "claude-sonnet-4-6", "type": "mid", "license": "proprietary", "released": "2026-02-17", "deprecated": false, "pricing": { "input_per_million": 3, "output_per_million": 15, "cache_input_per_million": 0.3, "batch_discount": 0.5 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 64000 }, "benchmarks": { "values": { "swe_bench_verified": 79.6, "lmsys_arena": 1365, "mmlu": 89.6, "humaneval": 91.5, "math": 86, "gpqa_diamond": 89.9, "arc_agi_2": 15 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 88, "reasoning": 87, "writing": 89, "vision": 80, "long_context": 91, "multilingual": 86 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 480, "tokens_per_second": 95, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Production default", "Cost-effective coding", "Bulk content tasks", "Daily-driver API workloads" ], "skip_if": [ "You need frontier-grade reasoning" ], "official_evidence": { "record_id": "claude-sonnet-4-6", "api_model_id": "claude-sonnet-4-6", "verified_date": "2026-06-12", "sources": { "release": "https://www.anthropic.com/news/claude-sonnet-4-6", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/news/claude-sonnet-4-6" }, "pricing": { "inputPerM": 3, "outputPerM": 15, "batchInputPerM": 1.5, "batchOutputPerM": 7.5, "cachedInputPerM": 0.3 }, "context": { "windowTokens": 1000000, "maxOutputTokens": 64000, "maxOutputTokensBeta": 300000 }, "benchmarks": { "SWE-bench Verified": 79.6, "SWE-bench Multilingual": 75.9, "Terminal-Bench 2.0": 59.1, "OSWorld-Verified": 72.5, "GPQA Diamond": 89.9, "MMMLU": 89.3, "AIME 2025 (no tools)": 95.6, "Humanity's Last Exam (no tools)": 33.2, "Humanity's Last Exam (with tools)": 49, "ARC-AGI-2": 58.3, "tau2-bench Telecom": 97.9, "tau2-bench Retail": 91.7, "GDPval-AA (Elo)": 1633 }, "notes": "No fast-mode tier (fast mode is Opus-only). Cache-hit input $0.30 confirmed on the official pricing page June 12, 2026; official tentative retirement floor: not sooner than February 17, 2027. Benchmark values read from the official Sonnet 4.6 System Card (Table 2.1.A). SWE-bench Verified 79.6% averaged over 25 trials (80.2% with a stated prompt modification). Anthropic flags possible AIME 2025 contamination. Anthropic reports MMMLU, not plain MMLU." } }, { "id": "claude-haiku-4-5", "name": "Claude Haiku 4.5", "provider": "Anthropic", "api_name": "claude-haiku-4-5", "type": "small", "license": "proprietary", "released": "2025-10-15", "deprecated": false, "pricing": { "input_per_million": 1, "output_per_million": 5, "cache_input_per_million": 0.1, "batch_discount": 0.5 }, "context": { "max_tokens": 200000, "effective_tokens": 150000, "max_output_tokens": 64000 }, "benchmarks": { "values": { "swe_bench_verified": 73.3, "lmsys_arena": 1285, "mmlu": 82.5, "humaneval": 85, "math": 77, "gpqa_diamond": null, "arc_agi_2": 8.5 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 75, "reasoning": 76, "writing": 80, "vision": 72, "long_context": 82, "multilingual": 78 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 240, "tokens_per_second": 145, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "High-volume simple tasks", "Real-time chat", "Classification, routing, extraction" ], "skip_if": [ "Complex reasoning needed", "Long-document analysis" ], "official_evidence": { "record_id": "claude-haiku-4-5", "api_model_id": "claude-haiku-4-5", "verified_date": "2026-05-31", "sources": { "release": "https://www.anthropic.com/news/claude-haiku-4-5", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/news/claude-haiku-4-5" }, "pricing": { "inputPerM": 1, "outputPerM": 5, "batchInputPerM": 0.5, "batchOutputPerM": 2.5, "cachedInputPerM": 0.1 }, "context": { "windowTokens": 200000, "maxOutputTokens": 64000, "maxOutputTokensBeta": null }, "benchmarks": { "SWE-bench Verified": 73.3, "Terminal-Bench (no thinking)": 40.21, "Terminal-Bench (32K thinking)": 41.75, "GPQA Diamond": null, "OSWorld-Verified": null, "AIME 2025": null, "MMMLU": null }, "notes": "Pinned snapshot claude-haiku-4-5-20251001. Pricing (incl. cache-hit $0.10) and tentative retirement floor (not sooner than October 15, 2026) re-confirmed on official pages June 12, 2026; the 200K context window stands from the May 31 verification. SWE-bench Verified 73.3% (avg of 50 trials, 128K thinking budget) and Terminal-Bench are the only headline numbers Anthropic publishes as readable official text; GPQA/OSWorld/AIME/MMMLU appear only inside a launch-page image, so they are null (not guessed)." } }, { "id": "gpt-5-5", "name": "GPT-5.5", "provider": "OpenAI", "api_name": "gpt-5.5", "type": "frontier", "license": "proprietary", "released": "2026-04-23", "deprecated": false, "pricing": { "input_per_million": 5, "output_per_million": 30, "cache_input_per_million": 0.5, "batch_discount": 0.5, "batch_input": 2.5, "batch_output": 15 }, "context": { "max_tokens": 1050000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 84, "lmsys_arena": 1418, "mmlu": 93, "humaneval": 94.5, "math": 95, "gpqa_diamond": null, "arc_agi_2": 22 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 93, "reasoning": 95, "writing": 88, "vision": 89, "long_context": 86, "multilingual": 90 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 520, "tokens_per_second": 82, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Frontier math and reasoning", "Computer use", "Multi-step agents", "Vision + reasoning tasks" ], "skip_if": [ "Cost-sensitive workloads — use GPT-5", "Quick chat — the price is hard to justify" ], "official_evidence": { "record_id": "gpt-5-5", "api_model_id": "gpt-5.5", "verified_date": "2026-06-23", "sources": { "release": "https://deploymentsafety.openai.com/gpt-5-5", "pricing": "https://developers.openai.com/api/docs/models/gpt-5.5", "context": "https://developers.openai.com/api/docs/models/gpt-5.5", "benchmarks": "https://deploymentsafety.openai.com/gpt-5-5" }, "pricing": { "inputPerM": 5, "outputPerM": 30, "cachedInputPerM": 0.5, "proInputPerM": 30, "proOutputPerM": 180, "proBatchInputPerM": 15, "proBatchOutputPerM": 90, "extendedContextSurcharge": "For sessions >272K input tokens: 2x input, 1.5x output (standard/batch/flex)." }, "context": { "windowTokens": 1050000, "maxOutputTokens": 128000 }, "benchmarks": { "HealthBench (length-adjusted)": 56.5, "HealthBench Professional": 51.8, "SWE-bench Verified": null, "SWE-bench Pro": null, "Terminal-Bench 2.0": null, "OSWorld-Verified": null }, "notes": "Flagship GPT-5.5 (and GPT-5.5 Pro) announced Apr 23, 2026 (do not confuse with GPT-5.5 Instant, the ChatGPT default released May 5, 2026). Context is 1,050,000 (not a round 1M). HealthBench figures are the only official system-card benchmarks. GPT-5.5 Pro STANDARD pricing is now officially readable on developers.openai.com/api/docs/models/gpt-5.5-pro (confirmed 2026-06-23): $30 input / $180 output per 1M, batch $15/$90, NO cached-input discount, context 1,050,000 — so proInputPerM/proOutputPerM are set. The coding/agent numbers circulating (Terminal-Bench 82.7%, OSWorld 78.7%, SWE-bench Pro 58.6%) remain vendor-relayed via the 403'd launch page and stay null until readable on an official page. The extended-context >272K surcharge IS officially confirmed on the model doc page." } }, { "id": "gpt-5-4", "name": "GPT-5.4", "provider": "OpenAI", "api_name": "gpt-5.4", "type": "frontier", "license": "proprietary", "released": "2026-03-05", "deprecated": false, "pricing": { "input_per_million": 2.5, "output_per_million": 15, "cache_input_per_million": 0.25 }, "context": { "max_tokens": 1000000, "effective_tokens": 600000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 80, "osworld_verified": 75, "lmsys_arena": 1400, "mmlu": 92, "humaneval": 93, "math": 91, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [ "swe_bench_verified", "lmsys_arena", "mmlu", "humaneval", "math" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 90, "reasoning": 91, "writing": 87, "vision": 88, "long_context": 85, "multilingual": 88 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 540, "tokens_per_second": 84, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Financial modeling and spreadsheet work", "Computer-use pipelines", "Professional knowledge work", "Long documents at a mid-tier price" ], "skip_if": [ "You want OpenAI's strongest — that's GPT-5.5 now", "Cheap volume — GPT-5 Mini costs a tenth", "Simple chat — GPT-5 is cheaper" ], "official_evidence": { "record_id": "gpt-5-4", "api_model_id": "gpt-5.4", "verified_date": "2026-06-10", "sources": { "release": "https://openai.com/index/introducing-gpt-5-4-mini-and-nano/", "pricing": "https://openai.com/api/pricing/", "context": "https://openai.com/api/pricing/", "benchmarks": "https://help.openai.com/en/articles/9624314-model-release-notes" }, "pricing": { "inputPerM": 2.5, "outputPerM": 15, "cachedInputPerM": 0.25, "note": "Long-context surcharge above 272K input per the official pricing page." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000, "note": "OpenAI markets context 'as large as 1 million tokens' on the API. Inputs beyond the standard 272K window carry a long-context surcharge on the official pricing page." }, "benchmarks": { "OSWorld-Verified": 75, "SWE-bench Verified": null, "note": "OSWorld-Verified 75% is from OpenAI's launch material (vs 47.3% for GPT-5.2; human baseline 72.4%). No official SWE-bench Verified figure found for GPT-5.4." }, "notes": "Released March 5, 2026 as GPT-5.4 Thinking + GPT-5.4 Pro ('most capable and efficient frontier model for professional work'); GPT-5.4 mini and nano followed March 17 (mini available to free tier, nano API-only). Built-in computer use; tuned for finance workflows (launched alongside ChatGPT for Excel, which it powered at launch). benchr removed an earlier gpt-5-4 entry on June 1, 2026 as 'unverified' — that removal was wrong; the model is real and was re-added after this June 10 verification. Superseded as OpenAI's flagship by GPT-5.5 (April 23, 2026)." } }, { "id": "gpt-5", "name": "GPT-5", "provider": "OpenAI", "api_name": "gpt-5", "type": "frontier", "license": "proprietary", "released": "2025-08-07", "deprecated": false, "pricing": { "input_per_million": 1.25, "output_per_million": 10, "cache_input_per_million": 0.125, "batch_discount": 0.5 }, "context": { "max_tokens": 400000, "effective_tokens": 300000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 74.9, "lmsys_arena": 1375, "mmlu": 90, "humaneval": 91, "math": 87, "gpqa_diamond": null, "arc_agi_2": 16 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 88, "reasoning": 88, "writing": 86, "vision": 85, "long_context": 78, "multilingual": 87 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 520, "tokens_per_second": 90, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Production workhorse at a rational price", "Breadth tasks", "Coding agents", "The everyday OpenAI pick" ], "skip_if": [ "You need the absolute deepest reasoning — use 5.5", "Very large context windows" ], "official_evidence": { "record_id": "gpt-5", "api_model_id": "gpt-5", "verified_date": "2026-06-12", "sources": { "release": "https://deploymentsafety.openai.com/gpt-5", "pricing": "https://developers.openai.com/api/docs/models/gpt-5", "context": "https://developers.openai.com/api/docs/models/gpt-5", "benchmarks": "https://cdn.openai.com/gpt-5-system-card.pdf" }, "pricing": { "inputPerM": 1.25, "outputPerM": 10, "cachedInputPerM": 0.125 }, "context": { "windowTokens": 400000, "maxOutputTokens": 128000 }, "benchmarks": { "SWE-bench Verified": 74.9, "AIME 2025": null, "GPQA Diamond": null, "HealthBench Hard": 46.2 }, "notes": "SWE-bench Verified 74.9% is officially OpenAI's launch-blog figure (verbosity=medium), confirmed via the system card PDF which cites it (p.36). AIME 2025 (~94.6%) and GPQA Diamond (~88.4%) are widely attributed to the launch page openai.com/index/introducing-gpt-5, which blocks automated fetchers (403); they could NOT be re-read from an accessible official source, so they are null. The 88.4% figure may refer to GPT-5 pro, not base GPT-5. OpenAI docs now label GPT-5 the previous model." } }, { "id": "gpt-5-mini", "name": "GPT-5 Mini", "provider": "OpenAI", "api_name": "gpt-5-mini", "type": "small", "license": "proprietary", "released": "2025-08", "deprecated": false, "pricing": { "input_per_million": 0.25, "output_per_million": 2, "cache_input_per_million": 0.025, "batch_discount": 0.5 }, "context": { "max_tokens": 400000, "effective_tokens": 200000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 48, "lmsys_arena": 1275, "mmlu": 82, "humaneval": 78, "math": 74, "gpqa_diamond": null, "arc_agi_2": 6 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 72, "reasoning": 74, "writing": 75, "vision": 72, "long_context": 70, "multilingual": 80 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 220, "tokens_per_second": 160, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Cheap chat at scale", "Simple extraction and routing", "High-volume classification" ], "skip_if": [ "Production-quality code review", "Complex reasoning", "Long documents" ], "official_evidence": { "record_id": "gpt-5-mini", "api_model_id": "gpt-5-mini", "verified_date": "2026-06-12", "sources": { "release": "https://developers.openai.com/api/docs/models/gpt-5-mini", "pricing": "https://developers.openai.com/api/docs/models/gpt-5-mini", "context": "https://developers.openai.com/api/docs/models/gpt-5-mini" }, "pricing": { "inputPerM": 0.25, "outputPerM": 2, "cachedInputPerM": 0.025 }, "context": { "windowTokens": 400000, "maxOutputTokens": 128000 }, "benchmarks": {}, "notes": "Official price is $0.25/$2.00 (NOT $0.50/$4.00 as in the legacy benchr verified-table). Release date inferred from snapshot id gpt-5-mini-2025-08-07. No headline benchmark published on the model doc page (empty benchmarks object, not null figures)." } }, { "id": "gemini-3-6-flash", "name": "Gemini 3.6 Flash", "provider": "Google", "api_name": "gemini-3.6-flash", "type": "mid", "license": "proprietary", "released": "2026-07-21", "deprecated": false, "pricing": { "input_per_million": 1.5, "output_per_million": 7.5, "cache_input_per_million": 0.15, "batch_discount": null, "batch_input": 0.75, "batch_output": 3.75, "free_tier": true }, "context": { "max_tokens": 1048576, "effective_tokens": 720000, "max_output_tokens": 65536 }, "benchmarks": { "values": { "swe_bench_verified": 82, "lmsys_arena": 1390, "mmlu": 90, "humaneval": 90, "math": 88, "gpqa_diamond": null, "arc_agi_2": 15 }, "estimated_fields": [ "swe_bench_verified", "lmsys_arena", "mmlu", "humaneval", "math", "arc_agi_2" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 89, "reasoning": 87, "writing": 84, "vision": 92, "long_context": 90, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 190, "tokens_per_second": 305, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Output-heavy coding agents", "Google Search and Maps grounded workflows", "Stable Flash migrations from 2.x and preview endpoints", "Multimodal input with text output" ], "skip_if": [ "You need published official benchmark tables before procurement", "You want the cheapest subagent tier ? use Gemini 3.5 Flash-Lite", "You need image generation or Live API voice output" ], "official_evidence": { "record_id": "gemini-3-6-flash", "api_model_id": "gemini-3.6-flash", "verified_date": "2026-07-22", "sources": { "release": "https://ai.google.dev/gemini-api/docs/changelog", "pricing": "https://ai.google.dev/gemini-api/docs/pricing", "context": "https://ai.google.dev/gemini-api/docs/models/gemini-3.6-flash", "deprecations": "https://ai.google.dev/gemini-api/docs/deprecations" }, "pricing": { "inputPerM": 1.5, "outputPerM": 7.5, "batchInputPerM": 0.75, "batchOutputPerM": 3.75, "flexInputPerM": 0.75, "flexOutputPerM": 3.75, "priorityInputPerM": 2.7, "priorityOutputPerM": 13.5, "cachedInputPerM": 0.15, "freeApiTier": true, "note": "Standard input matches Gemini 3.5 Flash; output drops from $9.00 to $7.50 per 1M. Cache storage remains $1 per 1M-token-hour. Verified 2026-07-22 against ai.google.dev/gemini-api/docs/pricing." }, "context": { "windowTokens": 1048576, "maxOutputTokens": 65536 }, "benchmarks": { "Terminal-Bench 2.1": null, "GDPval-AA (Elo)": null, "MCP Atlas": null, "CharXiv Reasoning": null, "SWE-bench Verified": null, "GPQA Diamond": null }, "notes": "Google released Gemini 3.6 Flash as a stable GA model on July 21, 2026, with improved token efficiency and code/agentic planning claims. Official model page lists text/image/video/audio/PDF inputs, text output, 1,048,576 input tokens, 65,536 output tokens, caching, code execution, file search, function calling, Search/Maps grounding, structured outputs, thinking, URL context, and Computer Use support in preview. No official benchmark table found on the model page or release note, so hard benchmark fields are null." } }, { "id": "gemini-3-5-flash", "name": "Gemini 3.5 Flash", "provider": "Google", "api_name": "gemini-3.5-flash", "type": "mid", "license": "proprietary", "released": "2026-05-19", "deprecated": false, "pricing": { "input_per_million": 1.5, "output_per_million": 9, "cache_input_per_million": 0.15, "batch_discount": null, "free_tier": true }, "context": { "max_tokens": 1048576, "effective_tokens": 700000, "max_output_tokens": 65536 }, "benchmarks": { "values": { "swe_bench_verified": 80.6, "lmsys_arena": 1385, "mmlu": 89.5, "humaneval": 89, "math": 87.5, "gpqa_diamond": null, "arc_agi_2": 14.5 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 88, "reasoning": 86, "writing": 84, "vision": 92, "long_context": 90, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 195, "tokens_per_second": 289, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Coding agents at speed", "Parallel agent execution", "Multimodal tasks", "Default frontier-quality model" ], "skip_if": [ "You need the deepest single-call reasoning — use Gemini 3.1 Pro" ], "official_evidence": { "record_id": "gemini-3-5-flash", "api_model_id": "gemini-3.5-flash", "verified_date": "2026-05-31", "sources": { "release": "https://ai.google.dev/gemini-api/docs/deprecations", "pricing": "https://ai.google.dev/gemini-api/docs/pricing", "context": "https://ai.google.dev/gemini-api/docs/models/gemini-3.5-flash", "benchmarks": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/" }, "pricing": { "inputPerM": 1.5, "outputPerM": 9, "batchInputPerM": 0.75, "batchOutputPerM": 4.5, "cachedInputPerM": 0.15, "freeApiTier": true, "note": "Flat rate — no >200K context tier. Cached input $0.15/1M (90% off the $1.50 input rate; Google additionally charges $1 per 1M-token-hour of cache storage), verified 2026-06-15 against ai.google.dev/gemini-api/docs/pricing." }, "context": { "windowTokens": 1048576, "maxOutputTokens": 65536 }, "benchmarks": { "Terminal-Bench 2.1": 76.2, "GDPval-AA (Elo)": 1656, "MCP Atlas": 83.6, "CharXiv Reasoning": 84.2, "SWE-bench Verified": null, "GPQA Diamond": null }, "notes": "Released at Google I/O (May 19, 2026), GA. Flat $1.50/$9 (not context-tiered). Free API tier available. The launch blog publishes only the four agentic/coding/multimodal benchmarks above; no official SWE-bench/GPQA for 3.5 Flash, so those are null." } }, { "id": "gemini-3-1-pro", "name": "Gemini 3.1 Pro", "provider": "Google", "api_name": "gemini-3.1-pro", "type": "frontier", "license": "proprietary", "released": "2026-02", "deprecated": false, "pricing": { "input_per_million": 2, "output_per_million": 12, "input_per_million_over_200k": 4, "output_per_million_over_200k": 18, "cache_input_per_million": 0.2, "batch_discount": null }, "context": { "max_tokens": 1000000, "effective_tokens": 800000, "max_output_tokens": 64000 }, "benchmarks": { "values": { "swe_bench_verified": 80.6, "lmsys_arena": 1380, "mmlu": 91, "humaneval": 90, "math": 90, "gpqa_diamond": 94.3, "arc_agi_2": 17 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 84, "reasoning": 90, "writing": 84, "vision": 95, "long_context": 92, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 380, "tokens_per_second": 108, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Deep reasoning in the Gemini family", "Long-context vision work", "Workspace integration" ], "skip_if": [ "Coding agents — Flash is faster and cheaper", "Cost-sensitive workloads — note the over-200K price bump" ], "official_evidence": { "record_id": "gemini-3-1-pro", "api_model_id": "gemini-3.1-pro-preview", "verified_date": "2026-05-31", "sources": { "release": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/", "pricing": "https://ai.google.dev/gemini-api/docs/pricing", "context": "https://deepmind.google/models/model-cards/gemini-3-1-pro/", "benchmarks": "https://deepmind.google/models/model-cards/gemini-3-1-pro/" }, "pricing": { "inputPerM": 2, "outputPerM": 12, "inputAbove200kPerM": 4, "outputAbove200kPerM": 18, "batchInputPerM": 1, "batchOutputPerM": 6, "batchInputAbove200kPerM": 2, "batchOutputAbove200kPerM": 9, "cachedInputPerM": 0.2, "freeApiTier": false }, "context": { "windowTokens": 1000000, "maxOutputTokens": 64000 }, "benchmarks": { "ARC-AGI-2": 77.1, "GPQA Diamond": 94.3, "Humanity's Last Exam (with tools)": 51.4, "MMMU-Pro": 80.5, "SWE-bench Verified": 80.6, "MMMLU": 92.6 }, "notes": "Tiered pricing: the >200K-token tier doubles input ($4) and raises output ($18). Output price includes thinking tokens. No free API tier (free trial in AI Studio UI only). Context-caching: cached input $0.20/1M (90% off the $2 input rate; Google additionally charges $1 per 1M-token-hour of cache storage), verified 2026-06-15 against ai.google.dev/gemini-api/docs/pricing." } }, { "id": "grok-4-3", "name": "Grok 4.3", "provider": "xAI", "api_name": "grok-4-3", "type": "frontier", "license": "proprietary", "released": "2026-04", "deprecated": false, "pricing": { "input_per_million": 1.25, "output_per_million": 2.5, "cache_input_per_million": 0.2, "batch_discount": null }, "context": { "max_tokens": 1000000, "effective_tokens": 800000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 68, "lmsys_arena": 1370, "mmlu": 87, "humaneval": 87, "math": 84, "gpqa_diamond": null, "arc_agi_2": 10 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 82, "reasoning": 84, "writing": 80, "vision": 78, "long_context": 85, "multilingual": 75 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 300, "tokens_per_second": 120, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Real-time data access via X", "Very cheap output tokens", "Long-context tasks on a budget" ], "skip_if": [ "Coding-first agent work — others edge it out", "Non-English-heavy workloads" ], "official_evidence": { "record_id": "grok-4-3", "api_model_id": "grok-4.3", "verified_date": "2026-06-12", "sources": { "pricing": "https://docs.x.ai/developers/models/grok-4.3", "context": "https://docs.x.ai/developers/models/grok-4.3" }, "pricing": { "inputPerM": 1.25, "outputPerM": 2.5, "cachedInputPerM": 0.2 }, "context": { "windowTokens": 1000000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "xAI officially publishes price ($1.25/$2.50, cached $0.20), 1M context, and the model id only. It publishes NO official release date, max output, or numeric benchmarks for Grok 4.3 (only a qualitative non-hallucination claim) — those are null. Figures like 'Intelligence Index 53' or 'tau2-bench Telecom 98%' are third-party (Artificial Analysis / leaderboards), not official xAI numbers." } }, { "id": "deepseek-v4-pro", "name": "DeepSeek V4-Pro", "provider": "DeepSeek", "api_name": "deepseek-v4-pro", "type": "frontier-open", "license": "MIT", "released": "2026-04-24", "deprecated": false, "pricing": { "input_per_million": 0.435, "output_per_million": 0.87, "cache_input_per_million": 0.003625, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 384000 }, "benchmarks": { "values": { "swe_bench_verified": 80.6, "lmsys_arena": 1390, "mmlu": 92.8, "humaneval": 93.5, "math": 96.1, "gpqa_diamond": 90.1, "arc_agi_2": 17.5 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 93, "reasoning": 92, "writing": 84, "vision": null, "long_context": 86, "multilingual": 87 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 380, "tokens_per_second": 95, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Frontier-grade open-weight coding", "Math-heavy work", "Self-hosted production at near-zero API cost" ], "skip_if": [ "You need vision/multimodal", "You can't manage GPU hosting" ], "official_evidence": { "record_id": "deepseek-v4-pro", "api_model_id": "deepseek-v4-pro", "verified_date": "2026-06-12", "sources": { "release": "https://api-docs.deepseek.com/news/news260424", "pricing": "https://api-docs.deepseek.com/quick_start/pricing", "license": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "benchmarks": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, "pricing": { "inputPerM": 0.435, "outputPerM": 0.87, "cacheHitInputPerM": 0.003625, "selfHost": true, "priceChangeNote": "These rates are PERMANENT, not a temporary low. A 75% promo ends 2026-05-31 15:59 UTC, after which DeepSeek officially sets the price to 1/4 of the original — the SAME level as the promo. The price does NOT rise." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 384000 }, "benchmarks": { "SWE-bench Verified": 80.6, "SWE-bench Pro": 55.4, "GPQA Diamond": 90.1, "LiveCodeBench": 93.5, "Terminal-Bench 2.0": 67.9, "MMLU-Pro (Max)": 87.5, "MRCR 1M": 83.5, "Codeforces (rating)": 3206 }, "notes": "RE-VERIFIED DIRECTLY on the official pricing page on 2026-06-03 (after the 2026-05-31 promo end): live price = $0.435 in / $0.87 out, cache-hit $0.003625 — the post-promo level held. Do NOT record a post-promo rise to $1.74/$3.48; that figure appears only in stale third-party search snippets, not on the live DeepSeek page. SWE-bench 80.6% is DeepSeek's own vendor-reported figure. Open weights, MIT." } }, { "id": "deepseek-v4-flash", "name": "DeepSeek V4-Flash", "provider": "DeepSeek", "api_name": "deepseek-v4-flash", "type": "open", "license": "MIT", "released": "2026-04-24", "deprecated": false, "pricing": { "input_per_million": 0.14, "output_per_million": 0.28, "cache_input_per_million": 0.0028, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 1000000, "effective_tokens": 600000, "max_output_tokens": 384000 }, "benchmarks": { "values": { "swe_bench_verified": 79, "lmsys_arena": 1350, "mmlu": 88, "humaneval": 87.5, "math": 89, "gpqa_diamond": 88.1, "arc_agi_2": 11 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 86, "reasoning": 86, "writing": 80, "vision": null, "long_context": 88, "multilingual": 86 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 290, "tokens_per_second": 135, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Cheapest frontier-grade inference on the market", "Self-hosted at low cost", "Volume coding tasks" ], "skip_if": [ "You need frontier-tier reasoning depth", "Vision-first workflows" ], "official_evidence": { "record_id": "deepseek-v4-flash", "api_model_id": "deepseek-v4-flash", "verified_date": "2026-06-12", "sources": { "release": "https://api-docs.deepseek.com/news/news260424", "pricing": "https://api-docs.deepseek.com/quick_start/pricing", "license": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "benchmarks": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash" }, "pricing": { "inputPerM": 0.14, "outputPerM": 0.28, "cacheHitInputPerM": 0.0028, "selfHost": true }, "context": { "windowTokens": 1000000, "maxOutputTokens": 384000 }, "benchmarks": { "SWE-bench Verified": 79, "GPQA Diamond": 88.1, "LiveCodeBench": 91.6, "MMLU-Pro (Think Max)": 86.2, "HMMT 2026 Feb": 94.8, "MRCR 1M": 78.7 }, "notes": "Flash pricing is standard (no promo). Cache-hit input reduced to $0.0028 (1/10) effective 2026-04-26. Open weights, MIT. Benchmarks are DeepSeek's own model-card figures (Instruct / Think Max mode)." } }, { "id": "kimi-k2-6", "name": "Kimi K2.6", "provider": "Moonshot AI", "api_name": "kimi-k2-6", "type": "frontier-open", "license": "Modified MIT", "released": "2026-04-20", "deprecated": false, "pricing": { "input_per_million": 0.95, "output_per_million": 4, "cache_input_per_million": 0.16, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 262144, "effective_tokens": 200000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 80.2, "lmsys_arena": 1355, "mmlu": 85, "humaneval": 84, "math": 82, "gpqa_diamond": 90.5, "arc_agi_2": 9 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 80, "reasoning": 78, "writing": 74, "vision": null, "long_context": 78, "multilingual": 82 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 350, "tokens_per_second": 100, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Mid-range open-weight option", "Multilingual tasks", "Cost-efficient API with self-hosting option" ], "skip_if": [ "You need the deepest reasoning or best coding", "Vision workloads" ], "official_evidence": { "record_id": "kimi-k2-6", "api_model_id": "kimi-k2.6", "verified_date": "2026-06-12", "sources": { "release": null, "pricing": "https://platform.kimi.ai/docs/pricing/chat-k26", "license": "https://huggingface.co/moonshotai/Kimi-K2.6/blob/main/LICENSE", "benchmarks": "https://huggingface.co/moonshotai/Kimi-K2.6" }, "pricing": { "inputPerM": 0.95, "outputPerM": 4, "cacheHitInputPerM": 0.16, "selfHost": true }, "context": { "windowTokens": 262144, "maxOutputTokens": null }, "benchmarks": { "SWE-bench Verified": 80.2, "SWE-bench Multilingual": 76.7, "SWE-bench Pro": 58.6, "Terminal-Bench 2.0": 66.7, "LiveCodeBench v6": 89.6, "AIME 2026": 96.4, "GPQA Diamond": 90.5, "Humanity's Last Exam (with tools)": 54, "OSWorld-Verified": 73.1 }, "notes": "Official release date not stated on any official Moonshot page (third-party says Apr 20, 2026), so releaseDate is null. Context 262,144 (256K) confirmed on both pricing page and model card. Input $0.95 is cache-miss; cache-hit $0.16. Model card body: 1T total / 32B active (the org listing page rounds to 1.1T). Benchmarks are Moonshot's own model-card figures." } }, { "id": "mistral-large-3", "name": "Mistral Large 3", "provider": "Mistral", "api_name": "mistral-large-2512", "type": "frontier-open", "license": "Apache 2.0", "released": "2025-12-02", "deprecated": false, "pricing": { "input_per_million": 0.5, "output_per_million": 1.5, "cache_input_per_million": null, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 256000, "effective_tokens": 200000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 62, "lmsys_arena": 1340, "mmlu": 84, "humaneval": 83, "math": 79, "gpqa_diamond": null, "arc_agi_2": 7.5 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 78, "reasoning": 79, "writing": 78, "vision": null, "long_context": 76, "multilingual": 88 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 280, "tokens_per_second": 115, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Apache-licensed production workloads", "European data residency", "Very cheap inference with decent reasoning" ], "skip_if": [ "Coding at frontier quality", "Vision or multimodal workflows" ], "official_evidence": { "record_id": "mistral-large-3", "api_model_id": "mistral-large-2512", "verified_date": "2026-06-12", "sources": { "release": "https://docs.mistral.ai/models/mistral-large-3-25-12", "pricing": "https://docs.mistral.ai/models/mistral-large-3-25-12", "license": "https://mistral.ai/news/mistral-3/", "benchmarks": "https://mistral.ai/news/mistral-3/" }, "pricing": { "inputPerM": 0.5, "outputPerM": 1.5, "cacheHitInputPerM": null, "selfHost": true }, "context": { "windowTokens": 256000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "Open weights, Apache-2.0. Mistral's announcement gives only relative/leaderboard claims for Large 3, no discrete official per-benchmark scores (the ~85% AIME figure on the page belongs to a smaller reasoning variant, NOT Large 3) — so benchmarks is intentionally empty, not guessed. Cache-hit price not published." } }, { "id": "mistral-medium-3-5", "name": "Mistral Medium 3.5", "provider": "Mistral", "api_name": "mistral-medium-3-5", "type": "frontier-open", "license": "Modified MIT", "released": "2026-04", "deprecated": false, "pricing": { "input_per_million": 1.5, "output_per_million": 7.5, "cache_input_per_million": null, "self_hosted": true }, "context": { "max_tokens": 256000, "effective_tokens": 200000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 77.6, "lmsys_arena": 1368, "mmlu": 89, "humaneval": 88, "math": 86, "gpqa_diamond": null, "arc_agi_2": 12 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 86, "reasoning": 87, "writing": 86, "vision": 84, "long_context": 84, "multilingual": 92 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 320, "tokens_per_second": 105, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "European data residency", "Multimodal + reasoning in one model", "Self-hosted on 4 GPUs" ], "skip_if": [ "You need MoE efficiency for ultra-cheap inference" ], "official_evidence": { "record_id": "mistral-medium-3-5", "api_model_id": "mistral-medium-3-5", "verified_date": "2026-06-23", "sources": { "release": "https://mistral.ai/news/vibe-remote-agents-mistral-medium-3-5/", "pricing": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "license": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "context": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04" }, "pricing": { "inputPerM": 1.5, "outputPerM": 7.5, "cacheHitInputPerM": null, "selfHost": true }, "context": { "windowTokens": 256000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "Verified 2026-06-03, re-confirmed 2026-06-23 against the official Mistral model card (docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04): Modified MIT license, open-weight (self-hostable on ~4 GPUs) AND offered as a hosted API, dense 128B, 256K context, $1.50/$7.50 per 1M. Official release date is April 28, 2026 (filled 2026-06-23; previously null). Max output tokens not stated on the card (null). No discrete official per-benchmark scores published — benchmarks intentionally empty." } }, { "id": "qwen-3-6-27b", "name": "Qwen3.6-27B", "provider": "Alibaba", "api_name": "Qwen/Qwen3.6-27B", "type": "open", "license": "Apache 2.0", "released": "2026-04-22", "deprecated": false, "pricing": { "input_per_million": null, "output_per_million": null, "self_hosted": true }, "context": { "max_tokens": 262144, "effective_tokens": 200000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 77.2, "lmsys_arena": 1370, "mmlu": 89.5, "humaneval": 89, "math": 87, "gpqa_diamond": 87.8, "arc_agi_2": 12 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 88, "reasoning": 86, "writing": 82, "vision": 82, "long_context": 84, "multilingual": 95 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 340, "tokens_per_second": 105, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Multilingual coding (Chinese, Japanese, Korean, Arabic)", "Local inference on consumer GPUs — dense 27B", "Tool-use agent loops at zero API cost" ], "skip_if": [ "You want a managed hosted API — these are open weights you self-host", "Absolute deepest single-language reasoning" ], "official_evidence": { "record_id": "qwen-3-6-27b", "api_model_id": "Qwen/Qwen3.6-27B", "verified_date": "2026-05-31", "sources": { "release": "https://huggingface.co/Qwen/Qwen3.6-27B", "license": "https://huggingface.co/Qwen/Qwen3.6-27B", "benchmarks": "https://huggingface.co/Qwen/Qwen3.6-27B" }, "pricing": { "selfHost": true, "inputPerM": null, "outputPerM": null }, "context": { "windowTokens": 262144, "windowTokensExtended": 1010000, "maxOutputTokens": null }, "benchmarks": { "SWE-bench Verified": 77.2, "SWE-bench Pro": 53.5, "Terminal-Bench 2.0": 59.3, "MMLU-Pro": 86.2, "GPQA Diamond": 87.8, "AIME 2026": 94.1, "MMMU": 82.9 }, "notes": "Open weight, Apache-2.0, self-host (no per-token list price). 262,144 native context, extensible ~1M via YaRN. Benchmarks from the official HF model card." } }, { "id": "llama-4-maverick", "name": "Llama 4 Maverick", "provider": "Meta", "api_name": "llama-4-maverick", "type": "frontier-open", "license": "Llama 4 Community", "released": "2025-04-05", "deprecated": false, "pricing": { "input_per_million": null, "output_per_million": null, "self_hosted": true }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 66, "lmsys_arena": 1340, "mmlu": 84, "humaneval": 83, "math": 80, "gpqa_diamond": 69.8, "arc_agi_2": 9 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 80, "reasoning": 81, "writing": 76, "vision": 80, "long_context": 85, "multilingual": 80 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 300, "tokens_per_second": 120, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Best open-weight frontier model", "Self-hosted production at zero licensing cost", "Multimodal at no API cost" ], "skip_if": [ "You need the very best reasoning or coding", "You can't manage GPU infrastructure" ], "official_evidence": { "record_id": "llama-4-maverick", "api_model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct", "verified_date": "2026-05-31", "sources": { "release": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "license": "https://www.llama.com/llama4/license/", "context": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama4/", "benchmarks": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" }, "pricing": { "selfHost": true, "inputPerM": null, "outputPerM": null }, "context": { "windowTokens": 1000000, "maxOutputTokens": null }, "benchmarks": { "MMLU-Pro (0-shot)": 80.5, "GPQA Diamond": 69.8, "LiveCodeBench": 43.4, "MGSM": 92.3 }, "notes": "Open weight; $0 to self-host. 1M-token context, 128 experts. Llama 4 Behemoth (288B active / ~2T total) was only ever previewed as 'still training' and was never released — do not list specs for it as a usable model." } }, { "id": "llama-4-scout", "name": "Llama 4 Scout", "provider": "Meta", "api_name": "llama-4-scout", "type": "open", "license": "Llama 4 Community", "released": "2025-04-05", "deprecated": false, "pricing": { "input_per_million": null, "output_per_million": null, "self_hosted": true }, "context": { "max_tokens": 10000000, "effective_tokens": 2000000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 56, "lmsys_arena": 1295, "mmlu": 80, "humaneval": 76, "math": 74, "gpqa_diamond": 57.2, "arc_agi_2": 6 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 73, "reasoning": 74, "writing": 70, "vision": 75, "long_context": 92, "multilingual": 78 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 180, "tokens_per_second": 180, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Ultra-long-context tasks (10M token window)", "Fast self-hosted inference", "Free multimodal at scale" ], "skip_if": [ "Deep reasoning tasks", "When frontier coding quality is needed" ], "official_evidence": { "record_id": "llama-4-scout", "api_model_id": "meta-llama/Llama-4-Scout-17B-16E", "verified_date": "2026-05-31", "sources": { "release": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "license": "https://www.llama.com/llama4/license/", "context": "https://www.llama.com/docs/model-cards-and-prompt-formats/llama4/", "benchmarks": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E" }, "pricing": { "selfHost": true, "inputPerM": null, "outputPerM": null }, "context": { "windowTokens": 10000000, "maxOutputTokens": null }, "benchmarks": { "MMLU-Pro (0-shot)": 74.3, "GPQA Diamond": 57.2, "MMMU": 73.4, "MathVista": 73.7, "LiveCodeBench": 32.8 }, "notes": "Open weight; $0 to self-host (Meta sells no API). Community License: >700M MAU needs a separate Meta license; not OSI-approved. 10M-token context. Benchmarks from Meta's official instruction-tuned model card." } }, { "id": "phi-4", "name": "Phi-4", "provider": "Microsoft", "api_name": "phi-4", "type": "small-open", "license": "MIT", "released": "2024-12-12", "deprecated": false, "pricing": { "input_per_million": null, "output_per_million": null, "self_hosted": true }, "context": { "max_tokens": 16000, "effective_tokens": 14000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 30, "lmsys_arena": 1180, "mmlu": 84.8, "humaneval": 82.6, "math": 80.4, "gpqa_diamond": 56.1, "arc_agi_2": 5 }, "estimated_fields": [ "swe_bench_verified" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 70, "reasoning": 78, "writing": 74, "vision": null, "long_context": 60, "multilingual": 72 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 95, "tokens_per_second": 220, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Local inference on consumer hardware", "Edge deployment", "Reasoning at tiny scale" ], "skip_if": [ "Long-context tasks", "Production-quality writing or coding" ], "official_evidence": { "record_id": "phi-4", "api_model_id": "microsoft/phi-4", "verified_date": "2026-06-03", "sources": { "release": "https://huggingface.co/microsoft/phi-4", "license": "https://huggingface.co/microsoft/phi-4", "context": "https://huggingface.co/microsoft/phi-4", "benchmarks": "https://www.microsoft.com/en-us/research/publication/phi-4-technical-report/" }, "pricing": { "selfHost": true, "inputPerM": null, "outputPerM": null }, "context": { "windowTokens": 16000, "maxOutputTokens": null }, "benchmarks": { "GPQA Diamond": 56.1, "MMLU": 84.8, "HumanEval": 82.6, "MATH": 80.4 }, "notes": "Verified 2026-06-03: MIT-licensed, 14B dense, 16K context, self-host only (no Microsoft per-token API; available on Azure AI Foundry + Hugging Face). Released 2024-12-12. Benchmarks from the Phi-4 technical report / model card." } }, { "id": "grok-4-5", "name": "Grok 4.5", "provider": "xAI", "api_name": "grok-4.5", "type": "frontier", "license": "proprietary", "released": "2026-07-08", "deprecated": false, "pricing": { "input_per_million": 2, "output_per_million": 6, "cache_input_per_million": 0.5, "batch_discount": null }, "context": { "max_tokens": 500000, "effective_tokens": 400000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 84, "lmsys_arena": 1415, "mmlu": 91, "humaneval": 93, "math": 91, "gpqa_diamond": 88, "arc_agi_2": 18 }, "estimated_fields": [ "swe_bench_verified", "gpqa_diamond" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 93, "reasoning": 92, "writing": 87, "vision": 85, "long_context": 82, "multilingual": 81 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 260, "tokens_per_second": 135, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Coding and agentic workflows on xAI", "Lower output cost than GPT-5.5 class models", "Search-augmented knowledge work" ], "skip_if": [ "You need 1M context — Grok 4.3 has the larger window", "You need official benchmark tables before procurement" ], "official_evidence": { "record_id": "grok-4-5", "api_model_id": "grok-4.5", "verified_date": "2026-07-13", "sources": { "release": "https://docs.x.ai/developers/release-notes", "pricing": "https://docs.x.ai/developers/pricing", "context": "https://docs.x.ai/developers/models" }, "pricing": { "inputPerM": 2, "outputPerM": 6, "cachedInputPerM": 0.5 }, "context": { "windowTokens": 500000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "xAI release notes list Grok 4.5 on July 8, 2026 for coding, agentic tasks, and knowledge work. Live models/pricing docs list model id grok-4.5, 500K context, $2 input / $6 output / $0.50 cached input per 1M tokens, configurable reasoning, and a February 1, 2026 knowledge cutoff. xAI publishes no numeric benchmark table or max output limit on the checked docs." } }, { "id": "glm-5-2", "name": "GLM-5.2", "provider": "Z.AI", "api_name": "glm-5.2", "type": "frontier-open", "license": "open-weight / hosted API", "released": "2026-06-16", "deprecated": false, "pricing": { "input_per_million": 1.4, "output_per_million": 4.4, "cache_input_per_million": 0.26, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 1000000, "effective_tokens": 750000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 83, "lmsys_arena": 1380, "mmlu": 89, "humaneval": 91, "math": 90, "gpqa_diamond": 87, "arc_agi_2": 16 }, "estimated_fields": [ "swe_bench_verified", "gpqa_diamond" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 91, "reasoning": 89, "writing": 82, "vision": null, "long_context": 94, "multilingual": 86 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 420, "tokens_per_second": 95, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Long-horizon coding with 1M context", "Teams that want hosted API plus open-weight optionality", "Cost-sensitive agent engineering" ], "skip_if": [ "You need broad Western enterprise platform support", "You need official public benchmark numbers for every metric" ], "official_evidence": { "record_id": "glm-5-2", "api_model_id": "glm-5.2", "verified_date": "2026-07-13", "sources": { "release": "https://docs.z.ai/release-notes/new-released", "pricing": "https://docs.z.ai/guides/overview/pricing", "context": "https://docs.z.ai/guides/llm/glm-5.2", "migration": "https://docs.z.ai/guides/overview/migrate-to-glm-new" }, "pricing": { "inputPerM": 1.4, "outputPerM": 4.4, "cachedInputPerM": 0.26, "cacheStorage": "Limited-time free" }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": {}, "notes": "Z.AI's release notes list GLM-5.2 on 2026-06-16 with 1M lossless context and stronger long-horizon/coding performance. Model docs list 1M context and 128K maximum output tokens. Pricing docs list $1.40 input, $0.26 cached input, and $4.40 output per 1M tokens; cached input storage is listed as limited-time free. Official docs describe open-source SOTA performance but the checked docs page does not restate a precise license string." } }, { "id": "minimax-m3", "name": "MiniMax M3", "provider": "MiniMax", "api_name": "MiniMax-M3", "type": "frontier-open", "license": "proprietary hosted API", "released": "2026-06-01", "deprecated": false, "pricing": { "input_per_million": 0.3, "output_per_million": 1.2, "cache_input_per_million": 0.06, "tier_prices_per_million": [ 0.3, 1.2, 0.6, 2.4, 0.45, 1.8, 0.9, 3.6 ], "batch_discount": null }, "context": { "max_tokens": 1000000, "effective_tokens": 750000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 82, "lmsys_arena": 1375, "mmlu": 88, "humaneval": 90, "math": 89, "gpqa_diamond": 86, "arc_agi_2": 15 }, "estimated_fields": [ "swe_bench_verified", "gpqa_diamond" ], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 90, "reasoning": 88, "writing": 80, "vision": 85, "long_context": 94, "multilingual": 84 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": 360, "tokens_per_second": 110, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Very low-cost long-context coding agents", "1M-context workflows under $1/M input", "Multimodal coding and tool-use experiments" ], "skip_if": [ "You need a mature Western provider ecosystem", "You need official benchmark tables before rollout" ], "official_evidence": { "record_id": "minimax-m3", "api_model_id": "MiniMax-M3", "verified_date": "2026-07-13", "sources": { "release": "https://platform.minimax.io/docs/release-notes/models", "pricing": "https://platform.minimax.io/docs/guides/pricing-paygo", "context": "https://platform.minimax.io/docs/guides/text-generation" }, "pricing": { "inputPerM": 0.3, "outputPerM": 1.2, "cacheHitInputPerM": 0.06, "longContextInputPerM": 0.6, "longContextOutputPerM": 2.4, "longContextCacheHitInputPerM": 0.12, "priorityInputPerM": 0.45, "priorityOutputPerM": 1.8, "priorityLongContextInputPerM": 0.9, "priorityLongContextOutputPerM": 3.6 }, "context": { "windowTokens": 1000000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "MiniMax release notes list MiniMax-M3 on June 1, 2026. The model overview/invocation docs describe it as a frontier multimodal coding model with 1M context. Pay-as-you-go pricing shows permanent 50% off rates: standard <=512K input $0.30/$1.20 per 1M with $0.06 prompt-cache read; >512K input $0.60/$2.40 with $0.12 cache read. Priority is 1.5x standard. No official max output or benchmark table was found on the checked pages." } } ] }