{ "meta": { "api_version": "v1", "endpoint": "/api/v1/models", "updated": "2026-09-02", "verified_date": "2026-09-02", "official_record_verified_date": "2026-09-02", "total": 38, "offset": 0, "limit": 50, "returned": 38, "provenance": "Pricing, context, release, and license fields are factual records with official_evidence when matched. Capabilities and latency are explicitly labelled benchr editorial data." }, "data": [ { "id": "qwen-3-6-27b", "name": "Qwen3.6-27B", "provider": "Alibaba (Qwen)", "api_name": "Qwen/Qwen3.6-27B", "type": "open", "license": "Apache-2.0", "released": "2026-04-22", "deprecated": false, "pricing": { "input_per_million": null, "output_per_million": null, "self_hosted": true }, "context": { "max_tokens": 262144, "effective_tokens": 200000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 77.2, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 87.8, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 88, "reasoning": 86, "writing": 82, "vision": 82, "long_context": 84, "multilingual": 95 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Multilingual coding (Chinese, Japanese, Korean, Arabic)", "Local inference on consumer GPUs — dense 27B", "Tool-use agent loops at zero API cost" ], "skip_if": [ "You want a managed hosted API — these are open weights you self-host", "Absolute deepest single-language reasoning" ], "official_evidence": { "record_id": "qwen-3-6-27b", "api_model_id": "Qwen/Qwen3.6-27B", "verified_date": "2026-05-31", "sources": { "release": "https://huggingface.co/Qwen/Qwen3.6-27B", "license": "https://huggingface.co/Qwen/Qwen3.6-27B", "context": "https://huggingface.co/Qwen/Qwen3.6-27B", "benchmarks": "https://huggingface.co/Qwen/Qwen3.6-27B" }, "pricing": { "selfHost": true, "inputPerM": null, "outputPerM": null }, "context": { "windowTokens": 262144, "windowTokensExtended": 1010000, "maxOutputTokens": null }, "benchmarks": { "SWE-bench Verified": 77.2, "SWE-bench Pro": 53.5, "Terminal-Bench 2.0": 59.3, "MMLU-Pro": 86.2, "GPQA Diamond": 87.8, "AIME 2026": 94.1, "MMMU": 82.9 }, "notes": "Open weight, Apache-2.0, self-host (no per-token list price). 262,144 native context, extensible ~1M via YaRN. Benchmarks from the official HF model card." } }, { "id": "claude-haiku-4-5", "name": "Claude Haiku 4.5", "provider": "Anthropic", "api_name": "claude-haiku-4-5", "type": "small", "license": "proprietary", "released": "2025-10-15", "deprecated": false, "pricing": { "input_per_million": 1, "output_per_million": 5, "cache_input_per_million": 0.1, "batch_discount": 0.5, "batch_input": 0.5, "batch_output": 2.5 }, "context": { "max_tokens": 200000, "effective_tokens": 150000, "max_output_tokens": 64000 }, "benchmarks": { "values": { "swe_bench_verified": 73.3, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 75, "reasoning": 76, "writing": 80, "vision": 72, "long_context": 82, "multilingual": 78 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "High-volume simple tasks", "Real-time chat", "Classification, routing, extraction" ], "skip_if": [ "Complex reasoning needed", "Long-document analysis" ], "official_evidence": { "record_id": "claude-haiku-4-5", "api_model_id": "claude-haiku-4-5", "verified_date": "2026-05-31", "sources": { "release": "https://www.anthropic.com/news/claude-haiku-4-5", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/news/claude-haiku-4-5" }, "pricing": { "inputPerM": 1, "outputPerM": 5, "batchInputPerM": 0.5, "batchOutputPerM": 2.5, "cachedInputPerM": 0.1 }, "context": { "windowTokens": 200000, "maxOutputTokens": 64000, "maxOutputTokensBeta": null }, "benchmarks": { "SWE-bench Verified": 73.3, "Terminal-Bench (no thinking)": 40.21, "Terminal-Bench (32K thinking)": 41.75, "GPQA Diamond": null, "OSWorld-Verified": null, "AIME 2025": null, "MMMLU": null }, "notes": "Pinned snapshot claude-haiku-4-5-20251001. Pricing (incl. cache-hit $0.10) and tentative retirement floor (not sooner than October 15, 2026) re-confirmed on official pages June 12, 2026; the 200K context window stands from the May 31 verification. SWE-bench Verified 73.3% (avg of 50 trials, 128K thinking budget) and Terminal-Bench are the only headline numbers Anthropic publishes as readable official text; GPQA/OSWorld/AIME/MMMLU appear only inside a launch-page image, so they are null (not guessed)." } }, { "id": "claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "provider": "Anthropic", "api_name": "claude-sonnet-4-6", "type": "mid", "license": "proprietary", "released": "2026-02-17", "deprecated": false, "pricing": { "input_per_million": 3, "output_per_million": 15, "cache_input_per_million": 0.3, "batch_discount": 0.5, "batch_input": 1.5, "batch_output": 7.5 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 64000 }, "benchmarks": { "values": { "swe_bench_verified": 79.6, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 89.9, "arc_agi_2": 58.3 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 88, "reasoning": 87, "writing": 89, "vision": 80, "long_context": 91, "multilingual": 86 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Production default", "Cost-effective coding", "Bulk content tasks", "Daily-driver API workloads" ], "skip_if": [ "You need frontier-grade reasoning" ], "official_evidence": { "record_id": "claude-sonnet-4-6", "api_model_id": "claude-sonnet-4-6", "verified_date": "2026-07-30", "sources": { "release": "https://www.anthropic.com/news/claude-sonnet-4-6", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www-cdn.anthropic.com/bbd8ef16d70b7a1665f14f306ee88b53f686aa75/Claude%20Sonnet%204.6%20System%20Card.pdf" }, "pricing": { "inputPerM": 3, "outputPerM": 15, "batchInputPerM": 1.5, "batchOutputPerM": 7.5, "cachedInputPerM": 0.3 }, "context": { "windowTokens": 1000000, "maxOutputTokens": 64000, "maxOutputTokensBeta": 300000 }, "benchmarks": { "SWE-bench Verified": 79.6, "SWE-bench Multilingual": 75.9, "Terminal-Bench 2.0": 59.1, "OSWorld-Verified": 72.5, "GPQA Diamond": 89.9, "MMMLU": 89.3, "AIME 2025 (no tools)": 95.6, "Humanity's Last Exam (no tools)": 33.2, "Humanity's Last Exam (with tools)": 49, "ARC-AGI-2": 58.3, "tau2-bench Telecom": 97.9, "tau2-bench Retail": 91.7, "GDPval-AA (Elo)": 1633 }, "notes": "No fast-mode tier (fast mode is Opus-only). Cache-hit input $0.30 confirmed on the official pricing page June 12, 2026; official tentative retirement floor: not sooner than February 17, 2027. Benchmark values read from the official Sonnet 4.6 System Card (Table 2.1.A). SWE-bench Verified 79.6% averaged over 25 trials (80.2% with a stated prompt modification). Anthropic flags possible AIME 2025 contamination. Anthropic reports MMMLU, not plain MMLU." } }, { "id": "claude-opus-4-7", "name": "Claude Opus 4.7", "provider": "Anthropic", "api_name": "claude-opus-4-7", "type": "frontier", "license": "proprietary", "released": "2026-04-16", "deprecated": false, "pricing": { "input_per_million": 5, "output_per_million": 25, "cache_input_per_million": 0.5, "batch_discount": 0.5, "batch_input": 2.5, "batch_output": 12.5 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 87.6, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 94.2, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 96, "reasoning": 96, "writing": 90, "vision": 85, "long_context": 94, "multilingual": 89 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Complex coding tasks", "Long-document analysis", "Production agent loops", "Architecture decisions" ], "skip_if": [ "You need cheap volume", "Simple summarization", "Sonnet covers your workload" ], "official_evidence": { "record_id": "claude-opus-4-7", "api_model_id": "claude-opus-4-7", "verified_date": "2026-06-12", "sources": { "release": "https://www.anthropic.com/news/claude-opus-4-7", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/claude-opus-4-7-system-card" }, "pricing": { "inputPerM": 5, "outputPerM": 25, "fastModeInputPerM": 30, "fastModeOutputPerM": 150, "batchInputPerM": 2.5, "batchOutputPerM": 12.5, "cachedInputPerM": 0.5 }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000, "maxOutputTokensBeta": 300000 }, "benchmarks": { "SWE-bench Verified": 87.6, "SWE-bench Pro": 64.3, "SWE-bench Multilingual": 80.5, "SWE-bench Multimodal": 34.5, "Terminal-Bench 2.0": 69.4, "GPQA Diamond": 94.2 }, "notes": "Fast mode on 4.7 is $30/$150 (3x more expensive than Opus 4.8's $10/$50 fast mode). Cache-hit input $0.50 confirmed on the official pricing page June 12, 2026; official tentative retirement floor: not sooner than April 16, 2027. Opus 4.8 later restated Opus 4.7's OSWorld score upward (~82.x) after a test-harness fix." } }, { "id": "claude-opus-4-8", "name": "Claude Opus 4.8", "provider": "Anthropic", "api_name": "claude-opus-4-8", "type": "frontier", "license": "proprietary", "released": "2026-05-28", "deprecated": false, "pricing": { "input_per_million": 5, "output_per_million": 25, "cache_input_per_million": 0.5, "batch_discount": 0.5, "fast_mode_input": 10, "fast_mode_output": 50, "batch_input": 2.5, "batch_output": 12.5 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 88.6, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 93.6, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 97, "reasoning": 96, "writing": 91, "vision": 86, "long_context": 94, "multilingual": 90 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Highest-stakes coding tasks", "Complex multi-step agents", "Architecture decisions", "Production SWE-bench-level work" ], "skip_if": [ "You need cheap volume", "Sonnet handles your workload", "Speed is the priority — use Fast Mode instead" ], "official_evidence": { "record_id": "claude-opus-4-8", "api_model_id": "claude-opus-4-8", "verified_date": "2026-06-12", "sources": { "release": "https://www.anthropic.com/news/claude-opus-4-8", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/claude-opus-4-8-system-card" }, "pricing": { "inputPerM": 5, "outputPerM": 25, "fastModeInputPerM": 10, "fastModeOutputPerM": 50, "batchInputPerM": 2.5, "batchOutputPerM": 12.5, "cachedInputPerM": 0.5 }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000, "maxOutputTokensBeta": 300000 }, "benchmarks": { "SWE-bench Verified": 88.6, "SWE-bench Pro": 69.2, "SWE-bench Multilingual": 84.4, "SWE-bench Multimodal": 38.4, "Terminal-Bench 2.1": 74.6, "GPQA Diamond": 93.6, "OSWorld-Verified": 83.4, "BrowseComp (single-agent)": 84.3, "Humanity's Last Exam (no tools)": 49.8, "GDPval-AA (Elo)": 1890, "ARC-AGI-2": null }, "notes": "Standard price unchanged from Opus 4.7. The $10/$50 figure is the optional fast-mode rate (~2.5x output speed), NOT the base price; base is $5/$25. Cache-hit input $0.50 (0.1x base) now explicit on the official pricing page (confirmed June 12, 2026). Official tentative retirement floor: not sooner than May 28, 2027. ARC-AGI-2 is not in the official Opus 4.8 headline summary table, so it is null (do not state one). New Opus 4.7+ tokenizer can use up to ~35% more tokens per the pricing page." } }, { "id": "claude-fable-5", "name": "Claude Fable 5", "provider": "Anthropic", "api_name": "claude-fable-5", "type": "frontier", "license": "proprietary", "released": "2026-06-09", "deprecated": false, "pricing": { "input_per_million": 10, "output_per_million": 50, "cache_input_per_million": 1 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": null, "swe_bench_pro": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 99, "reasoning": 98, "writing": 95, "vision": 94, "long_context": 95, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "The hardest long-horizon agentic coding", "Large-codebase migrations", "Frontier research and finance work", "Vision-driven agent loops" ], "skip_if": [ "Price matters — Opus 4.8 is half the cost", "Offensive-security or bio work — classifiers return an explicit refusal; another-model retry requires configured application logic", "Routine chat and drafting" ], "official_evidence": { "record_id": "claude-fable-5", "api_model_id": "claude-fable-5", "verified_date": "2026-09-02", "sources": { "release": "https://www.anthropic.com/news/claude-fable-5-mythos-5", "pricing": "https://platform.claude.com/docs/en/about-claude/models/overview", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www-cdn.anthropic.com/2f9323abbcc4abe219577539efe19a623c9ca2bd/Claude%20Fable%205%20%26%20Claude%20Mythos%205%20System%20Card.pdf", "status": "https://platform.claude.com/docs/en/about-claude/model-deprecations" }, "pricing": { "inputPerM": 10, "outputPerM": 50, "cachedInputPerM": 1, "note": "Prompt caching = 90% input discount per the launch announcement. Included free on Pro/Max/Team/seat-based Enterprise June 9-22, 2026; usage credits from June 23." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": { "SWE-bench Pro": null, "SWE-bench Verified": null, "GPQA Diamond": null, "note": "No Fable-only score is recorded here. Anthropic's public launch graphic combines Fable 5 and Mythos 5 and reports the higher configuration per row, so its 80.3 SWE-bench Pro figure must not be represented as a Fable-only result." }, "notes": "Generally available from June 9, 2026. Anthropic describes Fable 5 and Mythos 5 as two configurations of the same underlying model: Fable is the general-use configuration with added safeguards in high-risk biology and cybersecurity domains, while Mythos is restricted to approved Project Glasswing partners. A U.S. export-control directive on June 12 required Anthropic to block access by foreign nationals; because nationality could not be verified per request, Anthropic suspended both models globally. The controls were lifted June 30 and Fable returned globally July 1. Fable requires 30-day data retention for safety monitoring. Combined Fable/Mythos benchmark graphics are not treated as Fable-only scores. Rechecked September 2, 2026: Fable 5 remains Active; Fable 5.1 is an additional API model rather than a retirement notice for this version." } }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "provider": "Anthropic", "api_name": "claude-sonnet-5", "type": "frontier", "license": "proprietary", "released": "2026-07-01", "deprecated": false, "pricing": { "input_per_million": 2, "output_per_million": 10, "cache_input_per_million": 0.2, "batch_discount": 0.5, "batch_input": 1, "batch_output": 5 }, "context": { "max_tokens": 1000000, "effective_tokens": 800000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 89.4, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 92, "arc_agi_2": 20 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 96, "reasoning": 95, "writing": 93, "vision": 88, "long_context": 93, "multilingual": 89 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Frontier coding at Sonnet-tier pricing", "128K max output — well above Sonnet 4.6's 64K", "Teams upgrading off Sonnet 4.6 without Opus pricing", "Production agents that need Mythos-class quality" ], "skip_if": [ "Offensive-security or bio work — classifiers return an explicit refusal; another-model retry requires configured application logic", "Absolute cheapest volume — Haiku 4.5 remains cheaper", "You need Fable 5's higher ceiling on the hardest tasks" ], "official_evidence": { "record_id": "claude-sonnet-5", "api_model_id": "claude-sonnet-5", "verified_date": "2026-08-31", "sources": { "release": "https://www.anthropic.com/news/claude-sonnet-5", "pricing": "https://platform.claude.com/docs/en/about-claude/pricing", "context": "https://platform.claude.com/docs/en/about-claude/models/overview", "benchmarks": "https://www.anthropic.com/claude-sonnet-5-system-card" }, "pricing": { "inputPerM": 2, "outputPerM": 10, "cachedInputPerM": 0.2, "batchInputPerM": 1, "batchOutputPerM": 5, "note": "Anthropic's pricing page, read 2026-08-31: $2/$10 per 1M (cache-hit $0.20) is the standard price. The increase to $3/$15 that had been scheduled for September 1, 2026 was cancelled." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": { "SWE-bench Verified": 89.4, "SWE-bench Pro": 71.8, "GPQA Diamond": 92, "Terminal-Bench 2.1": 85.6, "ARC-AGI-2": 20, "Humanity's Last Exam (no tools)": 42.5 }, "notes": "CORRECTED 2026-07-08: max output is 128,000 tokens, not 200,000 as originally recorded. Re-confirmed against Anthropic's Models Overview. Pricing is $2/$10 per 1M; the increase to $3/$15 announced for September 1, 2026 was cancelled and the launch rate became the standard price. Context is 1,000,000 tokens and max output is 128,000. Safety-classified requests return an explicit refusal; use of another model requires configured application or product logic. Benchmark figures are provider-reported, not benchr measurements." } }, { "id": "claude-opus-5", "name": "Claude Opus 5", "provider": "Anthropic", "api_name": "claude-opus-5", "type": "frontier", "license": "proprietary", "released": "2026-07-24", "deprecated": false, "pricing": { "input_per_million": 5, "output_per_million": 25, "cache_input_per_million": 0.5 }, "context": { "max_tokens": 1000000, "effective_tokens": 800000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 97, "reasoning": 97, "writing": 92, "vision": 89, "long_context": 94, "multilingual": 90 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Complex agentic coding and enterprise work", "1M-context workloads that need 128K output", "Teams migrating from Claude Opus 4.8 at the same base API price" ], "skip_if": [ "You need a provider-published benchmark comparison — none is recorded in the verified facts yet", "You need the lowest-cost Claude tier — Sonnet and Haiku remain cheaper" ], "official_evidence": { "record_id": "claude-opus-5", "api_model_id": "claude-opus-5", "verified_date": "2026-08-31", "sources": { "release": "https://platform.claude.com/docs/en/release-notes/overview", "pricing": "https://platform.claude.com/docs/en/about-claude/models/overview", "context": "https://platform.claude.com/docs/en/about-claude/models/overview" }, "pricing": { "inputPerM": 5, "outputPerM": 25, "cachedInputPerM": 0.5, "note": " Cache-hit input $0.50/1M confirmed on Anthropic's pricing page 2026-08-31." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": {}, "notes": "Anthropic launched Claude Opus 5 on July 24, 2026. The official model overview lists the fixed API ID claude-opus-5, a 1M-token context window, 128K maximum output, and $5/$25 per 1M input/output tokens. Anthropic describes it as a step-change over Opus 4.8; no provider-published benchmark figure is recorded here without a specific official table. Adaptive thinking is on by default; disablement at xhigh or max effort returns a 400 error." } }, { "id": "claude-fable-5-1", "name": "Claude Fable 5.1", "provider": "Anthropic", "api_name": "claude-fable-5-1", "type": "frontier", "license": "proprietary", "released": "2026-09-01", "deprecated": false, "pricing": { "input_per_million": 10, "output_per_million": 50, "cache_input_per_million": 0.25, "batch_discount": 0.5, "batch_input": 5, "batch_output": 25, "cache_write_5m_per_million": 12.5, "cache_write_1h_per_million": 20 }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": null, "swe_bench_pro": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 99, "reasoning": 98, "writing": 95, "vision": 94, "long_context": 95, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Production agentic coding with a 1M-token context window", "Repeated-prefix workloads where $0.25 per 1M cache reads materially change cost", "Fable 5 migrations whose tool choice remains auto or none", "Long-running assistant workflows that can use adaptive thinking" ], "skip_if": [ "Your client forces tool_choice type any or a named tool — the 5.1 API returns HTTP 400", "You require Priority Tier — Anthropic does not support Fable 5.1 on it", "You require zero-data-retention access without Anthropic authorization", "You expect a published Fable 5.1 score in the tool's normalized benchmark columns" ], "official_evidence": { "record_id": "claude-fable-5-1", "api_model_id": "claude-fable-5-1", "verified_date": "2026-09-02", "sources": { "release": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "api": "https://platform.claude.com/docs/en/models/fable-5-1/overview", "pricing": "https://platform.claude.com/docs/en/models/fable-5-1/overview", "context": "https://platform.claude.com/docs/en/models/fable-5-1/overview", "benchmarks": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "status": "https://platform.claude.com/docs/en/about-claude/model-deprecations" }, "pricing": { "inputPerM": 10, "outputPerM": 50, "cachedInputPerM": 0.25, "cacheWrite5mPerM": 12.5, "cacheWrite1hPerM": 20, "batchInputPerM": 5, "batchOutputPerM": 25, "note": "Provider-published standard API rates. Cache reads are $0.25 per 1M tokens; five-minute and one-hour cache writes are $12.50 and $20.00 per 1M tokens. Batch halves standard input and output pricing." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": { "Terminal-Bench 4.0": 55.8, "Terminal-Bench-Science 0.1": 52.6, "OSWorld 2.0 (partial success)": 77.9, "OSWorld 2.0 (strict success)": 41.7, "Humanity's Last Exam (no tools)": 60.9, "Humanity's Last Exam (with tools)": 65, "note": "All populated results are provider-reported in Anthropic's September 1 announcement, not benchr measurements. The announcement describes its own evaluation configuration and safeguards; do not compare these figures to differently configured third-party runs as if they were interchangeable." }, "notes": "Released September 1, 2026. Fable 5.1 is a new API ID, not an alias for Fable 5. Anthropic documents that tool_choice type any and named-tool forcing return HTTP 400; auto and none are valid. Manual or disabled thinking configuration is not supported because adaptive thinking is always on. Assistant prefills are unsupported. Fable 5.1 is not supported on Priority Tier. Anthropic states that 30-day data retention normally applies; zero-data-retention access requires authorization." } }, { "id": "deepseek-v4-pro", "name": "DeepSeek V4-Pro", "provider": "DeepSeek", "api_name": "deepseek-v4-pro", "type": "frontier-open", "license": "MIT", "released": "2026-04-24", "deprecated": false, "pricing": { "input_per_million": 1.32, "output_per_million": 3.96, "cache_input_per_million": 0.044, "batch_discount": null, "self_hosted": true, "offpeak_input_per_million": 0.66, "offpeak_output_per_million": 1.98, "offpeak_cache_input_per_million": 0.022, "pricing_note": "Peak (list) rate. DeepSeek bills 01:00-04:00 and 06:00-10:00 UTC Monday to Friday at this rate and every other hour at half of it, effective 16:00 UTC on 2026-08-16." }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": 384000 }, "benchmarks": { "values": { "swe_bench_verified": 80.6, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 90.1, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 93, "reasoning": 92, "writing": 84, "vision": null, "long_context": 86, "multilingual": 87 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Frontier-grade open-weight coding", "Math-heavy work", "Self-hosted production, where the August 2026 API price rise does not apply" ], "skip_if": [ "You need vision/multimodal", "You can't manage GPU hosting" ], "official_evidence": { "record_id": "deepseek-v4-pro", "api_model_id": "deepseek-v4-pro", "verified_date": "2026-08-28", "sources": { "release": "https://api-docs.deepseek.com/news/news260424", "pricing": "https://api-docs.deepseek.com/quick_start/pricing", "license": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "context": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "benchmarks": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "ga_update": "https://api-docs.deepseek.com/updates" }, "pricing": { "inputPerM": 1.32, "outputPerM": 3.96, "cacheHitInputPerM": 0.044, "offPeakInputPerM": 0.66, "offPeakOutputPerM": 1.98, "offPeakCacheHitInputPerM": 0.022, "offPeakDiscount": 0.5, "peakHoursUTC": "01:00-04:00 and 06:00-10:00, Monday to Friday", "priceEffectiveFrom": "2026-08-16T16:00:00Z", "previousFlatPricing": { "inputPerM": 0.435, "outputPerM": 0.87, "cacheHitInputPerM": 0.003625 }, "selfHost": true, "note": "DeepSeek moved to peak/off-peak billing at 16:00 UTC on 2026-08-16. Peak hours are 01:00-04:00 and 06:00-10:00 UTC, Monday to Friday; every other hour bills at half the peak rate. The figures in inputPerM/outputPerM/cacheHitInputPerM are the peak (list) rates; the offPeak* fields are the discounted rates. Read on api-docs.deepseek.com/quick_start/pricing on 2026-08-28." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 384000 }, "benchmarks": { "SWE-bench Verified": 80.6, "SWE-bench Pro": 55.4, "GPQA Diamond": 90.1, "LiveCodeBench": 93.5, "Terminal-Bench 2.0": 67.9, "MMLU-Pro (Max)": 87.5, "MRCR 1M": 83.5, "Codeforces (rating)": 3206, "Terminal-Bench 2.1": 87.9, "Humanity's Last Exam (no tools)": 42.7, "Humanity's Last Exam (with tools)": 60, "NL2Repo": 61.5, "CyberGym": 83.3, "DeepSWE": 62.7, "Toolathlon-Verified": 74.1, "DSBench-Hard": 67.2 }, "notes": "UPDATED 2026-08-28: DeepSeek raised API prices and introduced peak/off-peak billing effective 16:00 UTC on August 16, 2026. Cache-miss input went from a flat $0.435 to $1.32 peak / $0.66 off-peak per 1M tokens, output from $0.87 to $3.96 / $1.98, and cache-hit input from $0.003625 to $0.044 / $0.022 - the steepest line on the sheet at roughly twelve times the old cache-hit rate. Peak hours are 01:00-04:00 and 06:00-10:00 UTC, Monday to Friday. DeepSeek also shipped a V4-Pro GA update on August 13, 2026 with a new provider-reported benchmark table (Terminal Bench 2.1 87.9, HLE 42.7 without tools / 60.0 with tools, NL2Repo 61.5, CyberGym 83.3, DeepSWE 62.7, Toolathlon-Verified 74.1, DSBench-Hard 67.2); those are DeepSeek's own numbers, not benchr tests. The April model-card figures are retained as published. Open weights, MIT." } }, { "id": "deepseek-v4-flash", "name": "DeepSeek V4-Flash", "provider": "DeepSeek", "api_name": "deepseek-v4-flash", "type": "open", "license": "MIT", "released": "2026-04-24", "deprecated": false, "pricing": { "input_per_million": 0.44, "output_per_million": 1.32, "cache_input_per_million": 0.014, "batch_discount": null, "self_hosted": true, "offpeak_input_per_million": 0.22, "offpeak_output_per_million": 0.66, "offpeak_cache_input_per_million": 0.007, "pricing_note": "Peak (list) rate. DeepSeek bills 01:00-04:00 and 06:00-10:00 UTC Monday to Friday at this rate and every other hour at half of it, effective 16:00 UTC on 2026-08-16." }, "context": { "max_tokens": 1000000, "effective_tokens": 600000, "max_output_tokens": 384000 }, "benchmarks": { "values": { "swe_bench_verified": 79, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 88.1, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 86, "reasoning": 86, "writing": 80, "vision": null, "long_context": 88, "multilingual": 86 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Batchable work you can schedule outside DeepSeek's peak hours, where the rate halves to one of the lowest hosted prices among models tracked by benchr — not a market-wide guarantee", "Self-hosted at low cost", "Volume coding tasks" ], "skip_if": [ "You need frontier-tier reasoning depth", "Vision-first workflows", "Your traffic is fixed to European or Asian business hours, which sit inside the peak window" ], "official_evidence": { "record_id": "deepseek-v4-flash", "api_model_id": "deepseek-v4-flash", "verified_date": "2026-08-28", "sources": { "release": "https://api-docs.deepseek.com/news/news260424", "pricing": "https://api-docs.deepseek.com/quick_start/pricing", "license": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "context": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "benchmarks": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash" }, "pricing": { "inputPerM": 0.44, "outputPerM": 1.32, "cacheHitInputPerM": 0.014, "offPeakInputPerM": 0.22, "offPeakOutputPerM": 0.66, "offPeakCacheHitInputPerM": 0.007, "offPeakDiscount": 0.5, "peakHoursUTC": "01:00-04:00 and 06:00-10:00, Monday to Friday", "priceEffectiveFrom": "2026-08-16T16:00:00Z", "previousFlatPricing": { "inputPerM": 0.14, "outputPerM": 0.28, "cacheHitInputPerM": 0.0028 }, "selfHost": true, "note": "DeepSeek moved to peak/off-peak billing at 16:00 UTC on 2026-08-16. Peak hours are 01:00-04:00 and 06:00-10:00 UTC, Monday to Friday; every other hour bills at half the peak rate. The figures in inputPerM/outputPerM/cacheHitInputPerM are the peak (list) rates; the offPeak* fields are the discounted rates. Read on api-docs.deepseek.com/quick_start/pricing on 2026-08-28." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 384000 }, "benchmarks": { "SWE-bench Verified": 79, "GPQA Diamond": 88.1, "LiveCodeBench": 91.6, "MMLU-Pro (Think Max)": 86.2, "HMMT 2026 Feb": 94.8, "MRCR 1M": 78.7 }, "notes": "UPDATED 2026-08-28: DeepSeek raised API prices and introduced peak/off-peak billing effective 16:00 UTC on August 16, 2026, announced alongside the V4 lineup release. Cache-miss input went from a flat $0.14 to $0.44 peak / $0.22 off-peak per 1M tokens, output from $0.28 to $1.32 / $0.66, and cache-hit input from $0.0028 to $0.014 / $0.007. Peak hours are 01:00-04:00 and 06:00-10:00 UTC, Monday to Friday, so roughly four fifths of the week bills at the off-peak rate; that share is a benchr calculation from the published hours, not a provider figure. SWE-bench Verified 79.0% is DeepSeek's provider-reported model-card figure, not an independently reproduced result. Open weights, MIT." } }, { "id": "gemini-3-1-pro", "name": "Gemini 3.1 Pro", "provider": "Google", "api_name": "gemini-3.1-pro-preview", "type": "frontier", "license": "proprietary", "released": "2026-02-19", "deprecated": false, "pricing": { "input_per_million": 2, "output_per_million": 12, "input_per_million_over_200k": 4, "output_per_million_over_200k": 18, "cache_input_per_million": 0.2, "batch_discount": null, "batch_input": 1, "batch_output": 6 }, "context": { "max_tokens": 1000000, "effective_tokens": 800000, "max_output_tokens": 64000 }, "benchmarks": { "values": { "swe_bench_verified": 80.6, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 94.3, "arc_agi_2": 77.1 }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 84, "reasoning": 90, "writing": 84, "vision": 95, "long_context": 92, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Deep reasoning in the Gemini family", "Long-context vision work", "Workspace integration" ], "skip_if": [ "Coding agents — Flash is faster and cheaper", "Cost-sensitive workloads — note the over-200K price bump" ], "official_evidence": { "record_id": "gemini-3-1-pro", "api_model_id": "gemini-3.1-pro-preview", "verified_date": "2026-05-31", "sources": { "release": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-1-pro/", "pricing": "https://ai.google.dev/gemini-api/docs/pricing", "context": "https://deepmind.google/models/model-cards/gemini-3-1-pro/", "benchmarks": "https://deepmind.google/models/model-cards/gemini-3-1-pro/" }, "pricing": { "inputPerM": 2, "outputPerM": 12, "inputAbove200kPerM": 4, "outputAbove200kPerM": 18, "batchInputPerM": 1, "batchOutputPerM": 6, "batchInputAbove200kPerM": 2, "batchOutputAbove200kPerM": 9, "cachedInputPerM": 0.2, "freeApiTier": false }, "context": { "windowTokens": 1000000, "maxOutputTokens": 64000 }, "benchmarks": { "ARC-AGI-2": 77.1, "GPQA Diamond": 94.3, "Humanity's Last Exam (with tools)": 51.4, "MMMU-Pro": 80.5, "SWE-bench Verified": 80.6, "MMMLU": 92.6 }, "notes": "Tiered pricing: the >200K-token tier doubles input ($4) and raises output ($18). Output price includes thinking tokens. No free API tier (free trial in AI Studio UI only). Context-caching: cached input $0.20/1M (90% off the $2 input rate; Google additionally charges $1 per 1M-token-hour of cache storage), verified 2026-06-15 against ai.google.dev/gemini-api/docs/pricing." } }, { "id": "gemini-3-5-flash", "name": "Gemini 3.5 Flash", "provider": "Google", "api_name": "gemini-3.5-flash", "type": "mid", "license": "proprietary", "released": "2026-05-19", "deprecated": false, "pricing": { "input_per_million": 1.5, "output_per_million": 9, "cache_input_per_million": 0.15, "batch_discount": null, "free_tier": true, "batch_input": 0.75, "batch_output": 4.5 }, "context": { "max_tokens": 1048576, "effective_tokens": 700000, "max_output_tokens": 65536 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 88, "reasoning": 86, "writing": 84, "vision": 92, "long_context": 90, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Coding agents at speed", "Parallel agent execution", "Multimodal tasks", "Default frontier-quality model" ], "skip_if": [ "You need the deepest single-call reasoning — use Gemini 3.1 Pro" ], "official_evidence": { "record_id": "gemini-3-5-flash", "api_model_id": "gemini-3.5-flash", "verified_date": "2026-07-30", "sources": { "release": "https://ai.google.dev/gemini-api/docs/deprecations", "pricing": "https://ai.google.dev/gemini-api/docs/pricing", "context": "https://ai.google.dev/gemini-api/docs/models/gemini-3.5-flash", "benchmarks": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-3-5/" }, "pricing": { "inputPerM": 1.5, "outputPerM": 9, "batchInputPerM": 0.75, "batchOutputPerM": 4.5, "cachedInputPerM": 0.15, "freeApiTier": true, "note": "Flat rate — no >200K context tier. Cached input $0.15/1M (90% off the $1.50 input rate; Google additionally charges $1 per 1M-token-hour of cache storage), verified 2026-06-15 against ai.google.dev/gemini-api/docs/pricing." }, "context": { "windowTokens": 1048576, "maxOutputTokens": 65536 }, "benchmarks": { "Terminal-Bench 2.1": 76.2, "GDPval-AA (Elo)": 1656, "MCP Atlas": 83.6, "CharXiv Reasoning": 84.2, "SWE-bench Verified": null, "GPQA Diamond": null }, "notes": "Released at Google I/O (May 19, 2026), GA. Google's live pricing page checked 2026-07-30 lists standard $1.50/$9, batch $0.75/$4.50, cached input $0.15 plus cache-storage charges, and a free API tier. The launch blog publishes only the four agentic/coding/multimodal benchmarks above; no official SWE-bench Verified or GPQA figure is published for 3.5 Flash, so those fields are null. No universal latency figure is recorded." } }, { "id": "gemini-3-5-flash-lite", "name": "Gemini 3.5 Flash-Lite", "provider": "Google", "api_name": "gemini-3.5-flash-lite", "type": "frontier", "license": "proprietary", "released": "2026-07-21", "deprecated": false, "pricing": { "input_per_million": 0.3, "output_per_million": 2.5 }, "context": { "max_tokens": 1048576, "effective_tokens": 750000, "max_output_tokens": 65536 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 84, "reasoning": 83, "writing": 78, "vision": 83, "long_context": 92, "multilingual": 84 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "High-volume automation and document extraction", "Low-cost subagents with a 1M-token context window", "Multimodal input workflows that need structured text output" ], "skip_if": [ "You need a provider-published score for the benchmarks shown in benchr's table", "You need the higher capability ceiling of Gemini 3.6 Flash" ], "official_evidence": { "record_id": "gemini-3-5-flash-lite", "api_model_id": "gemini-3.5-flash-lite", "verified_date": "2026-07-28", "sources": { "release": "https://ai.google.dev/gemini-api/docs/changelog", "pricing": "https://ai.google.dev/gemini-api/docs/latest-model", "context": "https://ai.google.dev/gemini-api/docs/models/gemini-3.5-flash-lite", "benchmarks": "https://ai.google.dev/gemini-api/docs/latest-model" }, "pricing": { "inputPerM": 0.3, "outputPerM": 2.5 }, "context": { "windowTokens": 1048576, "maxOutputTokens": 65536 }, "benchmarks": { "HLE": 18, "CharXIV": 74.5 }, "notes": "Google released the stable Gemini 3.5 Flash-Lite model on July 21, 2026 for high-throughput execution, document parsing, and subagent work. It accepts text, image, video, audio, and PDF input and returns text." } }, { "id": "gemini-3-6-flash", "name": "Gemini 3.6 Flash", "provider": "Google", "api_name": "gemini-3.6-flash", "type": "mid", "license": "proprietary", "released": "2026-07-21", "deprecated": false, "pricing": { "input_per_million": 0.75, "output_per_million": 3.75, "cache_input_per_million": 0.075, "batch_discount": null, "batch_input": 0.375, "batch_output": 1.875, "free_tier": true }, "context": { "max_tokens": 1048576, "effective_tokens": 720000, "max_output_tokens": 65536 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 89, "reasoning": 87, "writing": 84, "vision": 92, "long_context": 90, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Output-heavy coding agents", "Google Search and Maps grounded workflows", "Stable Flash migrations from 2.x and preview endpoints", "Multimodal input with text output" ], "skip_if": [ "You need published official benchmark tables before procurement", "Gemini 3.7 Flash costs the same and Google calls it the stronger model", "You need image generation or Live API voice output" ], "official_evidence": { "record_id": "gemini-3-6-flash", "api_model_id": "gemini-3.6-flash", "verified_date": "2026-08-24", "sources": { "release": "https://ai.google.dev/gemini-api/docs/changelog", "pricing": "https://ai.google.dev/gemini-api/docs/pricing", "context": "https://ai.google.dev/gemini-api/docs/models/gemini-3.6-flash", "deprecations": "https://ai.google.dev/gemini-api/docs/deprecations" }, "pricing": { "inputPerM": 0.75, "outputPerM": 3.75, "batchInputPerM": 0.375, "batchOutputPerM": 1.875, "flexInputPerM": 0.375, "flexOutputPerM": 1.875, "priorityInputPerM": 1.35, "priorityOutputPerM": 6.75, "cachedInputPerM": 0.075, "cacheStoragePerMTokenHour": 0.5, "postPromoInputPerM": 1.5, "postPromoOutputPerM": 7.5, "postPromoCachedInputPerM": 0.15, "postPromoStartDate": "2027-01-01", "freeApiTier": true, "note": "Google's live pricing table now prices Gemini 3.6 Flash at $0.75 input / $3.75 output per 1M tokens through December 31, 2026, then $1.50 / $7.50 starting January 1, 2027. Context caching is $0.075 (then $0.15) plus $0.50 per 1M-token-hour of storage (then $1.00). Batch and Flex are half of standard; Priority is $1.35 / $6.75. The $1.50 / $7.50 pair recorded on 2026-07-22 is the post-promotional rate. Read on ai.google.dev/gemini-api/docs/pricing on 2026-08-24." }, "context": { "windowTokens": 1048576, "maxOutputTokens": 65536 }, "benchmarks": { "Terminal-Bench 2.1": null, "GDPval-AA (Elo)": null, "MCP Atlas": null, "CharXiv Reasoning": null, "SWE-bench Verified": null, "GPQA Diamond": null }, "notes": "Google released Gemini 3.6 Flash as a stable GA model on July 21, 2026, with improved token efficiency and code/agentic planning claims. Official model page lists text/image/video/audio/PDF inputs, text output, 1,048,576 input tokens, 65,536 output tokens, caching, code execution, file search, function calling, Search/Maps grounding, structured outputs, thinking, URL context, and Computer Use support in preview. No official benchmark table found on the model page or release note, so hard benchmark fields are null. Re-verified 2026-08-24: Google now lists a promotional rate that halves input and output through December 31, 2026, matching the Gemini 3.7 Flash launch price." } }, { "id": "gemini-3-7-flash", "name": "Gemini 3.7 Flash", "provider": "Google", "api_name": "gemini-3.7-flash", "type": "mid", "license": "proprietary", "released": "2026-08-13", "deprecated": false, "pricing": { "input_per_million": 0.75, "output_per_million": 3.75, "cache_input_per_million": 0.075, "batch_discount": null, "batch_input": 0.375, "batch_output": 1.875, "free_tier": true }, "context": { "max_tokens": 1048576, "effective_tokens": 720000, "max_output_tokens": 65536 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 91, "reasoning": 88, "writing": 85, "vision": 92, "long_context": 90, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Coding and agent work at Flash prices", "Teams already on Gemini 3.6 Flash — same price, newer model", "Long-context multimodal input with text output", "Free-tier prototyping before a paid rollout" ], "skip_if": [ "You need published official benchmark tables — Google shipped none", "You want the cheapest subagent tier — use Gemini 3.5 Flash-Lite", "You need image generation, audio output, or the Live API" ], "official_evidence": { "record_id": "gemini-3-7-flash", "api_model_id": "gemini-3.7-flash", "verified_date": "2026-08-24", "sources": { "release": "https://ai.google.dev/gemini-api/docs/changelog", "pricing": "https://ai.google.dev/gemini-api/docs/pricing", "context": "https://ai.google.dev/gemini-api/docs/models/gemini-3.7-flash", "deprecations": "https://ai.google.dev/gemini-api/docs/deprecations" }, "pricing": { "inputPerM": 0.75, "outputPerM": 3.75, "batchInputPerM": 0.375, "batchOutputPerM": 1.875, "flexInputPerM": 0.375, "flexOutputPerM": 1.875, "priorityInputPerM": 1.35, "priorityOutputPerM": 6.75, "cachedInputPerM": 0.075, "cacheStoragePerMTokenHour": 0.5, "postPromoInputPerM": 1.5, "postPromoOutputPerM": 7.5, "postPromoCachedInputPerM": 0.15, "postPromoStartDate": "2027-01-01", "freeApiTier": true, "note": "Introductory pricing: $0.75 input / $3.75 output per 1M tokens through December 31, 2026, then $1.50 / $7.50 starting January 1, 2027. Context caching is $0.075 (then $0.15) plus $0.50 per 1M-token-hour of storage (then $1.00). Batch and Flex are half of standard; Priority is $1.35 / $6.75. A free tier is offered. Read on ai.google.dev/gemini-api/docs/pricing on 2026-08-24." }, "context": { "windowTokens": 1048576, "maxOutputTokens": 65536 }, "benchmarks": { "SWE-bench Verified": null, "GPQA Diamond": null, "Terminal-Bench 2.1": null, "note": "Google published no benchmark table with the release note or on the model page, so every formal benchmark field stays null." }, "notes": "Google made Gemini 3.7 Flash generally available on August 13, 2026 and describes it as its most intelligent workhorse model for coding and agents, with improvements across software engineering, web development, and agentic workflows. The model page lists text, image, video, audio, and PDF input with text output, a 1,048,576-token input limit, a 65,536-token output limit, caching, code execution, file search, function calling, Search and Maps grounding, structured outputs, thinking at low, medium, or high (no minimal level), URL context, and Computer Use in preview. Live API, image generation, and audio generation are not supported. Google publishes no benchmark figures for it, so none are claimed here." } }, { "id": "gemini-3-8-flash", "name": "Gemini 3.8 Flash", "provider": "Google", "api_name": "gemini-3.8-flash", "type": "mid", "license": "proprietary", "released": "2026-09-02", "deprecated": false, "pricing": { "input_per_million": 0.75, "output_per_million": 3.75, "cache_input_per_million": 0.075, "batch_discount": null, "batch_input": 0.375, "batch_output": 1.875, "free_tier": true, "intro_through": "2026-12-31", "standard_input_per_million_from_2027_01_01": 1.5, "standard_output_per_million_from_2027_01_01": 7.5 }, "context": { "max_tokens": 1048576, "effective_tokens": null, "max_output_tokens": 65536 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": null, "reasoning": null, "writing": null, "vision": null, "long_context": null, "multilingual": null }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Teams already on 3.7 Flash - identical price, newer model", "Agentic and multi-step coding work at Flash prices", "1M-token context with tunable thinking levels", "Free-tier prototyping before a paid rollout" ], "skip_if": [ "You need the figures benchr has measured - this record is days old and unmeasured", "You want published SWE-bench Verified or GPQA tables - Google published neither", "You are budgeting past 2026 - the price doubles on January 1, 2027", "You need the Cyber variant - it is Fairwind Program only, not a public API model" ], "official_evidence": { "record_id": "gemini-3-8-flash", "api_model_id": "gemini-3.8-flash", "verified_date": "2026-09-03", "sources": { "release": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/", "pricing": "https://ai.google.dev/gemini-api/docs/pricing", "context": "https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash", "deprecations": "https://ai.google.dev/gemini-api/docs/deprecations", "benchmarks": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/3-8-flash-and-3-8-flash-cyber/" }, "pricing": { "inputPerM": 0.75, "outputPerM": 3.75, "batchInputPerM": 0.375, "batchOutputPerM": 1.875, "flexInputPerM": 0.375, "flexOutputPerM": 1.875, "priorityInputPerM": 1.35, "priorityOutputPerM": 6.75, "cachedInputPerM": 0.075, "cacheStoragePerMTokenHour": 0.5, "postPromoInputPerM": 1.5, "postPromoOutputPerM": 7.5, "postPromoCachedInputPerM": 0.15, "postPromoStartDate": "2027-01-01", "freeApiTier": true, "note": "Introductory pricing: $0.75 input / $3.75 output per 1M tokens through December 31, 2026, then $1.50 / $7.50 from January 1, 2027. Every tier doubles on the same date, including batch, flex, priority and cached input." }, "context": { "windowTokens": 1048576, "maxOutputTokens": 65536 }, "benchmarks": { "SWE-bench Verified": null, "GPQA Diamond": null, "Terminal-Bench 2.1": null, "note": null, "HLE-Verified": 54.9 }, "notes": "Gemini 3.8 Flash Cyber is a specialised variant of the same foundational model, available only through Google's Fairwind Program for trusted defenders. It is not a separately priced API model and has no entry in the pricing documentation, so benchr does not carry a record for it. Google states 3.7 Flash remains fully supported." } }, { "id": "llama-4-maverick", "name": "Llama 4 Maverick", "provider": "Meta", "api_name": "meta-llama/Llama-4-Maverick-17B-128E-Instruct", "type": "frontier-open", "license": "Llama 4 Community License", "released": "2025-04-05", "deprecated": false, "pricing": { "input_per_million": null, "output_per_million": null, "self_hosted": true }, "context": { "max_tokens": 1000000, "effective_tokens": 700000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 69.8, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 80, "reasoning": 81, "writing": 76, "vision": 80, "long_context": 85, "multilingual": 80 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Best open-weight frontier model", "Self-hosted production at zero licensing cost", "Multimodal at no API cost" ], "skip_if": [ "You need the very best reasoning or coding", "You can't manage GPU infrastructure" ], "official_evidence": { "record_id": "llama-4-maverick", "api_model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct", "verified_date": "2026-05-31", "sources": { "release": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "license": "https://www.llama.com/llama4/license/", "context": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct", "benchmarks": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" }, "pricing": { "selfHost": true, "inputPerM": null, "outputPerM": null }, "context": { "windowTokens": 1000000, "maxOutputTokens": null }, "benchmarks": { "MMLU-Pro (0-shot)": 80.5, "GPQA Diamond": 69.8, "LiveCodeBench": 43.4, "MGSM": 92.3 }, "notes": "Open weight; $0 to self-host. 1M-token context, 128 experts. Llama 4 Behemoth (288B active / ~2T total) was only ever previewed as 'still training' and was never released — do not list specs for it as a usable model." } }, { "id": "llama-4-scout", "name": "Llama 4 Scout", "provider": "Meta", "api_name": "meta-llama/Llama-4-Scout-17B-16E", "type": "open", "license": "Llama 4 Community License", "released": "2025-04-05", "deprecated": false, "pricing": { "input_per_million": null, "output_per_million": null, "self_hosted": true }, "context": { "max_tokens": 10000000, "effective_tokens": 2000000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 57.2, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 73, "reasoning": 74, "writing": 70, "vision": 75, "long_context": 92, "multilingual": 78 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Ultra-long-context tasks (10M token window)", "Fast self-hosted inference", "Free multimodal at scale" ], "skip_if": [ "Deep reasoning tasks", "When frontier coding quality is needed" ], "official_evidence": { "record_id": "llama-4-scout", "api_model_id": "meta-llama/Llama-4-Scout-17B-16E", "verified_date": "2026-05-31", "sources": { "release": "https://ai.meta.com/blog/llama-4-multimodal-intelligence/", "license": "https://www.llama.com/llama4/license/", "context": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct", "benchmarks": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E" }, "pricing": { "selfHost": true, "inputPerM": null, "outputPerM": null }, "context": { "windowTokens": 10000000, "maxOutputTokens": null }, "benchmarks": { "MMLU-Pro (0-shot)": 74.3, "GPQA Diamond": 57.2, "MMMU": 73.4, "MathVista": 73.7, "LiveCodeBench": 32.8 }, "notes": "Open weight; $0 to self-host (Meta sells no API). Community License: >700M MAU needs a separate Meta license; not OSI-approved. 10M-token context. Benchmarks from Meta's official instruction-tuned model card." } }, { "id": "phi-4", "name": "Phi-4", "provider": "Microsoft", "api_name": "microsoft/phi-4", "type": "small-open", "license": "MIT", "released": "2024-12-12", "deprecated": false, "pricing": { "input_per_million": null, "output_per_million": null, "self_hosted": true }, "context": { "max_tokens": 16000, "effective_tokens": 14000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": 84.8, "humaneval": 82.6, "math": 80.4, "gpqa_diamond": 56.1, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 70, "reasoning": 78, "writing": 74, "vision": null, "long_context": 60, "multilingual": 72 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Local inference on consumer hardware", "Edge deployment", "Reasoning at tiny scale" ], "skip_if": [ "Long-context tasks", "Production-quality writing or coding" ], "official_evidence": { "record_id": "phi-4", "api_model_id": "microsoft/phi-4", "verified_date": "2026-06-03", "sources": { "release": "https://huggingface.co/microsoft/phi-4", "license": "https://huggingface.co/microsoft/phi-4", "context": "https://huggingface.co/microsoft/phi-4", "benchmarks": "https://www.microsoft.com/en-us/research/publication/phi-4-technical-report/" }, "pricing": { "selfHost": true, "inputPerM": null, "outputPerM": null }, "context": { "windowTokens": 16000, "maxOutputTokens": null }, "benchmarks": { "GPQA Diamond": 56.1, "MMLU": 84.8, "HumanEval": 82.6, "MATH": 80.4 }, "notes": "Verified 2026-06-03: MIT-licensed, 14B dense, 16K context, self-host only (no Microsoft per-token API; available on Azure AI Foundry + Hugging Face). Released 2024-12-12. Benchmarks from the Phi-4 technical report / model card." } }, { "id": "minimax-m3", "name": "MiniMax M3", "provider": "MiniMax", "api_name": "MiniMax-M3", "type": "frontier-open", "license": "proprietary hosted API", "released": "2026-06-01", "deprecated": false, "pricing": { "input_per_million": 0.3, "output_per_million": 1.2, "cache_input_per_million": 0.06, "tier_prices_per_million": [ 0.3, 1.2, 0.6, 2.4, 0.45, 1.8, 0.9, 3.6 ], "batch_discount": null, "input_per_million_over_200k": 0.6, "output_per_million_over_200k": 2.4 }, "context": { "max_tokens": 1000000, "effective_tokens": 750000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 90, "reasoning": 88, "writing": 80, "vision": 85, "long_context": 94, "multilingual": 84 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Very low-cost long-context coding agents", "1M-context workflows under $1/M input", "Multimodal coding and tool-use experiments" ], "skip_if": [ "You need a mature Western provider ecosystem", "You need official benchmark tables before rollout" ], "official_evidence": { "record_id": "minimax-m3", "api_model_id": "MiniMax-M3", "verified_date": "2026-07-13", "sources": { "release": "https://platform.minimax.io/docs/release-notes/models", "pricing": "https://platform.minimax.io/docs/guides/pricing-paygo", "context": "https://platform.minimax.io/docs/guides/text-generation" }, "pricing": { "inputPerM": 0.3, "outputPerM": 1.2, "cacheHitInputPerM": 0.06, "longContextInputPerM": 0.6, "longContextOutputPerM": 2.4, "longContextCacheHitInputPerM": 0.12, "priorityInputPerM": 0.45, "priorityOutputPerM": 1.8, "priorityLongContextInputPerM": 0.9, "priorityLongContextOutputPerM": 3.6 }, "context": { "windowTokens": 1000000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "MiniMax release notes list MiniMax-M3 on June 1, 2026. The model overview/invocation docs describe it as a frontier multimodal coding model with 1M context. Pay-as-you-go pricing shows permanent 50% off rates: standard <=512K input $0.30/$1.20 per 1M with $0.06 prompt-cache read; >512K input $0.60/$2.40 with $0.12 cache read. Priority is 1.5x standard. No official max output or benchmark table was found on the checked pages." } }, { "id": "mistral-large-3", "name": "Mistral Large 3", "provider": "Mistral AI", "api_name": "mistral-large-2512", "type": "frontier-open", "license": "Apache-2.0", "released": "2025-12-02", "deprecated": false, "pricing": { "input_per_million": 0.5, "output_per_million": 1.5, "cache_input_per_million": null, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 256000, "effective_tokens": 200000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 78, "reasoning": 79, "writing": 78, "vision": null, "long_context": 76, "multilingual": 88 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Apache-licensed production workloads", "European data residency", "Very cheap inference with decent reasoning" ], "skip_if": [ "Coding at frontier quality", "Vision or multimodal workflows" ], "official_evidence": { "record_id": "mistral-large-3", "api_model_id": "mistral-large-2512", "verified_date": "2026-06-12", "sources": { "release": "https://docs.mistral.ai/models/mistral-large-3-25-12", "pricing": "https://docs.mistral.ai/models/mistral-large-3-25-12", "license": "https://mistral.ai/news/mistral-3/", "context": "https://docs.mistral.ai/models/mistral-large-3-25-12", "benchmarks": "https://mistral.ai/news/mistral-3/" }, "pricing": { "inputPerM": 0.5, "outputPerM": 1.5, "cacheHitInputPerM": null, "selfHost": true }, "context": { "windowTokens": 256000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "Open weights, Apache-2.0. Mistral's announcement gives only relative/leaderboard claims for Large 3, no discrete official per-benchmark scores (the ~85% AIME figure on the page belongs to a smaller reasoning variant, NOT Large 3) — so benchmarks is intentionally empty, not guessed. Cache-hit price not published." } }, { "id": "mistral-medium-3-5", "name": "Mistral Medium 3.5", "provider": "Mistral AI", "api_name": "mistral-medium-3-5", "type": "frontier-open", "license": "Modified MIT", "released": "2026-04-28", "deprecated": false, "pricing": { "input_per_million": 1.5, "output_per_million": 7.5, "cache_input_per_million": null, "self_hosted": true }, "context": { "max_tokens": 256000, "effective_tokens": 200000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 86, "reasoning": 87, "writing": 86, "vision": 84, "long_context": 84, "multilingual": 92 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "European data residency", "Multimodal + reasoning in one model", "Self-hosted on 4 GPUs" ], "skip_if": [ "You need MoE efficiency for ultra-cheap inference" ], "official_evidence": { "record_id": "mistral-medium-3-5", "api_model_id": "mistral-medium-3-5", "verified_date": "2026-06-23", "sources": { "release": "https://mistral.ai/news/vibe-remote-agents-mistral-medium-3-5/", "pricing": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "license": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04", "context": "https://docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04" }, "pricing": { "inputPerM": 1.5, "outputPerM": 7.5, "cacheHitInputPerM": null, "selfHost": true }, "context": { "windowTokens": 256000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "Verified 2026-06-03, re-confirmed 2026-06-23 against the official Mistral model card (docs.mistral.ai/models/model-cards/mistral-medium-3-5-26-04): Modified MIT license, open-weight (self-hostable on ~4 GPUs) AND offered as a hosted API, dense 128B, 256K context, $1.50/$7.50 per 1M. Official release date is April 28, 2026 (filled 2026-06-23; previously null). Max output tokens not stated on the card (null). No discrete official per-benchmark scores published — benchmarks intentionally empty." } }, { "id": "kimi-k2-6", "name": "Kimi K2.6", "provider": "Moonshot AI", "api_name": "kimi-k2.6", "type": "frontier-open", "license": "Modified MIT", "released": null, "deprecated": false, "pricing": { "input_per_million": 0.95, "output_per_million": 4, "cache_input_per_million": 0.16, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 262144, "effective_tokens": 200000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": 80.2, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 90.5, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 80, "reasoning": 78, "writing": 74, "vision": null, "long_context": 78, "multilingual": 82 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Mid-range open-weight option", "Multilingual tasks", "Cost-efficient API with self-hosting option" ], "skip_if": [ "You need the deepest reasoning or best coding", "Vision workloads" ], "official_evidence": { "record_id": "kimi-k2-6", "api_model_id": "kimi-k2.6", "verified_date": "2026-06-12", "sources": { "release": null, "pricing": "https://platform.kimi.ai/docs/pricing/chat-k26", "license": "https://huggingface.co/moonshotai/Kimi-K2.6/blob/main/LICENSE", "context": "https://huggingface.co/moonshotai/Kimi-K2.6", "benchmarks": "https://huggingface.co/moonshotai/Kimi-K2.6" }, "pricing": { "inputPerM": 0.95, "outputPerM": 4, "cacheHitInputPerM": 0.16, "selfHost": true }, "context": { "windowTokens": 262144, "maxOutputTokens": null }, "benchmarks": { "SWE-bench Verified": 80.2, "SWE-bench Multilingual": 76.7, "SWE-bench Pro": 58.6, "Terminal-Bench 2.0": 66.7, "LiveCodeBench v6": 89.6, "AIME 2026": 96.4, "GPQA Diamond": 90.5, "Humanity's Last Exam (with tools)": 54, "OSWorld-Verified": 73.1 }, "notes": "Official release date not stated on any official Moonshot page (third-party says Apr 20, 2026), so releaseDate is null. Context 262,144 (256K) confirmed on both pricing page and model card. Input $0.95 is cache-miss; cache-hit $0.16. Model card body: 1T total / 32B active (the org listing page rounds to 1.1T). Benchmarks are Moonshot's own model-card figures." } }, { "id": "kimi-k3", "name": "Kimi K3", "provider": "Moonshot AI", "api_name": "kimi-k3", "type": "frontier-open", "license": "Kimi K3 License", "released": "2026-07-16", "deprecated": false, "pricing": { "input_per_million": 3, "output_per_million": 15, "cache_input_per_million": 0.3, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 1048576, "effective_tokens": null, "max_output_tokens": 1048576 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 93.5, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 95, "reasoning": 93, "writing": 86, "vision": 94, "long_context": 96, "multilingual": 90 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Open-weight long-horizon coding and multimodal research workflows", "One-million-token workloads with strong automatic cache economics", "Teams that can preserve full reasoning history across agent turns" ], "skip_if": [ "Your harness cannot return the complete reasoning and tool-call history unchanged", "You need non-thinking mode or low uncached-output cost" ], "official_evidence": { "record_id": "kimi-k3", "api_model_id": "kimi-k3", "verified_date": "2026-08-31", "sources": { "release": "https://www.kimi.ai/blog/kimi-k3", "releaseDate": "https://www.kimi.com/code/docs/en/kimi-code/whats-new.html", "pricing": "https://platform.kimi.ai/docs/pricing/chat-k3", "context": "https://platform.kimi.ai/docs/guide/kimi-k3-quickstart", "license": "https://huggingface.co/moonshotai/Kimi-K3", "benchmarks": "https://huggingface.co/moonshotai/Kimi-K3" }, "pricing": { "inputPerM": 3, "outputPerM": 15, "cacheHitInputPerM": 0.3, "selfHost": true, "note": "Global Kimi API rates. The model uses flat pay-as-you-go pricing with no context-length tier; applicable taxes are extra." }, "context": { "windowTokens": 1048576, "maxOutputTokens": 1048576, "defaultMaxOutputTokens": 131072 }, "benchmarks": { "GPQA Diamond": 93.5, "DeepSWE": 67.5, "ProgramBench": 77.8, "Terminal-Bench 2.1": 88.3, "FrontierSWE": 81.2, "BrowseComp": 91.2, "Toolathlon Verified": 76.5, "OSWorld Verified": 84.8, "MMMU-Pro without tools": 81.6, "MMMU-Pro with tools": 83.4 }, "notes": "Moonshot released Kimi K3 on July 16, 2026 and later published its weights. The official model card lists 2.8T total parameters, 104B active parameters, 896 experts with 16 selected per token, 1,048,576 context, native image/text understanding, and provider-run benchmark results. Hosted API documentation additionally supports video files, tools, strict structured output, and up to 1,048,576 completion tokens (131,072 default). Pricing and model card re-verified on the official pricing page and Hugging Face card on 2026-08-31." } }, { "id": "gpt-5", "name": "GPT-5", "provider": "OpenAI", "api_name": "gpt-5", "type": "frontier", "license": "proprietary", "released": "2025-08-07", "deprecated": false, "pricing": { "input_per_million": 1.25, "output_per_million": 10, "cache_input_per_million": 0.125, "batch_discount": 0.5 }, "context": { "max_tokens": 400000, "effective_tokens": 300000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 74.9, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 88, "reasoning": 88, "writing": 86, "vision": 85, "long_context": 78, "multilingual": 87 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Production workhorse at a rational price", "Breadth tasks", "Coding agents", "The everyday OpenAI pick" ], "skip_if": [ "You need the absolute deepest reasoning — use 5.5", "Very large context windows" ], "official_evidence": { "record_id": "gpt-5", "api_model_id": "gpt-5", "verified_date": "2026-06-12", "sources": { "release": "https://deploymentsafety.openai.com/gpt-5", "pricing": "https://developers.openai.com/api/docs/models/gpt-5", "context": "https://developers.openai.com/api/docs/models/gpt-5", "benchmarks": "https://cdn.openai.com/gpt-5-system-card.pdf" }, "pricing": { "inputPerM": 1.25, "outputPerM": 10, "cachedInputPerM": 0.125 }, "context": { "windowTokens": 400000, "maxOutputTokens": 128000 }, "benchmarks": { "SWE-bench Verified": 74.9, "AIME 2025": null, "GPQA Diamond": null, "HealthBench Hard": 46.2 }, "notes": "SWE-bench Verified 74.9% is officially OpenAI's launch-blog figure (verbosity=medium), confirmed via the system card PDF which cites it (p.36). AIME 2025 (~94.6%) and GPQA Diamond (~88.4%) are widely attributed to the launch page openai.com/index/introducing-gpt-5, which blocks automated fetchers (403); they could NOT be re-read from an accessible official source, so they are null. The 88.4% figure may refer to GPT-5 pro, not base GPT-5. OpenAI docs now label GPT-5 the previous model." } }, { "id": "gpt-5-mini", "name": "GPT-5 Mini", "provider": "OpenAI", "api_name": "gpt-5-mini", "type": "small", "license": "proprietary", "released": "2025-08-07", "deprecated": false, "pricing": { "input_per_million": 0.25, "output_per_million": 2, "cache_input_per_million": 0.025, "batch_discount": 0.5, "batch_input": 0.125, "batch_output": 1 }, "context": { "max_tokens": 400000, "effective_tokens": 200000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 72, "reasoning": 74, "writing": 75, "vision": 72, "long_context": 70, "multilingual": 80 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Cheap chat at scale", "Simple extraction and routing", "High-volume classification" ], "skip_if": [ "Production-quality code review", "Complex reasoning", "Long documents" ], "official_evidence": { "record_id": "gpt-5-mini", "api_model_id": "gpt-5-mini", "verified_date": "2026-07-30", "sources": { "release": "https://developers.openai.com/api/docs/models/gpt-5-mini", "pricing": "https://developers.openai.com/api/docs/models/gpt-5-mini", "context": "https://developers.openai.com/api/docs/models/gpt-5-mini" }, "pricing": { "inputPerM": 0.25, "outputPerM": 2, "cachedInputPerM": 0.025, "batchInputPerM": 0.125, "batchOutputPerM": 1 }, "context": { "windowTokens": 400000, "maxOutputTokens": 128000 }, "benchmarks": {}, "notes": "Official standard price is $0.25/$2.00 with $0.025 cached input. The OpenAI pricing table lists short-context batch at $0.125/$1.00; rechecked 2026-07-30. Release date is traced to snapshot id gpt-5-mini-2025-08-07. The checked model page publishes no SWE-bench or numeric throughput result, so formal benchmark fields remain empty." } }, { "id": "gpt-5-4", "name": "GPT-5.4", "provider": "OpenAI", "api_name": "gpt-5.4", "type": "frontier", "license": "proprietary", "released": "2026-03-05", "deprecated": false, "pricing": { "input_per_million": 2.5, "output_per_million": 15, "cache_input_per_million": 0.25 }, "context": { "max_tokens": 1050000, "effective_tokens": 600000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": null, "osworld_verified": 75, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 90, "reasoning": 91, "writing": 87, "vision": 88, "long_context": 85, "multilingual": 88 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Financial modeling and spreadsheet work", "Computer-use pipelines", "Professional knowledge work", "Long documents at a mid-tier price" ], "skip_if": [ "You want OpenAI's strongest — that's GPT-5.5 now", "Cheap volume — GPT-5 Mini costs a tenth", "Simple chat — GPT-5 is cheaper" ], "official_evidence": { "record_id": "gpt-5-4", "api_model_id": "gpt-5.4", "verified_date": "2026-07-29", "sources": { "release": "https://openai.com/index/introducing-gpt-5-4/", "pricing": "https://developers.openai.com/api/docs/models/gpt-5.4", "context": "https://developers.openai.com/api/docs/models/gpt-5.4", "benchmarks": "https://openai.com/index/introducing-gpt-5-4/" }, "pricing": { "inputPerM": 2.5, "outputPerM": 15, "cachedInputPerM": 0.25, "note": "Long-context surcharge above 272K input per the official pricing page." }, "context": { "windowTokens": 1050000, "maxOutputTokens": 128000, "note": "The current OpenAI API model page lists a 1,050,000-token context window and a 128,000-token maximum output. Inputs beyond the standard 272K window carry the long-context surcharge described in OpenAI's launch material." }, "benchmarks": { "GDPval (wins or ties)": 83, "SWE-Bench Pro (Public)": 57.7, "OSWorld-Verified": 75, "BrowseComp": 82.7, "Toolathlon": 54.6, "SWE-bench Verified": null, "note": "All populated figures are from OpenAI's GPT-5.4 launch evaluation table. OpenAI publishes SWE-Bench Pro (Public), not a SWE-bench Verified figure, for GPT-5.4." }, "notes": "Released March 5, 2026 as GPT-5.4 Thinking + GPT-5.4 Pro ('most capable and efficient frontier model for professional work'); GPT-5.4 mini and nano followed March 17 (mini available to free tier, nano API-only). Built-in computer use; tuned for finance workflows (launched alongside ChatGPT for Excel, which it powered at launch). benchr removed an earlier gpt-5-4 entry on June 1, 2026 as 'unverified' — that removal was wrong; the model is real and was re-added after this June 10 verification. Superseded as OpenAI's flagship by GPT-5.5 (April 23, 2026)." } }, { "id": "gpt-5-5", "name": "GPT-5.5", "provider": "OpenAI", "api_name": "gpt-5.5", "type": "frontier", "license": "proprietary", "released": "2026-04-23", "deprecated": false, "pricing": { "input_per_million": 5, "output_per_million": 30, "cache_input_per_million": 0.5, "batch_discount": 0.5, "batch_input": 2.5, "batch_output": 15, "input_per_million_over_200k": 10, "output_per_million_over_200k": 45, "cache_input_per_million_over_200k": 1 }, "context": { "max_tokens": 1050000, "effective_tokens": 700000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 93, "reasoning": 95, "writing": 88, "vision": 89, "long_context": 86, "multilingual": 90 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Frontier math and reasoning", "Computer use", "Multi-step agents", "Vision + reasoning tasks" ], "skip_if": [ "Cost-sensitive workloads — use GPT-5", "Quick chat — the price is hard to justify" ], "official_evidence": { "record_id": "gpt-5-5", "api_model_id": "gpt-5.5", "verified_date": "2026-07-30", "sources": { "release": "https://deploymentsafety.openai.com/gpt-5-5", "pricing": "https://developers.openai.com/api/docs/models/gpt-5.5", "context": "https://developers.openai.com/api/docs/models/gpt-5.5", "benchmarks": "https://deploymentsafety.openai.com/gpt-5-5" }, "pricing": { "inputPerM": 5, "outputPerM": 30, "cachedInputPerM": 0.5, "batchInputPerM": 2.5, "batchOutputPerM": 15, "longContextInputPerM": 10, "longContextCachedInputPerM": 1, "longContextOutputPerM": 45, "regionalProcessingUplift": 0.1, "proInputPerM": 30, "proOutputPerM": 180, "proBatchInputPerM": 15, "proBatchOutputPerM": 90, "extendedContextSurcharge": "For sessions >272K input tokens: 2x input, 1.5x output (standard/batch/flex)." }, "context": { "windowTokens": 1050000, "maxOutputTokens": 128000 }, "benchmarks": { "HealthBench (length-adjusted)": 56.5, "HealthBench Professional": 51.8, "SWE-bench Verified": null, "SWE-bench Pro": null, "Terminal-Bench 2.0": null, "OSWorld-Verified": null }, "notes": "Flagship GPT-5.5 (and GPT-5.5 Pro) announced Apr 23, 2026 (do not confuse with GPT-5.5 Instant, the ChatGPT default released May 5, 2026). Context is 1,050,000 (not a round 1M). HealthBench figures are the only official system-card benchmarks. Rechecked 2026-07-30 against developers.openai.com/api/docs/models/gpt-5.5 and /api/docs/pricing: standard $5/$30 with $0.50 cached input; short-context batch $2.50/$15; long-context $10 input, $1 cached input, $45 output above 272K; eligible regional processing carries a 10% uplift. GPT-5.5 Pro is $30/$180 standard and $15/$90 batch with no cached-input discount. The coding/agent numbers circulating (Terminal-Bench 82.7%, OSWorld 78.7%, SWE-bench Pro 58.6%) remain null because the checked model page does not publish directly comparable figures." } }, { "id": "gpt-5-6", "name": "GPT-5.6", "provider": "OpenAI", "api_name": "gpt-5.6-sol", "type": "frontier", "license": "proprietary", "released": "2026-07-09", "deprecated": false, "pricing": { "input_per_million": 4, "output_per_million": 20, "cache_input_per_million": 0.4, "batch_discount": 0.5, "tier_prices_per_million": [ 2, 10, 8, 30 ], "batch_input": 2, "batch_output": 10, "input_per_million_over_200k": 8, "output_per_million_over_200k": 30, "cache_input_per_million_over_200k": 0.8 }, "context": { "max_tokens": 1050000, "effective_tokens": 750000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": 89.8, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": 91.2, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 95, "reasoning": 96, "writing": 89, "vision": 90, "long_context": 87, "multilingual": 91 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Generally available via OpenAI API since July 9, 2026", "New SOTA agentic command-line work (Terminal-Bench)", "Frontier math and reasoning", "Computer use and multi-step agents", "Teams that need OpenAI's absolute best" ], "skip_if": [ "Cost-sensitive workloads — Terra is still five times cheaper on input", "You need the deepest single-call reasoning ceiling — GPQA still trails Opus 4.8/Fable 5" ], "official_evidence": { "record_id": "gpt-5-6-sol", "api_model_id": "gpt-5.6-sol", "verified_date": "2026-08-24", "sources": { "release": "https://developers.openai.com/api/docs/changelog", "pricing": "https://developers.openai.com/api/docs/pricing", "benchmarks": "https://deploymentsafety.openai.com/gpt-5-6-preview", "context": "https://developers.openai.com/api/docs/models" }, "pricing": { "inputPerM": 4, "outputPerM": 20, "cachedInputPerM": 0.4, "cacheWritePerM": 5, "longContextInputPerM": 8, "longContextOutputPerM": 30, "longContextCachedInputPerM": 0.8, "longContextCacheWritePerM": 10, "batchInputPerM": 2, "batchOutputPerM": 10, "batchCachedInputPerM": 0.2, "flexInputPerM": 2, "flexOutputPerM": 10, "fastInputPerM": 8, "fastOutputPerM": 40, "priorityInputPerM": null, "priorityOutputPerM": null, "priceStatus": "Promotional pricing effective 2026-08-21, read live on developers.openai.com/api/docs/pricing on 2026-08-24.", "note": "OpenAI cut Sol's standard rate on August 21, 2026: input $5.00 -> $4.00 (20% lower), output $30.00 -> $20.00 (33% lower), cached input $0.50 -> $0.40. Short-context cache writes are $5.00. The long-context tier (above 272K input tokens) is $8.00 input / $0.80 cached / $10.00 cache write / $30.00 output. Batch and Flex are half of standard ($2.00 / $10.00); Flex lists no long-context row. Fast mode replaced Priority processing and bills at 2x standard ($8.00 / $40.00 short context, $16.00 / $60.00 long context), so the priority fields are null rather than carrying stale numbers. OpenAI's changelog states the promotional rate is available at least through November 21, 2026." }, "context": { "windowTokens": 1050000, "maxOutputTokens": 128000 }, "benchmarks": { "Terminal-Bench 2.1": 88.8, "Terminal-Bench 2.1 (ultra mode)": 91.9, "SWE-bench Verified": 89.8, "GPQA Diamond": 91.2, "note": "Terminal-Bench 2.1 figures are from the June 26 preview system card. SWE-bench Verified (89.8%) and GPQA Diamond (91.2%) were added to the preview system-card page on 2026-07-01; GA followed on 2026-07-09. All benchmark figures here are provider-reported." }, "notes": "UPDATED 2026-07-13: GPT-5.6 is now officially generally available, per OpenAI's July 9 API changelog and live models/pricing pages. The prior July 8 correction was accurate at that time but is now superseded. OpenAI's live models page lists a 1.05M context window and 128K max output for this tier; pricing is listed on the official pricing page. Re-verified 2026-08-24 after OpenAI's August 21 price cut; the $5/$30 rate recorded on 2026-07-13 is now historical and is logged in history.json." } }, { "id": "glm-5-2", "name": "GLM-5.2", "provider": "Z.AI", "api_name": "glm-5.2", "type": "frontier-open", "license": "open-weight / hosted API", "released": "2026-06-16", "deprecated": false, "pricing": { "input_per_million": 1.4, "output_per_million": 4.4, "cache_input_per_million": 0.26, "batch_discount": null, "self_hosted": true }, "context": { "max_tokens": 1000000, "effective_tokens": 750000, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 91, "reasoning": 89, "writing": 82, "vision": null, "long_context": 94, "multilingual": 86 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Long-horizon coding with 1M context", "Teams that want hosted API plus open-weight optionality", "Cost-sensitive agent engineering" ], "skip_if": [ "You need broad Western enterprise platform support", "You need official public benchmark numbers for every metric" ], "official_evidence": { "record_id": "glm-5-2", "api_model_id": "glm-5.2", "verified_date": "2026-07-13", "sources": { "release": "https://docs.z.ai/release-notes/new-released", "pricing": "https://docs.z.ai/guides/overview/pricing", "context": "https://docs.z.ai/guides/llm/glm-5.2", "migration": "https://docs.z.ai/guides/overview/migrate-to-glm-new" }, "pricing": { "inputPerM": 1.4, "outputPerM": 4.4, "cachedInputPerM": 0.26, "cacheStorage": "Limited-time free" }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": {}, "notes": "Z.AI's release notes list GLM-5.2 on 2026-06-16 with 1M lossless context and stronger long-horizon/coding performance. Model docs list 1M context and 128K maximum output tokens. Pricing docs list $1.40 input, $0.26 cached input, and $4.40 output per 1M tokens; cached input storage is listed as limited-time free. Official docs describe open-source SOTA performance but the checked docs page does not restate a precise license string." } }, { "id": "glm-5-3", "name": "GLM-5.3", "provider": "Z.AI", "api_name": "glm-5.3", "type": "frontier", "license": "proprietary API for now; Z.AI says open weights will follow after safety hardening", "released": "2026-08-18", "deprecated": false, "pricing": { "input_per_million": 1.4, "output_per_million": 4.4, "cache_input_per_million": 0.26, "batch_discount": null, "self_hosted": false }, "context": { "max_tokens": 1000000, "effective_tokens": null, "max_output_tokens": 128000 }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 94, "reasoning": 91, "writing": 83, "vision": null, "long_context": 95, "multilingual": 87 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Long-horizon code and terminal work with 1M context and 128K output", "Cost-sensitive agent deployments using compatible API protocols", "Defensive software-security analysis with explicit human review" ], "skip_if": [ "You need multimodal input", "You require downloadable weights before Z.AI completes its safety-hardening release" ], "official_evidence": { "record_id": "glm-5-3", "api_model_id": "glm-5.3", "verified_date": "2026-08-28", "sources": { "release": "https://docs.z.ai/release-notes/new-released", "pricing": "https://docs.z.ai/guides/overview/pricing", "context": "https://docs.z.ai/guides/llm/glm-5.3", "benchmarks": "https://docs.z.ai/guides/llm/glm-5.3" }, "pricing": { "inputPerM": 1.4, "outputPerM": 4.4, "cachedInputPerM": 0.26, "cacheStorage": "Limited-time free", "note": "Same list rate as GLM-5.2. Read on docs.z.ai/guides/overview/pricing on 2026-08-28." }, "context": { "windowTokens": 1000000, "maxOutputTokens": 128000 }, "benchmarks": { "CyberGym": 84.5, "Terminal-Bench 3.0": 28.3 }, "notes": "Z.ai released GLM-5.3 on August 18, 2026. The model guide states it uses the same base model as GLM-5.2 with the gains coming from post-training, reasoning is always on with low, high, or max effort (max is the default), and inputs are text-only. Z.ai reports a 50% gain over GLM-5.2 on its own Z.ai Code Bench, 84.5% on CyberGym, and a move from 4.6 to 28.3 on Terminal-Bench 3.0 - all provider-reported figures on a provider-run harness, not benchr tests, and Z.ai Code Bench is the vendor's own benchmark. Parameter counts are not published on the model guide, so they are omitted rather than inferred from GLM-5.2." } }, { "id": "grok-4-3", "name": "Grok 4.3", "provider": "xAI", "api_name": "grok-4.3", "type": "frontier", "license": "proprietary", "released": null, "deprecated": false, "pricing": { "input_per_million": 1.25, "output_per_million": 2.5, "cache_input_per_million": 0.2, "batch_discount": null, "batch_input": 1, "batch_output": 2, "input_per_million_over_200k": 2.5, "output_per_million_over_200k": 5, "cache_input_per_million_over_200k": 0.4, "long_context_threshold_tokens": 200000 }, "context": { "max_tokens": 1000000, "effective_tokens": 800000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 82, "reasoning": 84, "writing": 80, "vision": 78, "long_context": 85, "multilingual": 75 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Real-time data access via X", "Very cheap output tokens", "Long-context tasks on a budget" ], "skip_if": [ "Coding-first agent work — others edge it out", "Non-English-heavy workloads" ], "official_evidence": { "record_id": "grok-4-3", "api_model_id": "grok-4.3", "verified_date": "2026-07-30", "sources": { "pricing": "https://docs.x.ai/developers/models/grok-4.3", "context": "https://docs.x.ai/developers/models/grok-4.3" }, "pricing": { "inputPerM": 1.25, "outputPerM": 2.5, "cachedInputPerM": 0.2, "longContextThresholdTokens": 200000, "longContextInputPerM": 2.5, "longContextCachedInputPerM": 0.4, "longContextOutputPerM": 5, "batchInputPerM": 1, "batchCachedInputPerM": 0.16, "batchOutputPerM": 2, "batchLongContextInputPerM": 2, "batchLongContextCachedInputPerM": 0.32, "batchLongContextOutputPerM": 4, "note": "xAI charges the long-context tier for the entire request once the prompt reaches 200K tokens. The Batch API discount is 20% for Grok 4.3." }, "context": { "windowTokens": 1000000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "xAI officially publishes short-context price ($1.25 input / $0.20 cached / $2.50 output), long-context price at or above a 200K-token prompt ($2.50 / $0.40 / $5.00), a 20% Batch discount, 1M context, and the model id. It publishes no official release date, max output, or numeric benchmark table for Grok 4.3; those fields remain null. Figures like 'Intelligence Index 53' or 'tau2-bench Telecom 98%' are third-party, not official xAI numbers." } }, { "id": "grok-4-5", "name": "Grok 4.5", "provider": "xAI", "api_name": "grok-4.5", "type": "frontier", "license": "proprietary", "released": "2026-07-08", "deprecated": false, "pricing": { "input_per_million": 2, "output_per_million": 6, "cache_input_per_million": 0.3, "batch_discount": null, "input_per_million_over_200k": 4, "output_per_million_over_200k": 12, "cache_input_per_million_over_200k": 0.6, "long_context_threshold_tokens": 200000 }, "context": { "max_tokens": 500000, "effective_tokens": 400000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 93, "reasoning": 92, "writing": 87, "vision": 85, "long_context": 82, "multilingual": 81 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Coding and agentic workflows on xAI", "Lower output cost than GPT-5.5 class models", "Search-augmented knowledge work" ], "skip_if": [ "You need 1M context — Grok 4.3 has the larger window", "You need official benchmark tables before procurement" ], "official_evidence": { "record_id": "grok-4-5", "api_model_id": "grok-4.5", "verified_date": "2026-07-30", "sources": { "release": "https://docs.x.ai/developers/release-notes", "pricing": "https://docs.x.ai/developers/pricing", "context": "https://docs.x.ai/developers/models" }, "pricing": { "inputPerM": 2, "outputPerM": 6, "cachedInputPerM": 0.3, "longContextThresholdTokens": 200000, "longContextInputPerM": 4, "longContextCachedInputPerM": 0.6, "longContextOutputPerM": 12, "note": "Current xAI rates. Long-context pricing applies to the entire request once the prompt reaches 200K tokens. The Batch API currently provides no discount for Grok 4.5." }, "context": { "windowTokens": 500000, "maxOutputTokens": null }, "benchmarks": {}, "notes": "xAI release notes list Grok 4.5 on July 8, 2026 for coding, agentic tasks, and knowledge work. Current docs list model id grok-4.5, 500K context, $2 input / $0.30 cached / $6 output below 200K prompt tokens, and $4 / $0.60 / $12 at or above 200K, with configurable reasoning and a February 1, 2026 knowledge cutoff. xAI publishes no max output limit in the checked model documentation." } }, { "id": "grok-4-6", "name": "Grok 4.6", "provider": "xAI", "api_name": "grok-4.6", "type": "frontier", "license": "proprietary", "released": "2026-08-12", "deprecated": false, "pricing": { "input_per_million": 2, "output_per_million": 6, "cache_input_per_million": 0.5, "batch_discount": null, "input_per_million_over_200k": 4, "output_per_million_over_200k": 12, "cache_input_per_million_over_200k": 1, "long_context_threshold_tokens": 200000 }, "context": { "max_tokens": 500000, "effective_tokens": 400000, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": 94, "reasoning": 93, "writing": 88, "vision": 86, "long_context": 82, "multilingual": 81 }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Long-running coding and agent sessions on xAI", "Interactive and visual build work", "Same $2/$6 rate as Grok 4.5 with a stronger launch table" ], "skip_if": [ "You need 1M context — Grok 4.3 has the larger window", "Cache-heavy pipelines — cached input rose to $0.50 from Grok 4.5's $0.30", "You need third-party benchmark confirmation before procurement" ], "official_evidence": { "record_id": "grok-4-6", "api_model_id": "grok-4.6", "verified_date": "2026-08-24", "sources": { "release": "https://x.ai/news/grok-4-6", "pricing": "https://docs.x.ai/developers/models", "context": "https://docs.x.ai/developers/models", "benchmarks": "https://x.ai/news/grok-4-6" }, "pricing": { "inputPerM": 2, "outputPerM": 6, "cachedInputPerM": 0.5, "longContextThresholdTokens": 200000, "longContextInputPerM": 4, "longContextCachedInputPerM": 1, "longContextOutputPerM": 12, "note": "xAI lists $2 input / $0.50 cached / $6 output per 1M tokens below 200K prompt tokens, and $4 / $1 / $12 at or above 200K. The launch post adds a fast variant at double the standard rate. Cached input costs more than Grok 4.5's $0.30. Read on docs.x.ai/developers/models on 2026-08-24." }, "context": { "windowTokens": 500000, "maxOutputTokens": null }, "benchmarks": { "GDPval-AA v2 (Elo)": 1753, "CursorBench v3.2": 69.9, "DeepSWE v1.1": 65.9, "FrontierCode v1.1": 61.3, "note": "Figures come from xAI's own launch table for Grok 4.6 High and are provider-reported, not a benchr test. The same table lists an Artificial Analysis Intelligence Index of 61 — a third-party index republished by xAI, so it is not recorded as an xAI benchmark here. For contrast the table shows Grok 4.5 High at GDPval-AA v2 1526, CursorBench v3.2 66.7%, DeepSWE v1.1 54%, FrontierCode v1.1 56.6%." }, "notes": "xAI announced Grok 4.6 on August 12, 2026 as its frontier model for coding, agentic tasks, and knowledge work, aimed at long-running agents and interactive or visual projects, with stronger first passes and more self-testing during long tasks. The model docs list a 500,000-token context window, text and image input, text-only output with no published output-token limit, and a February 1, 2026 knowledge cutoff." } }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "provider": "OpenAI", "api_name": "gpt-6-astra", "type": "frontier", "license": "proprietary", "released": "2026-09-03", "deprecated": false, "pricing": { "input_per_million": 10, "output_per_million": 50, "cache_input_per_million": 1, "batch_discount": null, "batch_input": 5, "batch_output": 25, "free_tier": false }, "context": { "max_tokens": 1050000, "effective_tokens": null, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": null, "reasoning": null, "writing": null, "vision": null, "long_context": null, "multilingual": null }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Long end-to-end work OpenAI aims this model at - reasoning, coding, computer use, research", "Prompts that need more than the 400K window of the GPT-5.6 family", "Teams already on the Responses API, which this model requires for tool calling" ], "skip_if": [ "You send custom temperature, top_p or logprobs - this model accepts none of them", "You rely on the `none` reasoning-effort level, which it does not support", "You call tools through Chat Completions rather than the Responses API", "Cost is the constraint: input is 2.5x GPT-5.6 Sol and output is 2.5x its rate", "You need benchr-held benchmark figures - OpenAI published none with this release" ], "official_evidence": { "record_id": "gpt-6-astra", "api_model_id": "gpt-6-astra", "verified_date": "2026-09-08", "sources": { "release": "https://developers.openai.com/api/docs/changelog", "pricing": "https://developers.openai.com/api/docs/pricing", "context": "https://developers.openai.com/api/docs/models" }, "pricing": { "inputPerM": 10, "outputPerM": 50, "cachedInputPerM": 1, "batchInputPerM": 5, "batchOutputPerM": 25, "batchCachedInputPerM": 0.5, "note": "Read on developers.openai.com/api/docs/pricing on 2026-09-08. Standard $10.00 input / $1.00 cached input / $50.00 output per 1M tokens; Batch is half of each. That is 2.5x GPT-5.6 Sol's post-cut input rate and 2.5x its output rate. No long-context, flex, fast or priority tier is published for this model, unlike Sol." }, "context": { "windowTokens": 1050000, "maxOutputTokens": null }, "benchmarks": { "note": "OpenAI published no benchmark table with the release announcement or on the model directory entry, so every numeric field stays null." }, "notes": "Announced in OpenAI's API changelog on September 3, 2026 as \"our most capable model, built for the hardest end-to-end work\", aimed at reasoning, coding, computer use, research and document creation. The model directory lists a 1.05M context window and an April 30, 2026 knowledge cutoff; it publishes no maximum output figure, so that field stays null rather than being copied from the GPT-5.6 family. The changelog records real interface constraints: the model does not support the `none` reasoning-effort level, does not accept custom temperature, top_p or logprobs, requires the Responses API for tool calling rather than Chat Completions, and is subject to misalignment monitoring. Those are migration blockers, not footnotes. benchr previously recorded that Astra was announced but unreleased with no model card, API id or price; that was accurate until this release and is superseded by this record." } }, { "id": "qwen-3-8-max", "name": "Qwen3.8-Max", "provider": "Alibaba (Qwen)", "api_name": "qwen3.8-max", "type": "frontier", "license": "proprietary API; Alibaba said weights would follow, and benchr has verified no licence page", "released": "2026-08-03", "deprecated": false, "pricing": { "input_per_million": 2, "output_per_million": 6, "cache_input_per_million": null, "batch_discount": null, "batch_input": null, "batch_output": null, "free_tier": false }, "context": { "max_tokens": 1000000, "effective_tokens": null, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": null, "reasoning": null, "writing": null, "vision": null, "long_context": null, "multilingual": null }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "Frontier-tier work at a rate well under the US flagships", "Text and vision in one model, on a context Alibaba describes as up to 1M tokens", "Workloads already running in Alibaba Cloud Model Studio" ], "skip_if": [ "You need published benchmark scores - Alibaba gave arena placements, not numbers", "Your licence review needs a named licence, which no official page states", "You are outside the Singapore region, where the published rate may not apply" ], "official_evidence": { "record_id": "qwen-3-8-max", "api_model_id": "qwen3.8-max", "verified_date": "2026-09-08", "sources": { "release": "https://www.alibabacloud.com/blog/alibaba-unveils-qwen3-8-max-its-largest-and-most-capable-flagship-model-to-date_603420", "pricing": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "context": "https://www.alibabacloud.com/blog/alibaba-unveils-qwen3-8-max-its-largest-and-most-capable-flagship-model-to-date_603420" }, "pricing": { "inputPerM": 2, "outputPerM": 6, "note": "Singapore region rate on the Model Studio pricing table, read 2026-09-08. Alibaba publishes region-specific rates; other regions may differ." }, "context": { "windowTokens": 1000000, "maxOutputTokens": null }, "benchmarks": { "note": "Alibaba published arena placements rather than numeric scores at launch — fifth in Text Arena, second in Vision Arena, fourth in Frontend Code Arena. A placement is not a score, so every numeric field stays null." }, "notes": "Alibaba announced Qwen3.8-Max on August 3, 2026 as its largest flagship: a sparse mixture-of-experts model with 2.4T total parameters activating 95B per token, multimodal across text and vision, with a context window the announcement describes as 'up to 1 million tokens'. The announcement said weights would follow the next week; benchr has not verified a weights release or a licence on an official page, so license stays null rather than being assumed. No maximum-output figure was published on either the announcement or the pricing table." } }, { "id": "qwen-3-8-flash", "name": "Qwen3.8-Flash", "provider": "Alibaba (Qwen)", "api_name": "qwen3.8-flash", "type": "mid", "license": "hosted API; the announcement calls it open-weight but names no licence, and benchr has verified none", "released": "2026-08-27", "deprecated": false, "pricing": { "input_per_million": 0.15, "output_per_million": 0.47, "cache_input_per_million": null, "batch_discount": null, "batch_input": null, "batch_output": null, "free_tier": false }, "context": { "max_tokens": 262144, "effective_tokens": null, "max_output_tokens": null }, "benchmarks": { "values": { "swe_bench_verified": null, "lmsys_arena": null, "mmlu": null, "humaneval": null, "math": null, "gpqa_diamond": null, "arc_agi_2": null }, "estimated_fields": [], "provenance": "Mixed: fields named in estimated_fields are benchr editorial estimates; other numeric benchmark fields are reconciled to the verified record when available." }, "capabilities": { "values": { "coding": null, "reasoning": null, "writing": null, "vision": null, "long_context": null, "multilingual": null }, "provenance": "benchr editorial 0–100 ratings; not provider benchmarks." }, "latency": { "first_token_ms": null, "tokens_per_second": null, "provenance": "benchr editorial tool estimate; not a controlled lab measurement." }, "best_for": [ "High-volume multimodal work - among the lowest published rates benchr records", "Agent loops that fit the native 262K window", "Cost-first prototyping before committing to a frontier tier" ], "skip_if": [ "You need the 1M context the announcement mentions - benchr records the native 262K", "You need scores: Alibaba named six benchmarks and published no result for any of them", "Your licence review needs a named licence" ], "official_evidence": { "record_id": "qwen-3-8-flash", "api_model_id": "qwen3.8-flash", "verified_date": "2026-09-08", "sources": { "release": "https://www.alibabacloud.com/blog/alibaba-releases-qwen3-8-flash-with-innovative-model-architecture-delivering-optimal-price-performance_603503", "pricing": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "context": "https://www.alibabacloud.com/blog/alibaba-releases-qwen3-8-flash-with-innovative-model-architecture-delivering-optimal-price-performance_603503" }, "pricing": { "inputPerM": 0.15, "outputPerM": 0.47, "note": "Singapore region rate on the Model Studio pricing table, read 2026-09-08. Alibaba publishes region-specific rates; other regions may differ." }, "context": { "windowTokens": 262144, "maxOutputTokens": null }, "benchmarks": { "note": "Alibaba named SWE-bench Pro, CoWorkBench, Toolathlon Verified, MathVision, AndroidWorld and ERQA in the release but published no scores for them, so every numeric field stays null." }, "notes": "Released August 27, 2026 as a multimodal mixture-of-experts model: a 125B main model with a further 51B of N-gram embeddings, activating 6B parameters per token. The announcement states a native 262K context extendable to 1 million tokens; benchr records the native figure, because the extended limit is a capability of a configuration rather than the default window. Alibaba's headline calls it open-weight but the announcement names no licence, and benchr has not verified one on an official page, so license stays null. No maximum-output figure was published." } } ] }