{ "schema": "modelrater-popular-telemetry/1", "updated_at": "2026-10-27T18:01:07.710859", "total_models": 35, "models": [ { "id": "anthropic/claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "maker": "anthropic", "family": "claude-3-5", "status": "active", "weights": "closed", "context_window": 200000, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 3.0, "completion_per_m": 15.0, "cache_read_per_m": 0.3, "cache_write_per_m": 3.75, "cache_discount_pct": 90.0, "cache_ttl": "5m" }, "benchmarks": { "lmsys_arena_elo": 1335, "coding_elo": 1335, "hard_prompts_elo": 1342, "swe_bench_verified": 49.0, "swe_bench_lite": 38.8, "livecodebench_pass1": 56.2, "humaneval_pass1": 93.7, "mbpp_pass1": 89.2, "aider_diff_pass": 77.2, "math_500": 78.0, "gsm8k": 96.4, "gpqa_diamond": 65.0, "ifeval": 88.0, "bfcl_tool_calling": 89.5, "aa_intelligence_index": 72.4, "aa_coding_index": 85.2, "aa_agentic_index": 68.9, "median_tps": 82.0, "median_ttft_ms": 380.0 }, "empirical_dyno": { "awarded_score": 63, "cautious_fill": 60, "high_fill": 71, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "qwen/qwen-2.5-coder-32b", "wingman_orthogonality": 8.4, "wingman_rescue_rate": 88.0 }, "providers": [ "Anthropic Native", "AWS Bedrock", "Google Vertex AI" ] }, { "id": "anthropic/claude-3-5-haiku", "name": "Claude 3.5 Haiku", "maker": "anthropic", "family": "claude-3-5", "status": "active", "weights": "closed", "context_window": 200000, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.8, "completion_per_m": 4.0, "cache_read_per_m": 0.08, "cache_write_per_m": 1.0, "cache_discount_pct": 90.0, "cache_ttl": "5m" }, "benchmarks": { "lmsys_arena_elo": 1290, "coding_elo": 1295, "hard_prompts_elo": 1280, "swe_bench_verified": 40.6, "swe_bench_lite": 32.4, "livecodebench_pass1": 46.8, "humaneval_pass1": 88.9, "mbpp_pass1": 85.0, "aider_diff_pass": 64.0, "math_500": 69.4, "gsm8k": 93.0, "gpqa_diamond": 51.2, "ifeval": 84.5, "bfcl_tool_calling": 84.0, "aa_intelligence_index": 52.0, "aa_coding_index": 68.0, "aa_agentic_index": 54.0, "median_tps": 115.0, "median_ttft_ms": 280.0 }, "empirical_dyno": { "awarded_score": 63, "cautious_fill": 60, "high_fill": 71, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4-flash", "wingman_orthogonality": 7.8, "wingman_rescue_rate": 84.0 }, "providers": [ "Anthropic Native", "AWS Bedrock", "Google Vertex AI" ] }, { "id": "anthropic/claude-3-opus", "name": "Claude 3 Opus", "maker": "anthropic", "family": "claude-3", "status": "active", "weights": "closed", "context_window": 200000, "max_output_tokens": 4096, "pricing": { "prompt_per_m": 15.0, "completion_per_m": 75.0, "cache_read_per_m": 1.5, "cache_write_per_m": 18.75, "cache_discount_pct": 90.0, "cache_ttl": "5m" }, "benchmarks": { "lmsys_arena_elo": 1270, "coding_elo": 1260, "hard_prompts_elo": 1285, "swe_bench_verified": 32.0, "swe_bench_lite": 28.7, "livecodebench_pass1": 38.0, "humaneval_pass1": 84.9, "mbpp_pass1": 86.4, "aider_diff_pass": 68.4, "math_500": 60.1, "gsm8k": 95.0, "gpqa_diamond": 50.4, "ifeval": 84.0, "bfcl_tool_calling": 81.0, "aa_intelligence_index": 48.0, "aa_coding_index": 62.0, "aa_agentic_index": 50.0, "median_tps": 35.0, "median_ttft_ms": 680.0 }, "empirical_dyno": { "awarded_score": 63, "cautious_fill": 60, "high_fill": 71, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "meta-llama/llama-3.3-70b-instruct", "wingman_orthogonality": 6.9, "wingman_rescue_rate": 79.0 }, "providers": [ "Anthropic Native", "AWS Bedrock", "Google Vertex AI" ] }, { "id": "openai/gpt-4o", "name": "GPT-4o", "maker": "openai", "family": "gpt-4", "status": "active", "weights": "closed", "context_window": 128000, "max_output_tokens": 16384, "pricing": { "prompt_per_m": 2.5, "completion_per_m": 10.0, "cache_read_per_m": 1.25, "cache_write_per_m": 2.5, "cache_discount_pct": 50.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1332, "coding_elo": 1330, "hard_prompts_elo": 1338, "swe_bench_verified": 38.8, "swe_bench_lite": 33.2, "livecodebench_pass1": 53.4, "humaneval_pass1": 90.2, "mbpp_pass1": 87.0, "aider_diff_pass": 72.9, "math_500": 76.6, "gsm8k": 95.8, "gpqa_diamond": 53.6, "ifeval": 84.3, "bfcl_tool_calling": 88.2, "aa_intelligence_index": 68.0, "aa_coding_index": 78.0, "aa_agentic_index": 64.0, "median_tps": 120.0, "median_ttft_ms": 320.0 }, "empirical_dyno": { "awarded_score": 58, "cautious_fill": 55, "high_fill": 68, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 7.4, "wingman_rescue_rate": 81.0 }, "providers": [ "OpenAI Native", "Microsoft Azure", "OpenRouter" ] }, { "id": "openai/gpt-4o-mini", "name": "GPT-4o Mini", "maker": "openai", "family": "gpt-4", "status": "active", "weights": "closed", "context_window": 128000, "max_output_tokens": 16384, "pricing": { "prompt_per_m": 0.15, "completion_per_m": 0.6, "cache_read_per_m": 0.075, "cache_write_per_m": 0.15, "cache_discount_pct": 50.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1275, "coding_elo": 1270, "hard_prompts_elo": 1265, "swe_bench_verified": 16.5, "swe_bench_lite": 14.2, "livecodebench_pass1": 39.5, "humaneval_pass1": 87.2, "mbpp_pass1": 82.4, "aider_diff_pass": 55.4, "math_500": 70.2, "gsm8k": 91.6, "gpqa_diamond": 40.2, "ifeval": 80.0, "bfcl_tool_calling": 84.0, "aa_intelligence_index": 44.0, "aa_coding_index": 56.0, "aa_agentic_index": 48.0, "median_tps": 145.0, "median_ttft_ms": 250.0 }, "empirical_dyno": { "awarded_score": 52, "cautious_fill": 50, "high_fill": 64, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "google/gemini-2.5-flash", "wingman_orthogonality": 7.1, "wingman_rescue_rate": 78.0 }, "providers": [ "OpenAI Native", "Microsoft Azure", "OpenRouter" ] }, { "id": "openai/o1", "name": "o1 (Full Reasoning)", "maker": "openai", "family": "o1", "status": "active", "weights": "closed", "context_window": 200000, "max_output_tokens": 100000, "pricing": { "prompt_per_m": 15.0, "completion_per_m": 60.0, "cache_read_per_m": 7.5, "cache_write_per_m": 15.0, "cache_discount_pct": 50.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1365, "coding_elo": 1375, "hard_prompts_elo": 1385, "swe_bench_verified": 48.9, "swe_bench_lite": 41.5, "livecodebench_pass1": 63.8, "humaneval_pass1": 92.4, "mbpp_pass1": 91.0, "aider_diff_pass": 79.0, "math_500": 96.4, "gsm8k": 98.2, "gpqa_diamond": 77.3, "ifeval": 89.0, "bfcl_tool_calling": 88.0, "aa_intelligence_index": 92.0, "aa_coding_index": 94.0, "aa_agentic_index": 85.0, "median_tps": 42.0, "median_ttft_ms": 2800.0 }, "empirical_dyno": { "awarded_score": 74, "cautious_fill": 74, "high_fill": 82, "axes_scored": 4, "scars": [ "REASON_DOUBLE_400" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.9, "wingman_rescue_rate": 92.0 }, "providers": [ "OpenAI Native", "Microsoft Azure" ] }, { "id": "openai/o1-mini", "name": "o1-mini (Reasoning)", "maker": "openai", "family": "o1", "status": "active", "weights": "closed", "context_window": 128000, "max_output_tokens": 65536, "pricing": { "prompt_per_m": 3.0, "completion_per_m": 12.0, "cache_read_per_m": 1.5, "cache_write_per_m": 3.0, "cache_discount_pct": 50.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1315, "coding_elo": 1320, "hard_prompts_elo": 1325, "swe_bench_verified": 41.0, "swe_bench_lite": 35.0, "livecodebench_pass1": 56.0, "humaneval_pass1": 90.0, "mbpp_pass1": 88.0, "aider_diff_pass": 71.0, "math_500": 90.0, "gsm8k": 96.0, "gpqa_diamond": 60.0, "ifeval": 84.0, "bfcl_tool_calling": 85.0, "aa_intelligence_index": 76.0, "aa_coding_index": 82.0, "aa_agentic_index": 68.0, "median_tps": 75.0, "median_ttft_ms": 1400.0 }, "empirical_dyno": { "awarded_score": 68, "cautious_fill": 65, "high_fill": 75, "axes_scored": 3, "scars": [ "REASON_DOUBLE_400" ], "wingman_partner": "deepseek/deepseek-v4-flash", "wingman_orthogonality": 7.8, "wingman_rescue_rate": 84.0 }, "providers": [ "OpenAI Native", "Microsoft Azure" ] }, { "id": "deepseek/deepseek-v4.1-flash", "name": "DeepSeek V4.1 Flash", "maker": "deepseek", "family": "deepseek-v4", "status": "active", "weights": "open-weights", "context_window": 128000, "max_output_tokens": 16384, "pricing": { "prompt_per_m": 0.14, "completion_per_m": 0.28, "cache_read_per_m": 0.014, "cache_write_per_m": 0.14, "cache_discount_pct": 90.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1315, "coding_elo": 1315, "hard_prompts_elo": 1310, "swe_bench_verified": 36.0, "swe_bench_lite": 30.5, "livecodebench_pass1": 51.0, "humaneval_pass1": 89.5, "mbpp_pass1": 86.2, "aider_diff_pass": 71.5, "math_500": 75.4, "gsm8k": 94.8, "gpqa_diamond": 52.0, "ifeval": 83.5, "bfcl_tool_calling": 86.0, "aa_intelligence_index": 54.2, "aa_coding_index": 72.8, "aa_agentic_index": 58.1, "median_tps": 142.5, "median_ttft_ms": 210.0 }, "empirical_dyno": { "awarded_score": 67, "cautious_fill": 66, "high_fill": 71, "axes_scored": 5, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "z-ai/glm-5.3-flash", "wingman_orthogonality": 9.2, "wingman_rescue_rate": 91.0 }, "providers": [ "DeepSeek Native", "DeepInfra", "Fireworks", "Together", "Novita", "Chutes" ] }, { "id": "deepseek/deepseek-v3", "name": "DeepSeek V3 (671B MoE)", "maker": "deepseek", "family": "deepseek-v3", "status": "active", "weights": "open-weights", "context_window": 163840, "max_output_tokens": 32768, "pricing": { "prompt_per_m": 0.27, "completion_per_m": 1.1, "cache_read_per_m": 0.07, "cache_write_per_m": 0.27, "cache_discount_pct": 74.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1338, "coding_elo": 1335, "hard_prompts_elo": 1340, "swe_bench_verified": 49.2, "swe_bench_lite": 41.0, "livecodebench_pass1": 58.0, "humaneval_pass1": 91.2, "mbpp_pass1": 89.0, "aider_diff_pass": 76.8, "math_500": 90.2, "gsm8k": 96.5, "gpqa_diamond": 59.1, "ifeval": 86.8, "bfcl_tool_calling": 88.5, "aa_intelligence_index": 78.5, "aa_coding_index": 84.0, "aa_agentic_index": 67.2, "median_tps": 95.0, "median_ttft_ms": 340.0 }, "empirical_dyno": { "awarded_score": 72, "cautious_fill": 70, "high_fill": 78, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.6, "wingman_rescue_rate": 89.0 }, "providers": [ "DeepSeek Native", "DeepInfra", "Fireworks", "Together", "Hyperbolic", "Novita" ] }, { "id": "deepseek/deepseek-r1", "name": "DeepSeek R1 (Reasoning)", "maker": "deepseek", "family": "deepseek-r1", "status": "active", "weights": "open-weights", "context_window": 163840, "max_output_tokens": 65536, "pricing": { "prompt_per_m": 0.55, "completion_per_m": 2.19, "cache_read_per_m": 0.14, "cache_write_per_m": 0.55, "cache_discount_pct": 74.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1360, "coding_elo": 1368, "hard_prompts_elo": 1375, "swe_bench_verified": 49.2, "swe_bench_lite": 42.0, "livecodebench_pass1": 65.9, "humaneval_pass1": 93.0, "mbpp_pass1": 91.5, "aider_diff_pass": 78.4, "math_500": 97.3, "gsm8k": 98.4, "gpqa_diamond": 71.5, "ifeval": 88.2, "bfcl_tool_calling": 87.0, "aa_intelligence_index": 91.0, "aa_coding_index": 92.0, "aa_agentic_index": 82.0, "median_tps": 58.0, "median_ttft_ms": 1800.0 }, "empirical_dyno": { "awarded_score": 75, "cautious_fill": 75, "high_fill": 82, "axes_scored": 4, "scars": [ "REASON_DOUBLE_400" ], "wingman_partner": "openai/o1", "wingman_orthogonality": 8.1, "wingman_rescue_rate": 86.0 }, "providers": [ "DeepSeek Native", "DeepInfra", "Fireworks", "Together", "Chutes", "Nebius" ] }, { "id": "google/gemini-2.5-flash", "name": "Gemini 2.5 Flash", "maker": "google", "family": "gemini-2-5", "status": "active", "weights": "closed", "context_window": 1048576, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.15, "completion_per_m": 0.6, "cache_read_per_m": 0.0375, "cache_write_per_m": 0.15, "cache_discount_pct": 75.0, "cache_ttl": "1h" }, "benchmarks": { "lmsys_arena_elo": 1310, "coding_elo": 1310, "hard_prompts_elo": 1305, "swe_bench_verified": 35.0, "swe_bench_lite": 30.0, "livecodebench_pass1": 48.0, "humaneval_pass1": 88.0, "mbpp_pass1": 85.0, "aider_diff_pass": 68.0, "math_500": 74.0, "gsm8k": 94.0, "gpqa_diamond": 50.0, "ifeval": 84.0, "bfcl_tool_calling": 87.0, "aa_intelligence_index": 56.0, "aa_coding_index": 70.0, "aa_agentic_index": 62.0, "median_tps": 160.0, "median_ttft_ms": 230.0 }, "empirical_dyno": { "awarded_score": 62, "cautious_fill": 60, "high_fill": 70, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 8.0, "wingman_rescue_rate": 85.0 }, "providers": [ "Google AI Studio", "Google Vertex AI" ] }, { "id": "google/gemini-1.5-pro", "name": "Gemini 1.5 Pro", "maker": "google", "family": "gemini-1-5", "status": "active", "weights": "closed", "context_window": 2097152, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 1.25, "completion_per_m": 5.0, "cache_read_per_m": 0.3125, "cache_write_per_m": 1.25, "cache_discount_pct": 75.0, "cache_ttl": "1h" }, "benchmarks": { "lmsys_arena_elo": 1300, "coding_elo": 1300, "hard_prompts_elo": 1315, "swe_bench_verified": 32.0, "swe_bench_lite": 27.5, "livecodebench_pass1": 46.0, "humaneval_pass1": 84.0, "mbpp_pass1": 83.5, "aider_diff_pass": 64.2, "math_500": 67.7, "gsm8k": 93.2, "gpqa_diamond": 58.5, "ifeval": 82.0, "bfcl_tool_calling": 85.0, "aa_intelligence_index": 62.0, "aa_coding_index": 72.0, "aa_agentic_index": 65.0, "median_tps": 70.0, "median_ttft_ms": 480.0 }, "empirical_dyno": { "awarded_score": 60, "cautious_fill": 58, "high_fill": 69, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 7.9, "wingman_rescue_rate": 83.0 }, "providers": [ "Google AI Studio", "Google Vertex AI" ] }, { "id": "x-ai/grok-4.5", "name": "Grok 4.5 / 4.7 Super Heavy", "maker": "xai", "family": "grok-4", "status": "active", "weights": "closed", "context_window": 2000000, "max_output_tokens": 16384, "pricing": { "prompt_per_m": 3.0, "completion_per_m": 15.0, "cache_read_per_m": 0.75, "cache_write_per_m": 3.0, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1325, "coding_elo": 1325, "hard_prompts_elo": 1330, "swe_bench_verified": 42.0, "swe_bench_lite": 36.0, "livecodebench_pass1": 54.0, "humaneval_pass1": 91.0, "mbpp_pass1": 89.0, "aider_diff_pass": 74.0, "math_500": 82.0, "gsm8k": 96.0, "gpqa_diamond": 58.0, "ifeval": 85.0, "bfcl_tool_calling": 87.5, "aa_intelligence_index": 70.0, "aa_coding_index": 80.0, "aa_agentic_index": 66.0, "median_tps": 90.0, "median_ttft_ms": 350.0 }, "empirical_dyno": { "awarded_score": 64, "cautious_fill": 63, "high_fill": 69, "axes_scored": 5, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.3, "wingman_rescue_rate": 87.0 }, "providers": [ "xAI Native", "OpenRouter" ] }, { "id": "qwen/qwen-2.5-coder-32b", "name": "Qwen 2.5 Coder 32B", "maker": "alibaba", "family": "qwen-2-5-coder", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.2, "completion_per_m": 0.4, "cache_read_per_m": 0.05, "cache_write_per_m": 0.2, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1305, "coding_elo": 1305, "hard_prompts_elo": 1290, "swe_bench_verified": 31.6, "swe_bench_lite": 28.0, "livecodebench_pass1": 50.2, "humaneval_pass1": 92.7, "mbpp_pass1": 90.1, "aider_diff_pass": 73.7, "math_500": 76.0, "gsm8k": 93.5, "gpqa_diamond": 48.0, "ifeval": 81.0, "bfcl_tool_calling": 85.0, "aa_intelligence_index": 58.0, "aa_coding_index": 76.0, "aa_agentic_index": 55.0, "median_tps": 110.0, "median_ttft_ms": 260.0 }, "empirical_dyno": { "awarded_score": 60, "cautious_fill": 58, "high_fill": 68, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 8.7, "wingman_rescue_rate": 89.0 }, "providers": [ "Alibaba Cloud", "DeepInfra", "Together", "Fireworks", "Groq", "Ollama" ] }, { "id": "qwen/qwen-2.5-72b-instruct", "name": "Qwen 2.5 72B Instruct", "maker": "alibaba", "family": "qwen-2-5", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.35, "completion_per_m": 0.8, "cache_read_per_m": 0.08, "cache_write_per_m": 0.35, "cache_discount_pct": 77.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1320, "coding_elo": 1310, "hard_prompts_elo": 1318, "swe_bench_verified": 32.5, "swe_bench_lite": 28.5, "livecodebench_pass1": 49.0, "humaneval_pass1": 89.0, "mbpp_pass1": 88.0, "aider_diff_pass": 71.0, "math_500": 83.0, "gsm8k": 95.0, "gpqa_diamond": 52.0, "ifeval": 84.0, "bfcl_tool_calling": 86.0, "aa_intelligence_index": 66.0, "aa_coding_index": 74.0, "aa_agentic_index": 60.0, "median_tps": 75.0, "median_ttft_ms": 320.0 }, "empirical_dyno": { "awarded_score": 62, "cautious_fill": 60, "high_fill": 70, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.2, "wingman_rescue_rate": 86.0 }, "providers": [ "Alibaba Cloud", "DeepInfra", "Together", "Fireworks", "Ollama" ] }, { "id": "meta-llama/llama-3.3-70b-instruct", "name": "LLaMA 3.3 70B Instruct", "maker": "meta", "family": "llama-3-3", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 32768, "pricing": { "prompt_per_m": 0.23, "completion_per_m": 0.4, "cache_read_per_m": 0.05, "cache_write_per_m": 0.23, "cache_discount_pct": 78.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1290, "coding_elo": 1285, "hard_prompts_elo": 1295, "swe_bench_verified": 28.0, "swe_bench_lite": 24.0, "livecodebench_pass1": 44.5, "humaneval_pass1": 86.0, "mbpp_pass1": 84.0, "aider_diff_pass": 66.0, "math_500": 72.0, "gsm8k": 93.0, "gpqa_diamond": 46.0, "ifeval": 82.0, "bfcl_tool_calling": 83.0, "aa_intelligence_index": 55.0, "aa_coding_index": 66.0, "aa_agentic_index": 54.0, "median_tps": 95.0, "median_ttft_ms": 280.0 }, "empirical_dyno": { "awarded_score": 58, "cautious_fill": 55, "high_fill": 67, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 8.1, "wingman_rescue_rate": 84.0 }, "providers": [ "Meta Native", "Groq", "DeepInfra", "Together", "Fireworks", "Cerebras", "AWS Bedrock" ] }, { "id": "meta-llama/llama-3.1-405b-instruct", "name": "LLaMA 3.1 405B Instruct", "maker": "meta", "family": "llama-3-1", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 16384, "pricing": { "prompt_per_m": 1.5, "completion_per_m": 3.0, "cache_read_per_m": 0.35, "cache_write_per_m": 1.5, "cache_discount_pct": 76.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1330, "coding_elo": 1320, "hard_prompts_elo": 1335, "swe_bench_verified": 38.0, "swe_bench_lite": 33.0, "livecodebench_pass1": 52.0, "humaneval_pass1": 89.0, "mbpp_pass1": 88.5, "aider_diff_pass": 71.0, "math_500": 78.5, "gsm8k": 96.0, "gpqa_diamond": 51.0, "ifeval": 87.0, "bfcl_tool_calling": 86.5, "aa_intelligence_index": 67.0, "aa_coding_index": 76.0, "aa_agentic_index": 62.0, "median_tps": 40.0, "median_ttft_ms": 520.0 }, "empirical_dyno": { "awarded_score": 64, "cautious_fill": 62, "high_fill": 72, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.3, "wingman_rescue_rate": 86.0 }, "providers": [ "Together", "Fireworks", "DeepInfra", "Azure", "AWS Bedrock" ] }, { "id": "mistralai/mistral-large-2411", "name": "Mistral Large 2 (2411)", "maker": "mistral", "family": "mistral-large", "status": "active", "weights": "closed", "context_window": 128000, "max_output_tokens": 16384, "pricing": { "prompt_per_m": 2.0, "completion_per_m": 6.0, "cache_read_per_m": 0.5, "cache_write_per_m": 2.0, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1300, "coding_elo": 1295, "hard_prompts_elo": 1305, "swe_bench_verified": 38.0, "swe_bench_lite": 31.0, "livecodebench_pass1": 49.0, "humaneval_pass1": 88.0, "mbpp_pass1": 87.0, "aider_diff_pass": 69.0, "math_500": 74.0, "gsm8k": 94.0, "gpqa_diamond": 50.0, "ifeval": 84.0, "bfcl_tool_calling": 85.0, "aa_intelligence_index": 62.0, "aa_coding_index": 73.0, "aa_agentic_index": 59.0, "median_tps": 75.0, "median_ttft_ms": 320.0 }, "empirical_dyno": { "awarded_score": 61, "cautious_fill": 59, "high_fill": 69, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 7.8, "wingman_rescue_rate": 83.0 }, "providers": [ "Mistral Native", "Azure", "AWS Bedrock" ] }, { "id": "mistralai/codestral-2501", "name": "Codestral 22B (2501)", "maker": "mistral", "family": "codestral", "status": "active", "weights": "open-weights", "context_window": 256000, "max_output_tokens": 16384, "pricing": { "prompt_per_m": 0.3, "completion_per_m": 0.9, "cache_read_per_m": 0.075, "cache_write_per_m": 0.3, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1280, "coding_elo": 1290, "hard_prompts_elo": 1275, "swe_bench_verified": 29.0, "swe_bench_lite": 25.0, "livecodebench_pass1": 46.0, "humaneval_pass1": 89.0, "mbpp_pass1": 86.5, "aider_diff_pass": 67.5, "math_500": 70.0, "gsm8k": 91.0, "gpqa_diamond": 45.0, "ifeval": 80.0, "bfcl_tool_calling": 83.0, "aa_intelligence_index": 52.0, "aa_coding_index": 71.0, "aa_agentic_index": 52.0, "median_tps": 105.0, "median_ttft_ms": 260.0 }, "empirical_dyno": { "awarded_score": 58, "cautious_fill": 55, "high_fill": 67, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.0, "wingman_rescue_rate": 85.0 }, "providers": [ "Mistral Native", "Together", "DeepInfra", "Fireworks" ] }, { "id": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "maker": "google", "family": "gemini-3", "status": "active", "weights": "closed", "context_window": 1048576, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.1, "completion_per_m": 0.4, "cache_read_per_m": 0.025, "cache_write_per_m": 0.1, "cache_discount_pct": 75.0, "cache_ttl": "1h" }, "benchmarks": { "lmsys_arena_elo": 1320, "coding_elo": 1320, "hard_prompts_elo": 1315, "swe_bench_verified": 37.0, "swe_bench_lite": 32.0, "livecodebench_pass1": 51.0, "humaneval_pass1": 89.0, "mbpp_pass1": 87.0, "aider_diff_pass": 71.0, "math_500": 76.0, "gsm8k": 95.0, "gpqa_diamond": 53.0, "ifeval": 85.0, "bfcl_tool_calling": 88.0, "aa_intelligence_index": 60.0, "aa_coding_index": 74.0, "aa_agentic_index": 65.0, "median_tps": 175.0, "median_ttft_ms": 205.0 }, "empirical_dyno": { "awarded_score": 64, "cautious_fill": 62, "high_fill": 72, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "z-ai/glm-5.3-flash", "wingman_orthogonality": 9.0, "wingman_rescue_rate": 92.0 }, "providers": [ "Google AI Studio", "Google Vertex AI" ] }, { "id": "google/gemini-1.5-flash", "name": "Gemini 1.5 Flash", "maker": "google", "family": "gemini-1-5", "status": "active", "weights": "closed", "context_window": 1048576, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.075, "completion_per_m": 0.3, "cache_read_per_m": 0.01875, "cache_write_per_m": 0.075, "cache_discount_pct": 75.0, "cache_ttl": "1h" }, "benchmarks": { "lmsys_arena_elo": 1285, "coding_elo": 1280, "hard_prompts_elo": 1270, "swe_bench_verified": 28.5, "swe_bench_lite": 24.0, "livecodebench_pass1": 42.0, "humaneval_pass1": 84.5, "mbpp_pass1": 82.0, "aider_diff_pass": 63.0, "math_500": 68.0, "gsm8k": 91.0, "gpqa_diamond": 44.0, "ifeval": 81.0, "bfcl_tool_calling": 84.0, "aa_intelligence_index": 48.0, "aa_coding_index": 62.0, "aa_agentic_index": 54.0, "median_tps": 180.0, "median_ttft_ms": 220.0 }, "empirical_dyno": { "awarded_score": 58, "cautious_fill": 56, "high_fill": 67, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "qwen/qwen-2.5-coder-32b", "wingman_orthogonality": 7.9, "wingman_rescue_rate": 83.0 }, "providers": [ "Google AI Studio", "Google Vertex AI" ] }, { "id": "google/gemma-2-27b-it", "name": "Gemma 2 27B IT", "maker": "google", "family": "gemma-2", "status": "active", "weights": "open-weights", "context_window": 8192, "max_output_tokens": 4096, "pricing": { "prompt_per_m": 0.25, "completion_per_m": 0.5, "cache_read_per_m": 0.05, "cache_write_per_m": 0.25, "cache_discount_pct": 80.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1250, "coding_elo": 1240, "hard_prompts_elo": 1235, "swe_bench_verified": 18.0, "swe_bench_lite": 14.0, "livecodebench_pass1": 34.0, "humaneval_pass1": 79.0, "mbpp_pass1": 77.0, "aider_diff_pass": 52.0, "math_500": 58.0, "gsm8k": 87.0, "gpqa_diamond": 38.0, "ifeval": 78.0, "bfcl_tool_calling": 79.0, "aa_intelligence_index": 42.0, "aa_coding_index": 52.0, "aa_agentic_index": 44.0, "median_tps": 85.0, "median_ttft_ms": 310.0 }, "empirical_dyno": { "awarded_score": 54, "cautious_fill": 52, "high_fill": 63, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 7.5, "wingman_rescue_rate": 80.0 }, "providers": [ "Google AI Studio", "DeepInfra", "Together", "Fireworks", "Ollama" ] }, { "id": "meta-llama/llama-3.1-8b-instruct", "name": "LLaMA 3.1 8B Instruct", "maker": "meta", "family": "llama-3-1", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.05, "completion_per_m": 0.1, "cache_read_per_m": 0.01, "cache_write_per_m": 0.05, "cache_discount_pct": 80.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1210, "coding_elo": 1205, "hard_prompts_elo": 1195, "swe_bench_verified": 12.0, "swe_bench_lite": 9.5, "livecodebench_pass1": 28.0, "humaneval_pass1": 73.0, "mbpp_pass1": 72.0, "aider_diff_pass": 46.0, "math_500": 51.0, "gsm8k": 84.0, "gpqa_diamond": 32.0, "ifeval": 76.0, "bfcl_tool_calling": 76.0, "aa_intelligence_index": 36.0, "aa_coding_index": 44.0, "aa_agentic_index": 38.0, "median_tps": 220.0, "median_ttft_ms": 170.0 }, "empirical_dyno": { "awarded_score": 48, "cautious_fill": 45, "high_fill": 58, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "qwen/qwen-2.5-coder-32b", "wingman_orthogonality": 7.2, "wingman_rescue_rate": 78.0 }, "providers": [ "Groq", "DeepInfra", "Together", "Fireworks", "Cerebras", "Ollama" ] }, { "id": "qwen/qwen-2.5-coder-7b-instruct", "name": "Qwen 2.5 Coder 7B Instruct", "maker": "alibaba", "family": "qwen-2-5-coder", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.08, "completion_per_m": 0.16, "cache_read_per_m": 0.02, "cache_write_per_m": 0.08, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1260, "coding_elo": 1270, "hard_prompts_elo": 1245, "swe_bench_verified": 22.0, "swe_bench_lite": 18.5, "livecodebench_pass1": 39.0, "humaneval_pass1": 84.0, "mbpp_pass1": 82.0, "aider_diff_pass": 62.0, "math_500": 64.0, "gsm8k": 88.0, "gpqa_diamond": 39.0, "ifeval": 78.0, "bfcl_tool_calling": 81.0, "aa_intelligence_index": 46.0, "aa_coding_index": 64.0, "aa_agentic_index": 48.0, "median_tps": 180.0, "median_ttft_ms": 210.0 }, "empirical_dyno": { "awarded_score": 52, "cautious_fill": 50, "high_fill": 62, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.0, "wingman_rescue_rate": 84.0 }, "providers": [ "Alibaba Cloud", "DeepInfra", "Together", "Fireworks", "Ollama" ] }, { "id": "cohere/command-r-plus", "name": "Command R+ (08-2024)", "maker": "cohere", "family": "command-r", "status": "active", "weights": "open-weights", "context_window": 128000, "max_output_tokens": 4096, "pricing": { "prompt_per_m": 2.5, "completion_per_m": 10.0, "cache_read_per_m": 0.5, "cache_write_per_m": 2.5, "cache_discount_pct": 80.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1260, "coding_elo": 1250, "hard_prompts_elo": 1245, "swe_bench_verified": 22.0, "swe_bench_lite": 18.0, "livecodebench_pass1": 36.0, "humaneval_pass1": 78.0, "mbpp_pass1": 79.0, "aider_diff_pass": 58.0, "math_500": 55.0, "gsm8k": 88.0, "gpqa_diamond": 38.0, "ifeval": 82.0, "bfcl_tool_calling": 86.0, "aa_intelligence_index": 52.0, "aa_coding_index": 58.0, "aa_agentic_index": 62.0, "median_tps": 65.0, "median_ttft_ms": 380.0 }, "empirical_dyno": { "awarded_score": 56, "cautious_fill": 53, "high_fill": 65, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 7.6, "wingman_rescue_rate": 81.0 }, "providers": [ "Cohere Native", "AWS Bedrock", "Microsoft Azure", "Together" ] }, { "id": "cohere/command-r", "name": "Command R (08-2024)", "maker": "cohere", "family": "command-r", "status": "active", "weights": "open-weights", "context_window": 128000, "max_output_tokens": 4096, "pricing": { "prompt_per_m": 0.15, "completion_per_m": 0.6, "cache_read_per_m": 0.03, "cache_write_per_m": 0.15, "cache_discount_pct": 80.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1210, "coding_elo": 1200, "hard_prompts_elo": 1195, "swe_bench_verified": 14.0, "swe_bench_lite": 11.0, "livecodebench_pass1": 27.0, "humaneval_pass1": 71.0, "mbpp_pass1": 72.0, "aider_diff_pass": 48.0, "math_500": 46.0, "gsm8k": 82.0, "gpqa_diamond": 31.0, "ifeval": 78.0, "bfcl_tool_calling": 82.0, "aa_intelligence_index": 40.0, "aa_coding_index": 46.0, "aa_agentic_index": 52.0, "median_tps": 110.0, "median_ttft_ms": 280.0 }, "empirical_dyno": { "awarded_score": 50, "cautious_fill": 48, "high_fill": 60, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 7.3, "wingman_rescue_rate": 79.0 }, "providers": [ "Cohere Native", "AWS Bedrock", "Microsoft Azure", "Together" ] }, { "id": "deepseek/deepseek-coder-v2.5", "name": "DeepSeek Coder V2.5 (236B)", "maker": "deepseek", "family": "deepseek-coder", "status": "active", "weights": "open-weights", "context_window": 128000, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.14, "completion_per_m": 0.28, "cache_read_per_m": 0.014, "cache_write_per_m": 0.14, "cache_discount_pct": 90.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1295, "coding_elo": 1310, "hard_prompts_elo": 1285, "swe_bench_verified": 30.5, "swe_bench_lite": 26.0, "livecodebench_pass1": 48.0, "humaneval_pass1": 90.2, "mbpp_pass1": 88.0, "aider_diff_pass": 72.0, "math_500": 72.0, "gsm8k": 92.5, "gpqa_diamond": 45.0, "ifeval": 81.0, "bfcl_tool_calling": 85.0, "aa_intelligence_index": 55.0, "aa_coding_index": 76.0, "aa_agentic_index": 56.0, "median_tps": 120.0, "median_ttft_ms": 250.0 }, "empirical_dyno": { "awarded_score": 60, "cautious_fill": 58, "high_fill": 69, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.5, "wingman_rescue_rate": 88.0 }, "providers": [ "DeepSeek Native", "DeepInfra", "Together", "Fireworks", "Novita" ] }, { "id": "qwen/qwen-2.5-7b-instruct", "name": "Qwen 2.5 7B Instruct", "maker": "alibaba", "family": "qwen-2-5", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.08, "completion_per_m": 0.16, "cache_read_per_m": 0.02, "cache_write_per_m": 0.08, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1240, "coding_elo": 1235, "hard_prompts_elo": 1220, "swe_bench_verified": 16.0, "swe_bench_lite": 13.0, "livecodebench_pass1": 34.0, "humaneval_pass1": 80.0, "mbpp_pass1": 78.0, "aider_diff_pass": 55.0, "math_500": 60.0, "gsm8k": 86.0, "gpqa_diamond": 37.0, "ifeval": 77.0, "bfcl_tool_calling": 80.0, "aa_intelligence_index": 44.0, "aa_coding_index": 56.0, "aa_agentic_index": 46.0, "median_tps": 190.0, "median_ttft_ms": 200.0 }, "empirical_dyno": { "awarded_score": 40, "cautious_fill": 44, "high_fill": 60, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 7.5, "wingman_rescue_rate": 81.0 }, "providers": [ "Alibaba Cloud", "DeepInfra", "Together", "Fireworks", "Ollama" ] }, { "id": "qwen/qwen-2.5-14b-instruct", "name": "Qwen 2.5 14B Instruct", "maker": "alibaba", "family": "qwen-2-5", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.12, "completion_per_m": 0.24, "cache_read_per_m": 0.03, "cache_write_per_m": 0.12, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1270, "coding_elo": 1265, "hard_prompts_elo": 1255, "swe_bench_verified": 22.0, "swe_bench_lite": 18.0, "livecodebench_pass1": 41.0, "humaneval_pass1": 85.0, "mbpp_pass1": 83.0, "aider_diff_pass": 62.0, "math_500": 70.0, "gsm8k": 90.0, "gpqa_diamond": 43.0, "ifeval": 80.0, "bfcl_tool_calling": 82.5, "aa_intelligence_index": 50.0, "aa_coding_index": 65.0, "aa_agentic_index": 52.0, "median_tps": 140.0, "median_ttft_ms": 240.0 }, "empirical_dyno": { "awarded_score": 54, "cautious_fill": 52, "high_fill": 64, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 7.8, "wingman_rescue_rate": 83.0 }, "providers": [ "Alibaba Cloud", "DeepInfra", "Together", "Fireworks", "Ollama" ] }, { "id": "qwen/qwen-2.5-32b-instruct", "name": "Qwen 2.5 32B Instruct", "maker": "alibaba", "family": "qwen-2-5", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.2, "completion_per_m": 0.4, "cache_read_per_m": 0.05, "cache_write_per_m": 0.2, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1295, "coding_elo": 1290, "hard_prompts_elo": 1285, "swe_bench_verified": 28.0, "swe_bench_lite": 24.0, "livecodebench_pass1": 46.0, "humaneval_pass1": 88.0, "mbpp_pass1": 86.0, "aider_diff_pass": 68.0, "math_500": 78.0, "gsm8k": 93.0, "gpqa_diamond": 48.0, "ifeval": 82.0, "bfcl_tool_calling": 84.5, "aa_intelligence_index": 58.0, "aa_coding_index": 72.0, "aa_agentic_index": 57.0, "median_tps": 105.0, "median_ttft_ms": 270.0 }, "empirical_dyno": { "awarded_score": 58, "cautious_fill": 55, "high_fill": 67, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 8.1, "wingman_rescue_rate": 85.0 }, "providers": [ "Alibaba Cloud", "DeepInfra", "Together", "Fireworks", "Ollama" ] }, { "id": "qwen/qwen-2.5-math-72b", "name": "Qwen 2.5 Math 72B", "maker": "alibaba", "family": "qwen-2-5-math", "status": "active", "weights": "open-weights", "context_window": 32768, "max_output_tokens": 4096, "pricing": { "prompt_per_m": 0.4, "completion_per_m": 0.8, "cache_read_per_m": 0.1, "cache_write_per_m": 0.4, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1270, "coding_elo": 1250, "hard_prompts_elo": 1290, "swe_bench_verified": 22.0, "swe_bench_lite": 18.0, "livecodebench_pass1": 42.0, "humaneval_pass1": 83.0, "mbpp_pass1": 82.0, "aider_diff_pass": 60.0, "math_500": 85.4, "gsm8k": 96.5, "gpqa_diamond": 52.0, "ifeval": 79.0, "bfcl_tool_calling": 80.0, "aa_intelligence_index": 62.0, "aa_coding_index": 65.0, "aa_agentic_index": 50.0, "median_tps": 75.0, "median_ttft_ms": 340.0 }, "empirical_dyno": { "awarded_score": 56, "cautious_fill": 53, "high_fill": 65, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 7.9, "wingman_rescue_rate": 82.0 }, "providers": [ "Alibaba Cloud", "DeepInfra", "Together", "Fireworks", "Ollama" ] }, { "id": "meta-llama/llama-3.2-3b-instruct", "name": "LLaMA 3.2 3B Instruct", "maker": "meta", "family": "llama-3-2", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.03, "completion_per_m": 0.06, "cache_read_per_m": 0.0075, "cache_write_per_m": 0.03, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1170, "coding_elo": 1160, "hard_prompts_elo": 1150, "swe_bench_verified": 8.0, "swe_bench_lite": 6.5, "livecodebench_pass1": 22.0, "humaneval_pass1": 65.0, "mbpp_pass1": 64.0, "aider_diff_pass": 38.0, "math_500": 42.0, "gsm8k": 77.0, "gpqa_diamond": 28.0, "ifeval": 72.0, "bfcl_tool_calling": 72.0, "aa_intelligence_index": 30.0, "aa_coding_index": 38.0, "aa_agentic_index": 32.0, "median_tps": 260.0, "median_ttft_ms": 150.0 }, "empirical_dyno": { "awarded_score": 42, "cautious_fill": 40, "high_fill": 54, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "qwen/qwen-2.5-coder-32b", "wingman_orthogonality": 7.0, "wingman_rescue_rate": 76.0 }, "providers": [ "Groq", "DeepInfra", "Together", "Fireworks", "Cerebras", "Ollama" ] }, { "id": "meta-llama/llama-3.2-11b-vision-instruct", "name": "LLaMA 3.2 11B Vision Instruct", "maker": "meta", "family": "llama-3-2", "status": "active", "weights": "open-weights", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.08, "completion_per_m": 0.16, "cache_read_per_m": 0.02, "cache_write_per_m": 0.08, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1230, "coding_elo": 1220, "hard_prompts_elo": 1215, "swe_bench_verified": 14.0, "swe_bench_lite": 11.5, "livecodebench_pass1": 31.0, "humaneval_pass1": 74.0, "mbpp_pass1": 73.0, "aider_diff_pass": 48.0, "math_500": 52.0, "gsm8k": 85.0, "gpqa_diamond": 35.0, "ifeval": 77.0, "bfcl_tool_calling": 78.0, "aa_intelligence_index": 42.0, "aa_coding_index": 50.0, "aa_agentic_index": 44.0, "median_tps": 160.0, "median_ttft_ms": 210.0 }, "empirical_dyno": { "awarded_score": 49, "cautious_fill": 47, "high_fill": 59, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "anthropic/claude-3-5-sonnet", "wingman_orthogonality": 7.4, "wingman_rescue_rate": 80.0 }, "providers": [ "Groq", "DeepInfra", "Together", "Fireworks", "Ollama" ] }, { "id": "x-ai/grok-2", "name": "Grok 2", "maker": "xai", "family": "grok-2", "status": "active", "weights": "closed", "context_window": 131072, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 2.0, "completion_per_m": 10.0, "cache_read_per_m": 0.5, "cache_write_per_m": 2.0, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1305, "coding_elo": 1300, "hard_prompts_elo": 1310, "swe_bench_verified": 37.0, "swe_bench_lite": 31.0, "livecodebench_pass1": 49.0, "humaneval_pass1": 88.5, "mbpp_pass1": 87.0, "aider_diff_pass": 70.0, "math_500": 76.0, "gsm8k": 94.0, "gpqa_diamond": 52.0, "ifeval": 84.0, "bfcl_tool_calling": 86.0, "aa_intelligence_index": 64.0, "aa_coding_index": 74.0, "aa_agentic_index": 62.0, "median_tps": 85.0, "median_ttft_ms": 340.0 }, "empirical_dyno": { "awarded_score": 60, "cautious_fill": 58, "high_fill": 68, "axes_scored": 4, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "deepseek/deepseek-v4.1-flash", "wingman_orthogonality": 8.0, "wingman_rescue_rate": 85.0 }, "providers": [ "xAI Native", "OpenRouter" ] }, { "id": "mistralai/ministral-8b-2410", "name": "Ministral 8B (2410)", "maker": "mistral", "family": "ministral", "status": "active", "weights": "closed", "context_window": 128000, "max_output_tokens": 8192, "pricing": { "prompt_per_m": 0.1, "completion_per_m": 0.1, "cache_read_per_m": 0.025, "cache_write_per_m": 0.1, "cache_discount_pct": 75.0, "cache_ttl": "ephemeral" }, "benchmarks": { "lmsys_arena_elo": 1245, "coding_elo": 1240, "hard_prompts_elo": 1230, "swe_bench_verified": 18.0, "swe_bench_lite": 15.0, "livecodebench_pass1": 34.0, "humaneval_pass1": 79.0, "mbpp_pass1": 78.0, "aider_diff_pass": 54.0, "math_500": 58.0, "gsm8k": 87.0, "gpqa_diamond": 38.0, "ifeval": 79.0, "bfcl_tool_calling": 81.0, "aa_intelligence_index": 44.0, "aa_coding_index": 54.0, "aa_agentic_index": 48.0, "median_tps": 150.0, "median_ttft_ms": 230.0 }, "empirical_dyno": { "awarded_score": 52, "cautious_fill": 50, "high_fill": 62, "axes_scored": 3, "scars": [ "LOOKS_SHAPED" ], "wingman_partner": "qwen/qwen-2.5-coder-32b", "wingman_orthogonality": 7.6, "wingman_rescue_rate": 82.0 }, "providers": [ "Mistral Native", "Azure", "OpenRouter" ] } ] }