[ { "model_name": "gpt-5 (high)", "model": "gpt-5 (high)", "polyglot_score": 88.0, "pass_rate": 88.0, "pass_rate_diff": 88.0, "pass_rate_whole": null, "edit_format": "diff", "cost": 29.08, "pass_rate_1": 52.0, "pass_rate_2": 88.0, "correct_edit_format": 91.6, "command": "aider --model openai/gpt-5", "dirname": "2025-08-23-15-47-21--gpt-5-high", "date": "2025-08-23", "prompt_tokens": 2675561, "completion_tokens": 2623429, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-5 (medium)", "model": "gpt-5 (medium)", "polyglot_score": 86.7, "pass_rate": 86.7, "pass_rate_diff": 86.7, "pass_rate_whole": null, "edit_format": "diff", "cost": 17.69, "pass_rate_1": 49.8, "pass_rate_2": 86.7, "correct_edit_format": 88.4, "command": "aider --model openai/gpt-5", "dirname": "2025-08-25-13-23-27--gpt-5-medium", "date": "2025-08-25", "prompt_tokens": 2827261, "completion_tokens": 1468799, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o3-pro (high)", "model": "o3-pro (high)", "polyglot_score": 84.9, "pass_rate": 84.9, "pass_rate_diff": 84.9, "pass_rate_whole": null, "edit_format": "diff", "cost": 146.32, "pass_rate_1": 43.6, "pass_rate_2": 84.9, "correct_edit_format": 97.8, "command": "aider --model o3-pro", "dirname": "2025-06-28-00-38-18--o3-pro-high", "date": "2025-06-28", "prompt_tokens": 2372636, "completion_tokens": 1235902, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gemini-2.5-pro-preview-06-05 (32k think)", "model": "gemini-2.5-pro-preview-06-05 (32k think)", "polyglot_score": 83.1, "pass_rate": 83.1, "pass_rate_diff": 83.1, "pass_rate_whole": null, "edit_format": "diff-fenced", "cost": 49.88, "pass_rate_1": 46.2, "pass_rate_2": 83.1, "correct_edit_format": 99.6, "command": "aider --model gemini/gemini-2.5-pro-preview-06-05 --thinking-tokens 32k", "dirname": "2025-06-06-16-36-21--gemini0605-32k-think-diff-fenced", "date": "2025-06-06", "prompt_tokens": 2719961, "completion_tokens": 4648227, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-5 (low)", "model": "gpt-5 (low)", "polyglot_score": 81.3, "pass_rate": 81.3, "pass_rate_diff": 81.3, "pass_rate_whole": null, "edit_format": "diff", "cost": 10.37, "pass_rate_1": 43.1, "pass_rate_2": 81.3, "correct_edit_format": 86.7, "command": "aider --model openai/gpt-5", "dirname": "2025-08-25-14-16-37--gpt-5-low", "date": "2025-08-25", "prompt_tokens": 2534059, "completion_tokens": 779568, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o3 (high)", "model": "o3 (high)", "polyglot_score": 81.3, "pass_rate": 81.3, "pass_rate_diff": 81.3, "pass_rate_whole": null, "edit_format": "diff", "cost": 21.23, "pass_rate_1": 40.0, "pass_rate_2": 81.3, "correct_edit_format": 94.7, "command": "aider --model o3 --reasoning-effort high", "dirname": "2025-06-25-21-04-24--o3-price-reduction-high", "date": "2025-06-25", "prompt_tokens": 3148932, "completion_tokens": 2047615, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "grok-4 (high)", "model": "grok-4 (high)", "polyglot_score": 79.6, "pass_rate": 79.6, "pass_rate_diff": 79.6, "pass_rate_whole": null, "edit_format": "diff", "cost": 59.62, "pass_rate_1": 40.9, "pass_rate_2": 79.6, "correct_edit_format": 97.3, "command": "aider --model openrouter/x-ai/grok-4", "dirname": "2025-07-11-19-37-40--xai-or-grok4-high", "date": "2025-07-11", "prompt_tokens": 2815347, "completion_tokens": 3411480, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gemini-2.5-pro-preview-06-05 (default think)", "model": "gemini-2.5-pro-preview-06-05 (default think)", "polyglot_score": 79.1, "pass_rate": 79.1, "pass_rate_diff": 79.1, "pass_rate_whole": null, "edit_format": "diff-fenced", "cost": 45.6, "pass_rate_1": 44.9, "pass_rate_2": 79.1, "correct_edit_format": 100.0, "command": "aider --model gemini/gemini-2.5-pro-preview-06-05", "dirname": "2025-06-06-18-38-56--gemini0605-diff-fenced", "date": "2025-06-06", "prompt_tokens": 2751296, "completion_tokens": 4142197, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o3 (high) + gpt-4.1", "model": "o3 (high) + gpt-4.1", "polyglot_score": 78.2, "pass_rate": 78.2, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "architect", "cost": 17.55, "pass_rate_1": 34.8, "pass_rate_2": 78.2, "correct_edit_format": 100.0, "command": "aider --model o3", "dirname": "2025-06-27-23-53-57--o3-mini-high-diff-arch", "date": "2025-06-27", "prompt_tokens": 1306877, "completion_tokens": 1327154, "test_cases": 224, "benchmark_source": "polyglot" }, { "model_name": "o3", "model": "o3", "polyglot_score": 76.9, "pass_rate": 76.9, "pass_rate_diff": 76.9, "pass_rate_whole": null, "edit_format": "diff", "cost": 13.75, "pass_rate_1": 40.9, "pass_rate_2": 76.9, "correct_edit_format": 93.8, "command": "aider --model o3", "dirname": "2025-06-25-20-30-16--o3-price-reduction", "date": "2025-06-25", "prompt_tokens": 2893189, "completion_tokens": 1154767, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Gemini 2.5 Pro Preview 05-06", "model": "Gemini 2.5 Pro Preview 05-06", "polyglot_score": 76.9, "pass_rate": 76.9, "pass_rate_diff": 76.9, "pass_rate_whole": null, "edit_format": "diff-fenced", "cost": 37.41, "pass_rate_1": 36.4, "pass_rate_2": 76.9, "correct_edit_format": 97.3, "command": "aider --model gemini/gemini-2.5-pro-preview-05-06", "dirname": "2025-05-07-19-32-40--gemini0506-diff-fenced-completion_cost", "date": "2025-05-07", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "DeepSeek-V3.2-Exp (Reasoner)", "model": "DeepSeek-V3.2-Exp (Reasoner)", "polyglot_score": 74.2, "pass_rate": 74.2, "pass_rate_diff": 74.2, "pass_rate_whole": null, "edit_format": "diff", "cost": 1.3, "pass_rate_1": 39.6, "pass_rate_2": 74.2, "correct_edit_format": 97.3, "command": "aider --model deepseek/deepseek-reasoner", "dirname": "2025-10-03-09-45-34--deepseek-v3.2-reasoner", "date": "2025-10-03", "prompt_tokens": 2191446, "completion_tokens": 1645129, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Gemini 2.5 Pro Preview 03-25", "model": "Gemini 2.5 Pro Preview 03-25", "polyglot_score": 72.9, "pass_rate": 72.9, "pass_rate_diff": 72.9, "pass_rate_whole": null, "edit_format": "diff-fenced", "cost": null, "pass_rate_1": 40.9, "pass_rate_2": 72.9, "correct_edit_format": 92.4, "command": "aider --model gemini/gemini-2.5-pro-preview-03-25", "dirname": "2025-04-12-04-55-50--gemini-25-pro-diff-fenced", "date": "2025-04-12", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "claude-opus-4-20250514 (32k thinking)", "model": "claude-opus-4-20250514 (32k thinking)", "polyglot_score": 72.0, "pass_rate": 72.0, "pass_rate_diff": 72.0, "pass_rate_whole": null, "edit_format": "diff", "cost": 65.75, "pass_rate_1": 37.3, "pass_rate_2": 72.0, "correct_edit_format": 97.3, "command": "aider --model claude-opus-4-20250514", "dirname": "2025-05-25-20-40-51--opus4-diff-exuser", "date": "2025-05-25", "prompt_tokens": 2567514, "completion_tokens": 363142, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o4-mini (high)", "model": "o4-mini (high)", "polyglot_score": 72.0, "pass_rate": 72.0, "pass_rate_diff": 72.0, "pass_rate_whole": null, "edit_format": "diff", "cost": 19.64, "pass_rate_1": 19.6, "pass_rate_2": 72.0, "correct_edit_format": 90.7, "command": "aider --model o4-mini", "dirname": "2025-04-16-22-01-58--o4-mini-high-diff-exsys", "date": "2025-04-16", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "DeepSeek R1 (0528)", "model": "DeepSeek R1 (0528)", "polyglot_score": 71.4, "pass_rate": 71.4, "pass_rate_diff": 71.4, "pass_rate_whole": null, "edit_format": "diff", "cost": 4.8, "pass_rate_1": 34.4, "pass_rate_2": 71.4, "correct_edit_format": 94.6, "command": "aider --model deepseek/deepseek-reasoner", "dirname": "2025-06-06-16-47-07--r1-diff", "date": "2025-06-06", "prompt_tokens": 2644169, "completion_tokens": 1842168, "test_cases": 224, "benchmark_source": "polyglot" }, { "model_name": "claude-opus-4-20250514 (no think)", "model": "claude-opus-4-20250514 (no think)", "polyglot_score": 70.7, "pass_rate": 70.7, "pass_rate_diff": 70.7, "pass_rate_whole": null, "edit_format": "diff", "cost": 68.63, "pass_rate_1": 32.9, "pass_rate_2": 70.7, "correct_edit_format": 98.7, "command": "aider --model claude-opus-4-20250514", "dirname": "2025-05-25-19-57-20--opus4-diff-exuser", "date": "2025-05-25", "prompt_tokens": 2671437, "completion_tokens": 380717, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "DeepSeek-V3.2-Exp (Chat)", "model": "DeepSeek-V3.2-Exp (Chat)", "polyglot_score": 70.2, "pass_rate": 70.2, "pass_rate_diff": 70.2, "pass_rate_whole": null, "edit_format": "diff", "cost": 0.88, "pass_rate_1": 38.7, "pass_rate_2": 70.2, "correct_edit_format": 98.2, "command": "aider --model deepseek/deepseek-chat", "dirname": "2025-10-03-09-21-36--deepseek-v3.2-chat", "date": "2025-10-03", "prompt_tokens": 2266868, "completion_tokens": 573477, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "claude-3-7-sonnet-20250219 (32k thinking tokens)", "model": "claude-3-7-sonnet-20250219 (32k thinking tokens)", "polyglot_score": 64.9, "pass_rate": 64.9, "pass_rate_diff": 64.9, "pass_rate_whole": null, "edit_format": "diff", "cost": 36.83, "pass_rate_1": 29.3, "pass_rate_2": 64.9, "correct_edit_format": 97.8, "command": "aider --model anthropic/claude-3-7-sonnet-20250219 --thinking-tokens 32k", "dirname": "2025-02-24-21-47-23--sonnet37-diff-think-32k-64k", "date": "2025-02-24", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "DeepSeek R1 + claude-3-5-sonnet-20241022", "model": "DeepSeek R1 + claude-3-5-sonnet-20241022", "polyglot_score": 64.0, "pass_rate": 64.0, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "architect", "cost": 13.29, "pass_rate_1": 27.1, "pass_rate_2": 64.0, "correct_edit_format": 100.0, "command": "aider --architect --model r1 --editor-model sonnet", "dirname": "2025-01-23-19-14-48--r1-architect-sonnet", "date": "2025-01-23", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o1-2024-12-17 (high)", "model": "o1-2024-12-17 (high)", "polyglot_score": 61.7, "pass_rate": 61.7, "pass_rate_diff": 61.7, "pass_rate_whole": null, "edit_format": "diff", "cost": 186.5, "pass_rate_1": 23.7, "pass_rate_2": 61.7, "correct_edit_format": 91.5, "command": "aider --model openrouter/openai/o1", "dirname": "2024-12-21-19-23-03--polyglot-o1-hard-diff", "date": "2024-12-21", "prompt_tokens": null, "completion_tokens": null, "test_cases": 224, "benchmark_source": "polyglot" }, { "model_name": "claude-sonnet-4-20250514 (32k thinking)", "model": "claude-sonnet-4-20250514 (32k thinking)", "polyglot_score": 61.3, "pass_rate": 61.3, "pass_rate_diff": 61.3, "pass_rate_whole": null, "edit_format": "diff", "cost": 26.58, "pass_rate_1": 25.8, "pass_rate_2": 61.3, "correct_edit_format": 97.3, "command": "aider --model claude-sonnet-4-20250514", "dirname": "2025-05-24-22-10-36--sonnet4-diff-exuser-think32k", "date": "2025-05-24", "prompt_tokens": 2863068, "completion_tokens": 1271074, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "claude-3-7-sonnet-20250219 (no thinking)", "model": "claude-3-7-sonnet-20250219 (no thinking)", "polyglot_score": 60.4, "pass_rate": 60.4, "pass_rate_diff": 60.4, "pass_rate_whole": null, "edit_format": "diff", "cost": 17.72, "pass_rate_1": 24.4, "pass_rate_2": 60.4, "correct_edit_format": 93.3, "command": "aider --model sonnet", "dirname": "2025-02-24-19-54-07--sonnet37-diff", "date": "2025-02-24", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o3-mini (high)", "model": "o3-mini (high)", "polyglot_score": 60.4, "pass_rate": 60.4, "pass_rate_diff": 60.4, "pass_rate_whole": null, "edit_format": "diff", "cost": 18.16, "pass_rate_1": 21.0, "pass_rate_2": 60.4, "correct_edit_format": 93.3, "command": "aider --model o3-mini --reasoning-effort high", "dirname": "2025-01-31-20-42-47--o3-mini-diff-high", "date": "2025-01-31", "prompt_tokens": null, "completion_tokens": null, "test_cases": 224, "benchmark_source": "polyglot" }, { "model_name": "Qwen3 235B A22B diff, no think, Alibaba API", "model": "Qwen3 235B A22B diff, no think, Alibaba API", "polyglot_score": 59.6, "pass_rate": 59.6, "pass_rate_diff": 59.6, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": 28.9, "pass_rate_2": 59.6, "correct_edit_format": 92.9, "command": "aider --model openai/qwen3-235b-a22b", "dirname": "2025-05-09-17-02-02--qwen3-235b-a22b.unthink_16k_diff", "date": "2025-05-09", "prompt_tokens": 2816192, "completion_tokens": 342062, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Kimi K2", "model": "Kimi K2", "polyglot_score": 59.1, "pass_rate": 59.1, "pass_rate_diff": 59.1, "pass_rate_whole": null, "edit_format": "diff", "cost": 1.24, "pass_rate_1": 20.4, "pass_rate_2": 59.1, "correct_edit_format": 92.9, "command": "aider --model openrouter/moonshotai/kimi-k2", "dirname": "2025-07-17-17-41-54--kimi-k2-diff-or-pricing", "date": "2025-07-17", "prompt_tokens": 2355141, "completion_tokens": 363846, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "DeepSeek R1", "model": "DeepSeek R1", "polyglot_score": 56.9, "pass_rate": 56.9, "pass_rate_diff": 56.9, "pass_rate_whole": null, "edit_format": "diff", "cost": 5.42, "pass_rate_1": 26.7, "pass_rate_2": 56.9, "correct_edit_format": 96.9, "command": "aider --model deepseek/deepseek-reasoner", "dirname": "2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer", "date": "2025-01-20", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "claude-sonnet-4-20250514 (no thinking)", "model": "claude-sonnet-4-20250514 (no thinking)", "polyglot_score": 56.4, "pass_rate": 56.4, "pass_rate_diff": 56.4, "pass_rate_whole": null, "edit_format": "diff", "cost": 15.82, "pass_rate_1": 20.4, "pass_rate_2": 56.4, "correct_edit_format": 98.2, "command": "aider --model claude-sonnet-4-20250514", "dirname": "2025-05-24-21-17-54--sonnet4-diff-exuser", "date": "2025-05-24", "prompt_tokens": 3460663, "completion_tokens": 433373, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gemini-2.5-flash-preview-05-20 (24k think)", "model": "gemini-2.5-flash-preview-05-20 (24k think)", "polyglot_score": 55.1, "pass_rate": 55.1, "pass_rate_diff": 55.1, "pass_rate_whole": null, "edit_format": "diff", "cost": 8.56, "pass_rate_1": 26.2, "pass_rate_2": 55.1, "correct_edit_format": 95.6, "command": "aider --model gemini/gemini-2.5-flash-preview-05-20", "dirname": "2025-05-25-22-58-44--flash25-05-20-24k-think", "date": "2025-05-25", "prompt_tokens": 3666792, "completion_tokens": 2703162, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "DeepSeek V3 (0324)", "model": "DeepSeek V3 (0324)", "polyglot_score": 55.1, "pass_rate": 55.1, "pass_rate_diff": 55.1, "pass_rate_whole": null, "edit_format": "diff", "cost": 1.12, "pass_rate_1": 28.0, "pass_rate_2": 55.1, "correct_edit_format": 99.6, "command": "aider --model deepseek/deepseek-chat", "dirname": "2025-03-24-15-41-33--deepseek-v3-0324-polyglot-diff", "date": "2025-03-24", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Quasar Alpha", "model": "Quasar Alpha", "polyglot_score": 54.7, "pass_rate": 54.7, "pass_rate_diff": 54.7, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": 21.8, "pass_rate_2": 54.7, "correct_edit_format": 98.2, "command": "aider --model openrouter/openrouter/quasar-alpha", "dirname": "2025-04-04-02-57-25--qalpha-diff-exsys", "date": "2025-04-04", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o3-mini (medium)", "model": "o3-mini (medium)", "polyglot_score": 53.8, "pass_rate": 53.8, "pass_rate_diff": 53.8, "pass_rate_whole": null, "edit_format": "diff", "cost": 8.86, "pass_rate_1": 19.1, "pass_rate_2": 53.8, "correct_edit_format": 95.1, "command": "aider --model o3-mini", "dirname": "2025-01-31-20-27-46--o3-mini-diff2", "date": "2025-01-31", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Grok 3 Beta", "model": "Grok 3 Beta", "polyglot_score": 53.3, "pass_rate": 53.3, "pass_rate_diff": 53.3, "pass_rate_whole": null, "edit_format": "diff", "cost": 11.03, "pass_rate_1": 22.2, "pass_rate_2": 53.3, "correct_edit_format": 99.6, "command": "aider --model openrouter/x-ai/grok-3-beta", "dirname": "2025-04-10-04-21-31--grok3-diff-exuser", "date": "2025-04-10", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Optimus Alpha", "model": "Optimus Alpha", "polyglot_score": 52.9, "pass_rate": 52.9, "pass_rate_diff": 52.9, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": 21.3, "pass_rate_2": 52.9, "correct_edit_format": 97.3, "command": "aider --model openrouter/openrouter/optimus-alpha", "dirname": "2025-04-10-19-02-44--oalpha-diff-exsys", "date": "2025-04-10", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-4.1", "model": "gpt-4.1", "polyglot_score": 52.4, "pass_rate": 52.4, "pass_rate_diff": 52.4, "pass_rate_whole": null, "edit_format": "diff", "cost": 9.86, "pass_rate_1": 20.0, "pass_rate_2": 52.4, "correct_edit_format": 98.2, "command": "aider --model gpt-4.1", "dirname": "2025-04-14-21-05-54--gpt41-diff-exuser", "date": "2025-04-14", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "claude-3-5-sonnet-20241022", "model": "claude-3-5-sonnet-20241022", "polyglot_score": 51.6, "pass_rate": 51.6, "pass_rate_diff": 51.6, "pass_rate_whole": null, "edit_format": "diff", "cost": 14.41, "pass_rate_1": 22.2, "pass_rate_2": 51.6, "correct_edit_format": 99.6, "command": "aider --model claude-3-5-sonnet-20241022", "dirname": "2025-01-17-19-44-33--sonnet-baseline-jan-17", "date": "2025-01-17", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Grok 3 Mini Beta (high)", "model": "Grok 3 Mini Beta (high)", "polyglot_score": 49.3, "pass_rate": 49.3, "pass_rate_diff": null, "pass_rate_whole": 49.3, "edit_format": "whole", "cost": 0.73, "pass_rate_1": 17.3, "pass_rate_2": 49.3, "correct_edit_format": 99.6, "command": "aider --model xai/grok-3-mini-beta --reasoning-effort high", "dirname": "2025-04-10-23-59-02--xai-grok3-mini-whole-high", "date": "2025-04-10", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "DeepSeek Chat V3 (prev)", "model": "DeepSeek Chat V3 (prev)", "polyglot_score": 48.4, "pass_rate": 48.4, "pass_rate_diff": 48.4, "pass_rate_whole": null, "edit_format": "diff", "cost": 0.34, "pass_rate_1": 22.7, "pass_rate_2": 48.4, "correct_edit_format": 98.7, "command": "aider --model deepseek/deepseek-chat", "dirname": "2024-12-25-13-31-51--deepseekv3preview-diff2", "date": "2024-12-25", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gemini-2.5-flash-preview-04-17 (default)", "model": "gemini-2.5-flash-preview-04-17 (default)", "polyglot_score": 47.1, "pass_rate": 47.1, "pass_rate_diff": 47.1, "pass_rate_whole": null, "edit_format": "diff", "cost": 1.85, "pass_rate_1": 21.8, "pass_rate_2": 47.1, "correct_edit_format": 85.3, "command": "aider --model gemini/gemini-2.5-flash-preview-04-17", "dirname": "2025-04-20-19-54-31--flash25-diff-no-think", "date": "2025-04-20", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "chatgpt-4o-latest (2025-03-29)", "model": "chatgpt-4o-latest (2025-03-29)", "polyglot_score": 45.3, "pass_rate": 45.3, "pass_rate_diff": 45.3, "pass_rate_whole": null, "edit_format": "diff", "cost": 19.74, "pass_rate_1": 16.4, "pass_rate_2": 45.3, "correct_edit_format": 64.4, "command": "aider --model chatgpt-4o-latest", "dirname": "2025-03-29-05-24-55--chatgpt4o-mar28-diff", "date": "2025-03-29", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-4.5-preview", "model": "gpt-4.5-preview", "polyglot_score": 44.9, "pass_rate": 44.9, "pass_rate_diff": 44.9, "pass_rate_whole": null, "edit_format": "diff", "cost": 183.18, "pass_rate_1": 22.3, "pass_rate_2": 44.9, "correct_edit_format": 97.3, "command": "aider --model openai/gpt-4.5-preview", "dirname": "2025-02-27-20-26-15--gpt45-diff3", "date": "2025-02-27", "prompt_tokens": null, "completion_tokens": null, "test_cases": 224, "benchmark_source": "polyglot" }, { "model_name": "gemini-2.5-flash-preview-05-20 (no think)", "model": "gemini-2.5-flash-preview-05-20 (no think)", "polyglot_score": 44.0, "pass_rate": 44.0, "pass_rate_diff": 44.0, "pass_rate_whole": null, "edit_format": "diff", "cost": 1.14, "pass_rate_1": 20.9, "pass_rate_2": 44.0, "correct_edit_format": 93.8, "command": "aider --model gemini/gemini-2.5-flash-preview-05-20", "dirname": "2025-05-26-15-56-31--flash25-05-20-24k-think", "date": "2025-05-26", "prompt_tokens": 5512458, "completion_tokens": 514145, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-oss-120b (high)", "model": "gpt-oss-120b (high)", "polyglot_score": 41.8, "pass_rate": 41.8, "pass_rate_diff": 41.8, "pass_rate_whole": null, "edit_format": "diff", "cost": 0.74, "pass_rate_1": 13.8, "pass_rate_2": 41.8, "correct_edit_format": 79.1, "command": "aider --model openrouter/openai/gpt-oss-120b --reasoning-effort high", "dirname": "2025-08-06-04-54-48--gpt-oss-120b-high-polyglot", "date": "2025-08-06", "prompt_tokens": 3123768, "completion_tokens": 856495, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Qwen3 32B", "model": "Qwen3 32B", "polyglot_score": 40.0, "pass_rate": 40.0, "pass_rate_diff": 40.0, "pass_rate_whole": null, "edit_format": "diff", "cost": 0.76, "pass_rate_1": 14.2, "pass_rate_2": 40.0, "correct_edit_format": 83.6, "command": "aider --model openrouter/qwen/qwen3-32b", "dirname": "2025-05-08-03-20-24--qwen3-32b-default", "date": "2025-05-08", "prompt_tokens": 317591, "completion_tokens": 120418, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gemini-exp-1206", "model": "gemini-exp-1206", "polyglot_score": 38.2, "pass_rate": 38.2, "pass_rate_diff": 69.2, "pass_rate_whole": 38.2, "edit_format": "whole", "cost": null, "pass_rate_1": 19.6, "pass_rate_2": 38.2, "correct_edit_format": 98.2, "command": "aider --model gemini/gemini-exp-1206", "dirname": "2024-12-22-18-43-25--gemini-exp-1206-polyglot-whole-2", "date": "2024-12-22", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Gemini 2.0 Pro exp-02-05", "model": "Gemini 2.0 Pro exp-02-05", "polyglot_score": 35.6, "pass_rate": 35.6, "pass_rate_diff": null, "pass_rate_whole": 35.6, "edit_format": "whole", "cost": null, "pass_rate_1": 20.4, "pass_rate_2": 35.6, "correct_edit_format": 100.0, "command": "aider --model gemini/gemini-2.0-pro-exp-02-05", "dirname": "2025-02-25-20-23-07--gemini-pro", "date": "2025-02-25", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Grok 3 Mini Beta (low)", "model": "Grok 3 Mini Beta (low)", "polyglot_score": 34.7, "pass_rate": 34.7, "pass_rate_diff": null, "pass_rate_whole": 34.7, "edit_format": "whole", "cost": 0.79, "pass_rate_1": 11.1, "pass_rate_2": 34.7, "correct_edit_format": 100.0, "command": "aider --model openrouter/x-ai/grok-3-mini-beta", "dirname": "2025-04-10-18-47-24--grok3-mini-whole-exuser", "date": "2025-04-10", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o1-mini-2024-09-12", "model": "o1-mini-2024-09-12", "polyglot_score": 32.9, "pass_rate": 32.9, "pass_rate_diff": null, "pass_rate_whole": 32.9, "edit_format": "whole", "cost": 18.58, "pass_rate_1": 5.8, "pass_rate_2": 32.9, "correct_edit_format": 96.9, "command": "aider --model o1-mini", "dirname": "2024-12-22-21-26-35--polyglot-o1mini-whole", "date": "2024-12-22", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-4.1-mini", "model": "gpt-4.1-mini", "polyglot_score": 32.4, "pass_rate": 32.4, "pass_rate_diff": 32.4, "pass_rate_whole": null, "edit_format": "diff", "cost": 1.99, "pass_rate_1": 11.1, "pass_rate_2": 32.4, "correct_edit_format": 92.4, "command": "aider --model gpt-4.1-mini", "dirname": "2025-04-14-21-27-53--gpt41mini-diff", "date": "2025-04-14", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "claude-3-5-haiku-20241022", "model": "claude-3-5-haiku-20241022", "polyglot_score": 28.0, "pass_rate": 28.0, "pass_rate_diff": 28.0, "pass_rate_whole": null, "edit_format": "diff", "cost": 6.06, "pass_rate_1": 7.1, "pass_rate_2": 28.0, "correct_edit_format": 91.1, "command": "aider --model claude-3-5-haiku-20241022", "dirname": "2024-12-21-21-46-27--polyglot-haiku-diff", "date": "2024-12-21", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "chatgpt-4o-latest (2025-02-15)", "model": "chatgpt-4o-latest (2025-02-15)", "polyglot_score": 27.1, "pass_rate": 27.1, "pass_rate_diff": 27.1, "pass_rate_whole": null, "edit_format": "diff", "cost": 14.37, "pass_rate_1": 9.0, "pass_rate_2": 27.1, "correct_edit_format": 93.3, "command": "aider --model chatgpt-4o-latest", "dirname": "2025-02-15-19-51-22--chatgpt4o-feb15-diff", "date": "2025-02-15", "prompt_tokens": null, "completion_tokens": null, "test_cases": 223, "benchmark_source": "polyglot" }, { "model_name": "QwQ-32B + Qwen 2.5 Coder Instruct", "model": "QwQ-32B + Qwen 2.5 Coder Instruct", "polyglot_score": 26.2, "pass_rate": 26.2, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "architect", "cost": null, "pass_rate_1": 9.8, "pass_rate_2": 26.2, "correct_edit_format": 100.0, "command": "aider --model fireworks_ai/accounts/fireworks/models/qwq-32b --architect", "dirname": "2025-03-07-15-11-27--qwq32b-arch-temp-topp-again", "date": "2025-03-07", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-4o-2024-08-06", "model": "gpt-4o-2024-08-06", "polyglot_score": 23.1, "pass_rate": 23.1, "pass_rate_diff": 23.1, "pass_rate_whole": null, "edit_format": "diff", "cost": 7.03, "pass_rate_1": 4.9, "pass_rate_2": 23.1, "correct_edit_format": 94.2, "command": "aider --model gpt-4o-2024-08-06", "dirname": "2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt", "date": "2024-12-30", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gemini-2.0-flash-exp", "model": "gemini-2.0-flash-exp", "polyglot_score": 22.2, "pass_rate": 22.2, "pass_rate_diff": 69.9, "pass_rate_whole": 22.2, "edit_format": "whole", "cost": null, "pass_rate_1": 11.6, "pass_rate_2": 22.2, "correct_edit_format": 100.0, "command": "aider --model gemini/gemini-2.0-flash-exp", "dirname": "2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole", "date": "2024-12-22", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "qwen-max-2025-01-25", "model": "qwen-max-2025-01-25", "polyglot_score": 21.8, "pass_rate": 21.8, "pass_rate_diff": 21.8, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": 9.3, "pass_rate_2": 21.8, "correct_edit_format": 90.2, "command": "OPENAI_API_BASE=https://dashscope-intl.aliyuncs.com/compatible-mode/v1 aider --model openai/qwen-max-2025-01-25", "dirname": "2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff", "date": "2025-01-28", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "QwQ-32B", "model": "QwQ-32B", "polyglot_score": 20.9, "pass_rate": 20.9, "pass_rate_diff": 20.9, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": 8.0, "pass_rate_2": 20.9, "correct_edit_format": 67.6, "command": "aider --model fireworks_ai/accounts/fireworks/models/qwq-32b", "dirname": "2025-03-06-17-40-24--qwq32b-diff-temp-topp-ex-sys-remind-user-for-real", "date": "2025-03-06", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gemini-2.0-flash-thinking-exp-01-21", "model": "gemini-2.0-flash-thinking-exp-01-21", "polyglot_score": 18.2, "pass_rate": 18.2, "pass_rate_diff": 18.2, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": 5.8, "pass_rate_2": 18.2, "correct_edit_format": 77.8, "command": "aider --model gemini/gemini-2.0-flash-thinking-exp-01-21", "dirname": "2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff", "date": "2025-01-21", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-4o-2024-11-20", "model": "gpt-4o-2024-11-20", "polyglot_score": 18.2, "pass_rate": 18.2, "pass_rate_diff": 18.2, "pass_rate_whole": null, "edit_format": "diff", "cost": 6.74, "pass_rate_1": 4.9, "pass_rate_2": 18.2, "correct_edit_format": 95.1, "command": "aider --model gpt-4o-2024-11-20", "dirname": "2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys", "date": "2024-12-30", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "DeepSeek Chat V2.5", "model": "DeepSeek Chat V2.5", "polyglot_score": 17.8, "pass_rate": 17.8, "pass_rate_diff": 17.8, "pass_rate_whole": null, "edit_format": "diff", "cost": 0.51, "pass_rate_1": 5.3, "pass_rate_2": 17.8, "correct_edit_format": 92.9, "command": "aider --model deepseek/deepseek-chat", "dirname": "2024-12-21-20-56-21--polyglot-deepseek-diff", "date": "2024-12-21", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Qwen2.5-Coder-32B-Instruct", "model": "Qwen2.5-Coder-32B-Instruct", "polyglot_score": 8.0, "pass_rate": 8.0, "pass_rate_diff": 8.0, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": 4.4, "pass_rate_2": 8.0, "correct_edit_format": 71.6, "command": "aider --model openai/Qwen/Qwen2.5-Coder-32B-Instruct # via hyperbolic", "dirname": "2024-12-22-13-22-32--polyglot-qwen-diff", "date": "2024-12-22", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Llama 4 Maverick", "model": "Llama 4 Maverick", "polyglot_score": 15.6, "pass_rate": 15.6, "pass_rate_diff": null, "pass_rate_whole": 15.6, "edit_format": "whole", "cost": null, "pass_rate_1": 4.4, "pass_rate_2": 15.6, "correct_edit_format": 99.1, "command": "aider --model nvidia_nim/meta/llama-4-maverick-17b-128e-instruct", "dirname": "2025-04-06-08-39-52--llama-4-maverick-17b-128e-instruct-polyglot-whole", "date": "2025-04-06", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "yi-lightning", "model": "yi-lightning", "polyglot_score": 12.9, "pass_rate": 12.9, "pass_rate_diff": null, "pass_rate_whole": 12.9, "edit_format": "whole", "cost": null, "pass_rate_1": 5.8, "pass_rate_2": 12.9, "correct_edit_format": 92.9, "command": "aider --model openai/yi-lightning", "dirname": "2024-12-23-01-11-56--yi-test", "date": "2024-12-23", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "command-a-03-2025-quality", "model": "command-a-03-2025-quality", "polyglot_score": 12.0, "pass_rate": 12.0, "pass_rate_diff": null, "pass_rate_whole": 12.0, "edit_format": "whole", "cost": null, "pass_rate_1": 2.2, "pass_rate_2": 12.0, "correct_edit_format": 99.6, "command": "OPENAI_API_BASE=https://api.cohere.ai/compatibility/v1 aider --model openai/command-a-03-2025-quality", "dirname": "2025-03-14-23-40-00--cmda-quality-whole2", "date": "2025-03-14", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "Codestral 25.01", "model": "Codestral 25.01", "polyglot_score": 11.1, "pass_rate": 11.1, "pass_rate_diff": null, "pass_rate_whole": 11.1, "edit_format": "whole", "cost": 1.98, "pass_rate_1": 4.0, "pass_rate_2": 11.1, "correct_edit_format": 100.0, "command": "aider --model mistral/codestral-latest", "dirname": "2025-01-13-18-17-25--codestral-whole2", "date": "2025-01-13", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "openhands-lm-32b-v0.1", "model": "openhands-lm-32b-v0.1", "polyglot_score": 10.2, "pass_rate": 10.2, "pass_rate_diff": null, "pass_rate_whole": 10.2, "edit_format": "whole", "cost": null, "pass_rate_1": 4.0, "pass_rate_2": 10.2, "correct_edit_format": 95.1, "command": "aider --model openrouter/all-hands/openhands-lm-32b-v0.1", "dirname": "2025-04-19-14-43-04--o4-mini-patch", "date": "2025-04-19", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-4.1-nano", "model": "gpt-4.1-nano", "polyglot_score": 8.9, "pass_rate": 8.9, "pass_rate_diff": null, "pass_rate_whole": 8.9, "edit_format": "whole", "cost": 0.43, "pass_rate_1": 3.1, "pass_rate_2": 8.9, "correct_edit_format": 94.2, "command": "aider --model gpt-4.1-nano", "dirname": "2025-04-14-22-46-01--gpt41nano-diff", "date": "2025-04-14", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gemma-3-27b-it", "model": "gemma-3-27b-it", "polyglot_score": 4.9, "pass_rate": 4.9, "pass_rate_diff": null, "pass_rate_whole": 4.9, "edit_format": "whole", "cost": null, "pass_rate_1": 1.8, "pass_rate_2": 4.9, "correct_edit_format": 100.0, "command": "aider --model openrouter/google/gemma-3-27b-it", "dirname": "2025-03-15-01-21-24--gemma3-27b-or", "date": "2025-03-15", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "gpt-4o-mini-2024-07-18", "model": "gpt-4o-mini-2024-07-18", "polyglot_score": 3.6, "pass_rate": 3.6, "pass_rate_diff": null, "pass_rate_whole": 3.6, "edit_format": "whole", "cost": 0.32, "pass_rate_1": 0.9, "pass_rate_2": 3.6, "correct_edit_format": 100.0, "command": "aider --model gpt-4o-mini-2024-07-18", "dirname": "2024-12-21-18-41-18--polyglot-gpt-4o-mini", "date": "2024-12-21", "prompt_tokens": null, "completion_tokens": null, "test_cases": 225, "benchmark_source": "polyglot" }, { "model_name": "o1", "model": "o1", "polyglot_score": null, "pass_rate": 84.2, "pass_rate_diff": 84.2, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 99.2, "command": "aider --model openrouter/openai/o1", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "o1-preview", "model": "o1-preview", "polyglot_score": null, "pass_rate": 79.7, "pass_rate_diff": 79.7, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 93.2, "command": "aider --model o1-preview", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "claude-3.5-sonnet-20240620", "model": "claude-3.5-sonnet-20240620", "polyglot_score": null, "pass_rate": 77.4, "pass_rate_diff": 77.4, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 99.2, "command": "aider --model claude-3.5-sonnet-20240620", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/qwen2.5-coder:32b", "model": "ollama/qwen2.5-coder:32b", "polyglot_score": null, "pass_rate": 72.9, "pass_rate_diff": null, "pass_rate_whole": 72.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/qwen2.5-coder:32b", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "DeepSeek Coder V2 0724", "model": "DeepSeek Coder V2 0724", "polyglot_score": null, "pass_rate": 72.9, "pass_rate_diff": 72.9, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 97.7, "command": "aider --model deepseek/deepseek-coder", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-4o-2024-05-13", "model": "gpt-4o-2024-05-13", "polyglot_score": null, "pass_rate": 72.9, "pass_rate_diff": 72.9, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 96.2, "command": "aider", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "DeepSeek-V2.5-1210", "model": "DeepSeek-V2.5-1210", "polyglot_score": null, "pass_rate": 72.2, "pass_rate_diff": 72.2, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 99.2, "command": "aider --model deepseek/deepseek-chat", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "openai/chatgpt-4o-latest", "model": "openai/chatgpt-4o-latest", "polyglot_score": null, "pass_rate": 72.2, "pass_rate_diff": 72.2, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 97.0, "command": "aider --model openai/chatgpt-4o-latest", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "DeepSeek V2.5", "model": "DeepSeek V2.5", "polyglot_score": null, "pass_rate": 72.2, "pass_rate_diff": 72.2, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 96.2, "command": "aider --deepseek", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "o1-mini (whole)", "model": "o1-mini (whole)", "polyglot_score": null, "pass_rate": 70.7, "pass_rate_diff": null, "pass_rate_whole": 70.7, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 90.0, "command": "aider --model o1-mini", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "DeepSeek Chat V2 0628", "model": "DeepSeek Chat V2 0628", "polyglot_score": null, "pass_rate": 69.9, "pass_rate_diff": 69.9, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 97.7, "command": "aider --model deepseek/deepseek-chat", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Qwen2.5-Coder-14B-Instruct", "model": "Qwen2.5-Coder-14B-Instruct", "polyglot_score": null, "pass_rate": 69.2, "pass_rate_diff": null, "pass_rate_whole": 69.2, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openai/Qwen2.5-Coder-14B-Instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "claude-3-opus-20240229", "model": "claude-3-opus-20240229", "polyglot_score": null, "pass_rate": 68.4, "pass_rate_diff": 68.4, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --opus", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-4-0613", "model": "gpt-4-0613", "polyglot_score": null, "pass_rate": 67.7, "pass_rate_diff": 67.7, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider -4", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Dracarys2-72B-Instruct", "model": "Dracarys2-72B-Instruct", "polyglot_score": null, "pass_rate": 66.9, "pass_rate_diff": null, "pass_rate_whole": 66.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "(via glhf.chat)", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-1.5-pro-exp-0827", "model": "gemini-1.5-pro-exp-0827", "polyglot_score": null, "pass_rate": 66.9, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "diff-fenced", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 94.7, "command": "aider --model gemini/gemini-1.5-pro-exp-0827", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "llama-3.1-405b-instruct (whole)", "model": "llama-3.1-405b-instruct (whole)", "polyglot_score": null, "pass_rate": 66.2, "pass_rate_diff": null, "pass_rate_whole": 66.2, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openrouter/meta-llama/llama-3.1-405b-instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-4-0314", "model": "gpt-4-0314", "polyglot_score": null, "pass_rate": 66.2, "pass_rate_diff": 66.2, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 93.2, "command": "aider --model gpt-4-0314", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-4-0125-preview", "model": "gpt-4-0125-preview", "polyglot_score": null, "pass_rate": 66.2, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "udiff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 97.7, "command": "aider --model gpt-4-0125-preview", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "openrouter/qwen/qwen-2.5-coder-32b-instruct", "model": "openrouter/qwen/qwen-2.5-coder-32b-instruct", "polyglot_score": null, "pass_rate": 65.4, "pass_rate_diff": 65.4, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 84.2, "command": "aider --model openrouter/qwen/qwen-2.5-coder-32b-instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Mistral Large (2411)", "model": "Mistral Large (2411)", "polyglot_score": null, "pass_rate": 65.4, "pass_rate_diff": 65.4, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 96.2, "command": "aider --model mistral/mistral-large-latest", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-1.5-pro-002", "model": "gemini-1.5-pro-002", "polyglot_score": null, "pass_rate": 65.4, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "diff-fenced", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 96.2, "command": "aider --model gemini/gemini-1.5-pro-002", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "qwen-2.5-72b-instruct (bf16)", "model": "qwen-2.5-72b-instruct (bf16)", "polyglot_score": null, "pass_rate": 65.4, "pass_rate_diff": 65.4, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 96.2, "command": "aider --model openrouter/qwen/qwen-2.5-72b-instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-4-1106-preview", "model": "gpt-4-1106-preview", "polyglot_score": null, "pass_rate": 65.4, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "udiff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 92.5, "command": "aider --model gpt-4-1106-preview", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/Qwen2.5.1-Coder-7B-Instruct-GGUF:Q8_0-32k", "model": "ollama/Qwen2.5.1-Coder-7B-Instruct-GGUF:Q8_0-32k", "polyglot_score": null, "pass_rate": 63.9, "pass_rate_diff": null, "pass_rate_whole": 63.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/Qwen2.5.1-Coder-7B-Instruct-GGUF:Q8_0-32k", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "nousresearch/hermes-3-llama-3.1-405b", "model": "nousresearch/hermes-3-llama-3.1-405b", "polyglot_score": null, "pass_rate": 63.9, "pass_rate_diff": null, "pass_rate_whole": 63.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openrouter/nousresearch/hermes-3-llama-3.1-405b", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "llama-3.1-405b-instruct (diff)", "model": "llama-3.1-405b-instruct (diff)", "polyglot_score": null, "pass_rate": 63.9, "pass_rate_diff": 63.9, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 92.5, "command": "aider --model openrouter/meta-llama/llama-3.1-405b-instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-4-turbo-2024-04-09 (udiff)", "model": "gpt-4-turbo-2024-04-09 (udiff)", "polyglot_score": null, "pass_rate": 63.9, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "udiff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 97.0, "command": "aider --gpt-4-turbo", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/qwen2.5-coder:14b", "model": "ollama/qwen2.5-coder:14b", "polyglot_score": null, "pass_rate": 61.7, "pass_rate_diff": null, "pass_rate_whole": 61.7, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 98.5, "command": "aider --model ollama/qwen2.5-coder:14b", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "o1-mini", "model": "o1-mini", "polyglot_score": null, "pass_rate": 61.1, "pass_rate_diff": 61.1, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model o1-mini", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-exp-1114", "model": "gemini-exp-1114", "polyglot_score": null, "pass_rate": 60.9, "pass_rate_diff": 60.9, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 85.7, "command": "aider --model gemini/gemini-exp-1114", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Mistral Large 2 (2407)", "model": "Mistral Large 2 (2407)", "polyglot_score": null, "pass_rate": 60.2, "pass_rate_diff": null, "pass_rate_whole": 60.2, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model mistral/mistral-large-2407", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "llama-3.3-70b-instruct", "model": "llama-3.3-70b-instruct", "polyglot_score": null, "pass_rate": 59.4, "pass_rate_diff": 59.4, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 88.7, "command": "aider --model openrouter/meta-llama/llama-3.3-70b-instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/qwen2.5:32b-instruct-q8_0", "model": "ollama/qwen2.5:32b-instruct-q8_0", "polyglot_score": null, "pass_rate": 58.6, "pass_rate_diff": null, "pass_rate_whole": 58.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/qwen2.5:32b-instruct-q8_0", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Grok-2", "model": "Grok-2", "polyglot_score": null, "pass_rate": 58.6, "pass_rate_diff": null, "pass_rate_whole": 58.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 98.5, "command": "aider --model openrouter/x-ai/grok-2", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "llama-3.1-70b-instruct", "model": "llama-3.1-70b-instruct", "polyglot_score": null, "pass_rate": 58.6, "pass_rate_diff": null, "pass_rate_whole": 58.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-exp-1121", "model": "gemini-exp-1121", "polyglot_score": null, "pass_rate": 57.9, "pass_rate_diff": 57.9, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 83.5, "command": "aider --model gemini/gemini-exp-1121", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Qwen2.5-Coder-7B-Instruct", "model": "Qwen2.5-Coder-7B-Instruct", "polyglot_score": null, "pass_rate": 57.9, "pass_rate_diff": null, "pass_rate_whole": 57.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openai/Qwen2.5-Coder-7B-Instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-3.5-turbo-0301", "model": "gpt-3.5-turbo-0301", "polyglot_score": null, "pass_rate": 57.9, "pass_rate_diff": null, "pass_rate_whole": 57.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gpt-3.5-turbo-0301", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-4-turbo-2024-04-09 (diff)", "model": "gpt-4-turbo-2024-04-09 (diff)", "polyglot_score": null, "pass_rate": 57.6, "pass_rate_diff": 57.6, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gpt-4-turbo-2024-04-09", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-1.5-pro-001", "model": "gemini-1.5-pro-001", "polyglot_score": null, "pass_rate": 57.1, "pass_rate_diff": null, "pass_rate_whole": null, "edit_format": "diff-fenced", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 87.2, "command": "aider --model gemini/gemini-1.5-pro-latest", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-3.5-turbo-1106", "model": "gpt-3.5-turbo-1106", "polyglot_score": null, "pass_rate": 56.1, "pass_rate_diff": null, "pass_rate_whole": 56.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gpt-3.5-turbo-1106", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-4o-mini", "model": "gpt-4o-mini", "polyglot_score": null, "pass_rate": 55.6, "pass_rate_diff": null, "pass_rate_whole": 55.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gpt-4o-mini", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Qwen2 72B Instruct", "model": "Qwen2 72B Instruct", "polyglot_score": null, "pass_rate": 55.6, "pass_rate_diff": null, "pass_rate_whole": 55.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model together_ai/qwen/Qwen2-72B-Instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Llama-3.1-Nemotron-70B-Instruct-HF", "model": "Llama-3.1-Nemotron-70B-Instruct-HF", "polyglot_score": null, "pass_rate": 54.9, "pass_rate_diff": null, "pass_rate_whole": 54.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 99.2, "command": "(via glhf.chat)", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Grok-2-mini", "model": "Grok-2-mini", "polyglot_score": null, "pass_rate": 54.9, "pass_rate_diff": null, "pass_rate_whole": 54.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openrouter/x-ai/grok-2-mini", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "claude-3-sonnet-20240229", "model": "claude-3-sonnet-20240229", "polyglot_score": null, "pass_rate": 54.9, "pass_rate_diff": null, "pass_rate_whole": 54.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --sonnet", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Nova Pro", "model": "Nova Pro", "polyglot_score": null, "pass_rate": 54.1, "pass_rate_diff": null, "pass_rate_whole": 54.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model bedrock/us.amazon.nova-pro-v1:0", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/qwen2.5:32b", "model": "ollama/qwen2.5:32b", "polyglot_score": null, "pass_rate": 54.1, "pass_rate_diff": null, "pass_rate_whole": 54.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/qwen2.5:32b", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Yi Coder 9B Chat", "model": "Yi Coder 9B Chat", "polyglot_score": null, "pass_rate": 54.1, "pass_rate_diff": null, "pass_rate_whole": 54.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openai/hf:01-ai/Yi-Coder-9B-Chat --openai-api-base https://glhf.chat/api/openai/v1", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-1.5-flash-exp-0827", "model": "gemini-1.5-flash-exp-0827", "polyglot_score": null, "pass_rate": 52.6, "pass_rate_diff": null, "pass_rate_whole": 52.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gemini/gemini-1.5-flash-exp-0827", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "qwen2.5-coder:7b-instruct-q8_0", "model": "qwen2.5-coder:7b-instruct-q8_0", "polyglot_score": null, "pass_rate": 51.9, "pass_rate_diff": null, "pass_rate_whole": 51.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/qwen2.5-coder:7b-instruct-q8_0", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-1.5-flash-002 (0924)", "model": "gemini-1.5-flash-002 (0924)", "polyglot_score": null, "pass_rate": 51.1, "pass_rate_diff": null, "pass_rate_whole": 51.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gemini/gemini-1.5-flash-002", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "codestral-2405", "model": "codestral-2405", "polyglot_score": null, "pass_rate": 51.1, "pass_rate_diff": null, "pass_rate_whole": 51.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model mistral/codestral-2405", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-3.5-turbo-0613", "model": "gpt-3.5-turbo-0613", "polyglot_score": null, "pass_rate": 50.4, "pass_rate_diff": null, "pass_rate_whole": 50.4, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gpt-3.5-turbo-0613", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gpt-3.5-turbo-0125", "model": "gpt-3.5-turbo-0125", "polyglot_score": null, "pass_rate": 50.4, "pass_rate_diff": null, "pass_rate_whole": 50.4, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider -3", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "qwen2:72b-instruct-q8_0", "model": "qwen2:72b-instruct-q8_0", "polyglot_score": null, "pass_rate": 49.6, "pass_rate_diff": null, "pass_rate_whole": 49.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/qwen2:72b-instruct-q8_0", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "llama3-70b-8192", "model": "llama3-70b-8192", "polyglot_score": null, "pass_rate": 49.2, "pass_rate_diff": 49.2, "pass_rate_whole": null, "edit_format": "diff", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 73.5, "command": "aider --model groq/llama3-70b-8192", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Codestral-22B-v0.1-Q4_K_M", "model": "Codestral-22B-v0.1-Q4_K_M", "polyglot_score": null, "pass_rate": 48.1, "pass_rate_diff": null, "pass_rate_whole": 48.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model Codestral-22B-v0.1-Q4_K_M", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "codestral:22b-v0.1-q8_0", "model": "codestral:22b-v0.1-q8_0", "polyglot_score": null, "pass_rate": 48.1, "pass_rate_diff": null, "pass_rate_whole": 48.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/codestral:22b-v0.1-q8_0", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "claude-3-haiku-20240307", "model": "claude-3-haiku-20240307", "polyglot_score": null, "pass_rate": 47.4, "pass_rate_diff": null, "pass_rate_whole": 47.4, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model claude-3-haiku-20240307", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/codestral", "model": "ollama/codestral", "polyglot_score": null, "pass_rate": 45.9, "pass_rate_diff": null, "pass_rate_whole": 45.9, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 98.5, "command": "aider --model ollama/codestral", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "yi-coder:9b-chat-q4_0", "model": "yi-coder:9b-chat-q4_0", "polyglot_score": null, "pass_rate": 45.1, "pass_rate_diff": null, "pass_rate_whole": 45.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/yi-coder:9b-chat-q4_0", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-1.5-flash-latest", "model": "gemini-1.5-flash-latest", "polyglot_score": null, "pass_rate": 44.4, "pass_rate_diff": null, "pass_rate_whole": 44.4, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gemini/gemini-1.5-flash-latest", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "WizardLM-2 8x22B", "model": "WizardLM-2 8x22B", "polyglot_score": null, "pass_rate": 44.4, "pass_rate_diff": null, "pass_rate_whole": 44.4, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openrouter/microsoft/wizardlm-2-8x22b", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/yi-coder:9b-chat-fp16", "model": "ollama/yi-coder:9b-chat-fp16", "polyglot_score": null, "pass_rate": 43.6, "pass_rate_diff": null, "pass_rate_whole": 43.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 99.2, "command": "aider --model ollama/yi-coder:9b-chat-fp16", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Reflection-70B", "model": "Reflection-70B", "polyglot_score": null, "pass_rate": 42.1, "pass_rate_diff": null, "pass_rate_whole": 42.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "(not currently supported)", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Qwen2.5-Coder-3B-Instruct", "model": "Qwen2.5-Coder-3B-Instruct", "polyglot_score": null, "pass_rate": 39.1, "pass_rate_diff": null, "pass_rate_whole": 39.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openai/Qwen2.5-Coder-3B-Instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/mistral-small", "model": "ollama/mistral-small", "polyglot_score": null, "pass_rate": 38.3, "pass_rate_diff": null, "pass_rate_whole": 38.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 99.2, "command": "aider --model ollama/mistral-small", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-1.5-flash-8b-exp-0924", "model": "gemini-1.5-flash-8b-exp-0924", "polyglot_score": null, "pass_rate": 38.3, "pass_rate_diff": null, "pass_rate_whole": 38.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gemini/gemini-1.5-flash-8b-exp-0924", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Command R (08-24)", "model": "Command R (08-24)", "polyglot_score": null, "pass_rate": 38.3, "pass_rate_diff": null, "pass_rate_whole": 38.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model command-r-08-2024", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Command R+ (08-24)", "model": "Command R+ (08-24)", "polyglot_score": null, "pass_rate": 38.3, "pass_rate_diff": null, "pass_rate_whole": 38.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model command-r-plus-08-2024", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemini-1.5-flash-8b-exp-0827", "model": "gemini-1.5-flash-8b-exp-0827", "polyglot_score": null, "pass_rate": 38.3, "pass_rate_diff": null, "pass_rate_whole": 38.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model gemini/gemini-1.5-flash-8b-exp-0827", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "llama-3.1-8b-instruct", "model": "llama-3.1-8b-instruct", "polyglot_score": null, "pass_rate": 37.6, "pass_rate_diff": null, "pass_rate_whole": 37.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model fireworks_ai/accounts/fireworks/models/llama-v3p1-8b-instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "qwen1.5-110b-chat", "model": "qwen1.5-110b-chat", "polyglot_score": null, "pass_rate": 37.6, "pass_rate_diff": null, "pass_rate_whole": 37.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model together_ai/qwen/qwen1.5-110b-chat", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "gemma2:27b-instruct-q8_0", "model": "gemma2:27b-instruct-q8_0", "polyglot_score": null, "pass_rate": 36.1, "pass_rate_diff": null, "pass_rate_whole": 36.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/gemma2:27b-instruct-q8_0", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "codeqwen:7b-chat-v1.5-q8_0", "model": "codeqwen:7b-chat-v1.5-q8_0", "polyglot_score": null, "pass_rate": 34.6, "pass_rate_diff": null, "pass_rate_whole": 34.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/codeqwen:7b-chat-v1.5-q8_0", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/mistral-nemo:12b-instruct-2407-q4_K_M", "model": "ollama/mistral-nemo:12b-instruct-2407-q4_K_M", "polyglot_score": null, "pass_rate": 33.1, "pass_rate_diff": null, "pass_rate_whole": 33.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/mistral-nemo:12b-instruct-2407-q4_K_M", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/codegeex4", "model": "ollama/codegeex4", "polyglot_score": null, "pass_rate": 32.3, "pass_rate_diff": null, "pass_rate_whole": 32.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 97.0, "command": "aider --model ollama/codegeex4", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Qwen2.5-Coder-1.5B-Instruct", "model": "Qwen2.5-Coder-1.5B-Instruct", "polyglot_score": null, "pass_rate": 31.6, "pass_rate_diff": null, "pass_rate_whole": 31.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openai/Qwen2.5-Coder-1.5B-Instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "command-r-plus", "model": "command-r-plus", "polyglot_score": null, "pass_rate": 31.6, "pass_rate_diff": null, "pass_rate_whole": 31.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model command-r-plus", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/hermes3:8b-llama3.1-fp16", "model": "ollama/hermes3:8b-llama3.1-fp16", "polyglot_score": null, "pass_rate": 30.1, "pass_rate_diff": null, "pass_rate_whole": 30.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 98.5, "command": "aider --model ollama/hermes3:8b-llama3.1-fp16", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/wojtek/opencodeinterpreter:6.7b", "model": "ollama/wojtek/opencodeinterpreter:6.7b", "polyglot_score": null, "pass_rate": 30.1, "pass_rate_diff": null, "pass_rate_whole": 30.1, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 91.0, "command": "aider --model ollama/wojtek/opencodeinterpreter:6.7b", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/tulu3", "model": "ollama/tulu3", "polyglot_score": null, "pass_rate": 26.3, "pass_rate_diff": null, "pass_rate_whole": 26.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model ollama/tulu3", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/llama3.2:3b-instruct-fp16", "model": "ollama/llama3.2:3b-instruct-fp16", "polyglot_score": null, "pass_rate": 26.3, "pass_rate_diff": null, "pass_rate_whole": 26.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 97.0, "command": "aider --model ollama/llama3.2:3b-instruct-fp16", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/hermes3", "model": "ollama/hermes3", "polyglot_score": null, "pass_rate": 22.6, "pass_rate_diff": null, "pass_rate_whole": 22.6, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 98.5, "command": "aider --model ollama/hermes3", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "ollama/granite3-dense:8b", "model": "ollama/granite3-dense:8b", "polyglot_score": null, "pass_rate": 20.3, "pass_rate_diff": null, "pass_rate_whole": 20.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 78.9, "command": "aider --model ollama/granite3-dense:8b", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" }, { "model_name": "Qwen2.5-Coder-0.5B-Instruct", "model": "Qwen2.5-Coder-0.5B-Instruct", "polyglot_score": null, "pass_rate": 14.3, "pass_rate_diff": null, "pass_rate_whole": 14.3, "edit_format": "whole", "cost": null, "pass_rate_1": null, "pass_rate_2": null, "correct_edit_format": 100.0, "command": "aider --model openai/Qwen2.5-Coder-0.5B-Instruct", "dirname": null, "date": null, "prompt_tokens": null, "completion_tokens": null, "test_cases": null, "benchmark_source": "code_editing" } ]