[ { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 1, "model": "Claude-Opus-4-5-20251101 (FC)", "model_name": "Claude-Opus-4-5-20251101 (FC)", "overall_accuracy": 77.47, "cost": 86.55, "latency_mean_s": 4.38, "latency_std_s": 3.13, "latency_p95_s": 7.56, "non_live_ast_accuracy": 88.58, "non_live_simple_ast": 76.83, "non_live_multiple_ast": 95.5, "non_live_parallel_ast": 93.5, "non_live_parallel_multiple_ast": 88.5, "live_accuracy": 79.79, "live_simple_ast": 86.43, "live_multiple_ast": 78.16, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 68.38, "multi_turn_base": 81.0, "multi_turn_miss_func": 64.0, "multi_turn_miss_param": 58.0, "multi_turn_long_context": 70.5, "web_search_accuracy": 84.5, "web_search_base": 84.0, "web_search_no_snippet": 85.0, "memory_accuracy": 73.76, "memory_kv": 70.97, "memory_vector": 72.9, "memory_recursive_summarization": 77.42, "relevance_detection": 62.5, "irrelevance_detection": 84.72, "organization": "Anthropic", "license": "Proprietary", "model_link": "https://www.anthropic.com/news/claude-4" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 2, "model": "Claude-Sonnet-4-5-20250929 (FC)", "model_name": "Claude-Sonnet-4-5-20250929 (FC)", "overall_accuracy": 73.24, "cost": 43.73, "latency_mean_s": 4.31, "latency_std_s": 4.43, "latency_p95_s": 7.27, "non_live_ast_accuracy": 88.65, "non_live_simple_ast": 72.58, "non_live_multiple_ast": 95.5, "non_live_parallel_ast": 94.5, "non_live_parallel_multiple_ast": 92.0, "live_accuracy": 81.13, "live_simple_ast": 89.53, "live_multiple_ast": 78.92, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 83.33, "multi_turn_accuracy": 61.37, "multi_turn_base": 69.0, "multi_turn_miss_func": 65.0, "multi_turn_miss_param": 52.5, "multi_turn_long_context": 59.0, "web_search_accuracy": 81.0, "web_search_base": 82.0, "web_search_no_snippet": 80.0, "memory_accuracy": 64.95, "memory_kv": 54.19, "memory_vector": 57.42, "memory_recursive_summarization": 83.23, "relevance_detection": 68.75, "irrelevance_detection": 86.61, "organization": "Anthropic", "license": "Proprietary", "model_link": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 3, "model": "Gemini-3-Pro-Preview (Prompt)", "model_name": "Gemini-3-Pro-Preview (Prompt)", "overall_accuracy": 72.51, "cost": 298.47, "latency_mean_s": 12.08, "latency_std_s": 21.3, "latency_p95_s": 32.73, "non_live_ast_accuracy": 90.65, "non_live_simple_ast": 79.58, "non_live_multiple_ast": 96.0, "non_live_parallel_ast": 95.0, "non_live_parallel_multiple_ast": 92.0, "live_accuracy": 83.12, "live_simple_ast": 87.6, "live_multiple_ast": 81.77, "live_parallel_ast": 93.75, "live_parallel_multiple_ast": 87.5, "multi_turn_accuracy": 60.75, "multi_turn_base": 64.5, "multi_turn_miss_func": 60.0, "multi_turn_miss_param": 54.5, "multi_turn_long_context": 64.0, "web_search_accuracy": 80.0, "web_search_base": 78.0, "web_search_no_snippet": 82.0, "memory_accuracy": 61.72, "memory_kv": 59.35, "memory_vector": 62.58, "memory_recursive_summarization": 63.23, "relevance_detection": 68.75, "irrelevance_detection": 85.59, "organization": "Google", "license": "Proprietary", "model_link": "https://deepmind.google/technologies/gemini/pro/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 4, "model": "GLM-4.6 (FC thinking)", "model_name": "GLM-4.6 (FC thinking)", "overall_accuracy": 72.38, "cost": 4.64, "latency_mean_s": 4.34, "latency_std_s": 7.22, "latency_p95_s": 13.5, "non_live_ast_accuracy": 87.56, "non_live_simple_ast": 74.25, "non_live_multiple_ast": 95.0, "non_live_parallel_ast": 91.5, "non_live_parallel_multiple_ast": 89.5, "live_accuracy": 80.9, "live_simple_ast": 89.53, "live_multiple_ast": 78.92, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 68.0, "multi_turn_base": 74.5, "multi_turn_miss_func": 68.0, "multi_turn_miss_param": 63.0, "multi_turn_long_context": 66.5, "web_search_accuracy": 77.5, "web_search_base": 79.0, "web_search_no_snippet": 76.0, "memory_accuracy": 55.7, "memory_kv": 43.87, "memory_vector": 56.13, "memory_recursive_summarization": 67.1, "relevance_detection": 75.0, "irrelevance_detection": 84.96, "organization": "Zhipu AI", "license": "MIT", "model_link": "https://huggingface.co/zai-org/GLM-4.6" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 5, "model": "Grok-4-1-fast-reasoning (FC)", "model_name": "Grok-4-1-fast-reasoning (FC)", "overall_accuracy": 69.57, "cost": 17.26, "latency_mean_s": 6.74, "latency_std_s": 12.78, "latency_p95_s": 17.57, "non_live_ast_accuracy": 88.27, "non_live_simple_ast": 77.58, "non_live_multiple_ast": 93.0, "non_live_parallel_ast": 92.5, "non_live_parallel_multiple_ast": 90.0, "live_accuracy": 78.46, "live_simple_ast": 84.11, "live_multiple_ast": 77.3, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 58.87, "multi_turn_base": 70.5, "multi_turn_miss_func": 59.5, "multi_turn_miss_param": 43.0, "multi_turn_long_context": 62.5, "web_search_accuracy": 82.5, "web_search_base": 82.0, "web_search_no_snippet": 83.0, "memory_accuracy": 53.98, "memory_kv": 41.29, "memory_vector": 57.42, "memory_recursive_summarization": 63.23, "relevance_detection": 81.25, "irrelevance_detection": 79.43, "organization": "xAI", "license": "Proprietary", "model_link": "https://docs.x.ai/docs/models" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 6, "model": "Claude-Haiku-4-5-20251001 (FC)", "model_name": "Claude-Haiku-4-5-20251001 (FC)", "overall_accuracy": 68.7, "cost": 14.23, "latency_mean_s": 1.68, "latency_std_s": 3.92, "latency_p95_s": 3.15, "non_live_ast_accuracy": 86.5, "non_live_simple_ast": 71.0, "non_live_multiple_ast": 94.0, "non_live_parallel_ast": 92.5, "non_live_parallel_multiple_ast": 88.5, "live_accuracy": 78.68, "live_simple_ast": 83.72, "live_multiple_ast": 77.59, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 53.62, "multi_turn_base": 63.5, "multi_turn_miss_func": 42.5, "multi_turn_miss_param": 52.5, "multi_turn_long_context": 56.0, "web_search_accuracy": 83.5, "web_search_base": 86.0, "web_search_no_snippet": 81.0, "memory_accuracy": 54.41, "memory_kv": 51.61, "memory_vector": 55.48, "memory_recursive_summarization": 56.13, "relevance_detection": 62.5, "irrelevance_detection": 85.11, "organization": "Anthropic", "license": "Proprietary", "model_link": "https://www.anthropic.com/news/claude-haiku-4-5" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 7, "model": "Gemini-3-Pro-Preview (FC)", "model_name": "Gemini-3-Pro-Preview (FC)", "overall_accuracy": 68.14, "cost": 224.69, "latency_mean_s": 15.87, "latency_std_s": 41.41, "latency_p95_s": 58.48, "non_live_ast_accuracy": 85.75, "non_live_simple_ast": 75.5, "non_live_multiple_ast": 94.0, "non_live_parallel_ast": 91.0, "non_live_parallel_multiple_ast": 82.5, "live_accuracy": 81.72, "live_simple_ast": 87.6, "live_multiple_ast": 80.44, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 79.17, "multi_turn_accuracy": 63.12, "multi_turn_base": 69.0, "multi_turn_miss_func": 63.0, "multi_turn_miss_param": 56.5, "multi_turn_long_context": 64.0, "web_search_accuracy": 68.5, "web_search_base": 63.0, "web_search_no_snippet": 74.0, "memory_accuracy": 54.84, "memory_kv": 50.32, "memory_vector": 63.23, "memory_recursive_summarization": 50.97, "relevance_detection": 75.0, "irrelevance_detection": 77.85, "organization": "Google", "license": "Proprietary", "model_link": "https://deepmind.google/technologies/gemini/pro/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 8, "model": "o3-2025-04-16 (Prompt)", "model_name": "o3-2025-04-16 (Prompt)", "overall_accuracy": 63.05, "cost": 234.64, "latency_mean_s": 4.83, "latency_std_s": 7.01, "latency_p95_s": 11.7, "non_live_ast_accuracy": 81.94, "non_live_simple_ast": 74.25, "non_live_multiple_ast": 89.0, "non_live_parallel_ast": 86.5, "non_live_parallel_multiple_ast": 78.0, "live_accuracy": 73.21, "live_simple_ast": 83.33, "live_multiple_ast": 70.75, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 62.25, "multi_turn_base": 68.0, "multi_turn_miss_func": 63.5, "multi_turn_miss_param": 54.5, "multi_turn_long_context": 63.0, "web_search_accuracy": 50.5, "web_search_base": 51.0, "web_search_no_snippet": 50.0, "memory_accuracy": 51.83, "memory_kv": 33.55, "memory_vector": 50.32, "memory_recursive_summarization": 71.61, "relevance_detection": 93.75, "irrelevance_detection": 83.98, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/introducing-o3-and-o4-mini/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 9, "model": "Grok-4-0709 (Prompt)", "model_name": "Grok-4-0709 (Prompt)", "overall_accuracy": 62.97, "cost": 348.19, "latency_mean_s": 30.38, "latency_std_s": 36.19, "latency_p95_s": 101.54, "non_live_ast_accuracy": 82.75, "non_live_simple_ast": 67.0, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 89.0, "non_live_parallel_multiple_ast": 81.5, "live_accuracy": 72.54, "live_simple_ast": 81.78, "live_multiple_ast": 70.18, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 47.0, "multi_turn_base": 55.5, "multi_turn_miss_func": 46.0, "multi_turn_miss_param": 36.0, "multi_turn_long_context": 50.5, "web_search_accuracy": 74.0, "web_search_base": 74.0, "web_search_no_snippet": 74.0, "memory_accuracy": 50.54, "memory_kv": 43.87, "memory_vector": 59.35, "memory_recursive_summarization": 48.39, "relevance_detection": 81.25, "irrelevance_detection": 84.3, "organization": "xAI", "license": "Proprietary", "model_link": "https://docs.x.ai/docs/models" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 10, "model": "Grok-4-0709 (FC)", "model_name": "Grok-4-0709 (FC)", "overall_accuracy": 61.38, "cost": 355.17, "latency_mean_s": 15.49, "latency_std_s": 26.22, "latency_p95_s": 44.28, "non_live_ast_accuracy": 85.38, "non_live_simple_ast": 73.5, "non_live_multiple_ast": 92.5, "non_live_parallel_ast": 88.5, "non_live_parallel_multiple_ast": 87.0, "live_accuracy": 75.57, "live_simple_ast": 82.17, "live_multiple_ast": 73.88, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 79.17, "multi_turn_accuracy": 33.88, "multi_turn_base": 44.0, "multi_turn_miss_func": 19.0, "multi_turn_miss_param": 28.5, "multi_turn_long_context": 44.0, "web_search_accuracy": 82.0, "web_search_base": 80.0, "web_search_no_snippet": 84.0, "memory_accuracy": 55.91, "memory_kv": 57.42, "memory_vector": 58.71, "memory_recursive_summarization": 51.61, "relevance_detection": 87.5, "irrelevance_detection": 75.4, "organization": "xAI", "license": "Proprietary", "model_link": "https://docs.x.ai/docs/models" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 11, "model": "Moonshotai-Kimi-K2-Instruct (FC)", "model_name": "Moonshotai-Kimi-K2-Instruct (FC)", "overall_accuracy": 59.06, "cost": 6.19, "latency_mean_s": 6.4, "latency_std_s": 9.38, "latency_p95_s": 13.78, "non_live_ast_accuracy": 81.6, "non_live_simple_ast": 69.42, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 82.0, "non_live_parallel_multiple_ast": 83.0, "live_accuracy": 78.68, "live_simple_ast": 81.78, "live_multiple_ast": 78.06, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 66.67, "multi_turn_accuracy": 50.63, "multi_turn_base": 62.0, "multi_turn_miss_func": 41.0, "multi_turn_miss_param": 44.5, "multi_turn_long_context": 55.0, "web_search_accuracy": 66.5, "web_search_base": 72.0, "web_search_no_snippet": 61.0, "memory_accuracy": 29.03, "memory_kv": 21.94, "memory_vector": 20.0, "memory_recursive_summarization": 45.16, "relevance_detection": 75.0, "irrelevance_detection": 87.34, "organization": "MoonshotAI", "license": "modified-mit", "model_link": "https://huggingface.co/moonshotai/Kimi-K2-Instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 12, "model": "Grok-4-1-fast-non-reasoning (FC)", "model_name": "Grok-4-1-fast-non-reasoning (FC)", "overall_accuracy": 58.29, "cost": 16.27, "latency_mean_s": 2.29, "latency_std_s": 7.31, "latency_p95_s": 5.34, "non_live_ast_accuracy": 88.13, "non_live_simple_ast": 76.0, "non_live_multiple_ast": 93.0, "non_live_parallel_ast": 93.0, "non_live_parallel_multiple_ast": 90.5, "live_accuracy": 77.94, "live_simple_ast": 82.95, "live_multiple_ast": 76.92, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 46.75, "multi_turn_base": 58.0, "multi_turn_miss_func": 39.5, "multi_turn_miss_param": 37.5, "multi_turn_long_context": 52.0, "web_search_accuracy": 75.0, "web_search_base": 74.0, "web_search_no_snippet": 76.0, "memory_accuracy": 26.24, "memory_kv": 20.65, "memory_vector": 20.0, "memory_recursive_summarization": 38.06, "relevance_detection": 81.25, "irrelevance_detection": 74.09, "organization": "xAI", "license": "Proprietary", "model_link": "https://docs.x.ai/docs/models" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 13, "model": "Command A Reasoning (FC)", "model_name": "Command A Reasoning (FC)", "overall_accuracy": 57.06, "cost": 3.04, "latency_mean_s": 3.44, "latency_std_s": 4.91, "latency_p95_s": 8.39, "non_live_ast_accuracy": 86.27, "non_live_simple_ast": 73.58, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 89.5, "non_live_parallel_multiple_ast": 88.5, "live_accuracy": 78.61, "live_simple_ast": 80.23, "live_multiple_ast": 78.35, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 50.12, "multi_turn_base": 61.5, "multi_turn_miss_func": 41.0, "multi_turn_miss_param": 49.5, "multi_turn_long_context": 48.5, "web_search_accuracy": 55.5, "web_search_base": 65.0, "web_search_no_snippet": 46.0, "memory_accuracy": 28.82, "memory_kv": 16.13, "memory_vector": 23.87, "memory_recursive_summarization": 46.45, "relevance_detection": 68.75, "irrelevance_detection": 86.75, "organization": "Cohere", "license": "CC-BY-NC 4.0 License (w/ Acceptable Use Addendum)", "model_link": "https://cohere.com/blog/command-a-reasoning" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 14, "model": "DeepSeek-V3.2-Exp (Prompt + Thinking)", "model_name": "DeepSeek-V3.2-Exp (Prompt + Thinking)", "overall_accuracy": 56.73, "cost": 57.75, "latency_mean_s": 37.89, "latency_std_s": 49.56, "latency_p95_s": 102.09, "non_live_ast_accuracy": 85.52, "non_live_simple_ast": 74.08, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 89.5, "non_live_parallel_multiple_ast": 86.5, "live_accuracy": 76.02, "live_simple_ast": 82.56, "live_multiple_ast": 74.74, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 54.17, "multi_turn_accuracy": 44.88, "multi_turn_base": 55.0, "multi_turn_miss_func": 49.0, "multi_turn_miss_param": 27.0, "multi_turn_long_context": 48.5, "web_search_accuracy": 58.0, "web_search_base": 64.0, "web_search_no_snippet": 52.0, "memory_accuracy": 44.09, "memory_kv": 46.45, "memory_vector": 46.45, "memory_recursive_summarization": 39.35, "relevance_detection": 93.75, "irrelevance_detection": 67.0, "organization": "DeepSeek", "license": "MIT", "model_link": "https://api-docs.deepseek.com/news/news250528" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 15, "model": "Gemini-2.5-Flash (FC)", "model_name": "Gemini-2.5-Flash (FC)", "overall_accuracy": 56.24, "cost": 26.36, "latency_mean_s": 2.99, "latency_std_s": 9.22, "latency_p95_s": 5.62, "non_live_ast_accuracy": 84.96, "non_live_simple_ast": 74.33, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 94.0, "non_live_parallel_multiple_ast": 79.5, "live_accuracy": 74.39, "live_simple_ast": 85.27, "live_multiple_ast": 71.7, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 36.25, "multi_turn_base": 41.5, "multi_turn_miss_func": 36.0, "multi_turn_miss_param": 32.0, "multi_turn_long_context": 35.5, "web_search_accuracy": 59.0, "web_search_base": 59.0, "web_search_no_snippet": 59.0, "memory_accuracy": 41.29, "memory_kv": 19.35, "memory_vector": 50.32, "memory_recursive_summarization": 54.19, "relevance_detection": 75.0, "irrelevance_detection": 93.67, "organization": "Google", "license": "Proprietary", "model_link": "https://deepmind.google/technologies/gemini/flash/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 16, "model": "GPT-5.2-2025-12-11 (FC)", "model_name": "GPT-5.2-2025-12-11 (FC)", "overall_accuracy": 55.87, "cost": 85.65, "latency_mean_s": 2.23, "latency_std_s": 9.75, "latency_p95_s": 5.26, "non_live_ast_accuracy": 81.85, "non_live_simple_ast": 72.92, "non_live_multiple_ast": 88.0, "non_live_parallel_ast": 89.0, "non_live_parallel_multiple_ast": 77.5, "live_accuracy": 70.39, "live_simple_ast": 71.71, "live_multiple_ast": 70.37, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 28.12, "multi_turn_base": 36.5, "multi_turn_miss_func": 18.0, "multi_turn_miss_param": 27.5, "multi_turn_long_context": 30.5, "web_search_accuracy": 75.5, "web_search_base": 78.0, "web_search_no_snippet": 73.0, "memory_accuracy": 45.81, "memory_kv": 33.55, "memory_vector": 43.23, "memory_recursive_summarization": 60.65, "relevance_detection": 75.0, "irrelevance_detection": 79.42, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/zh-Hans-CN/index/introducing-gpt-5-2/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 17, "model": "GPT-5-mini-2025-08-07 (FC)", "model_name": "GPT-5-mini-2025-08-07 (FC)", "overall_accuracy": 55.46, "cost": 22.18, "latency_mean_s": 8.32, "latency_std_s": 17.35, "latency_p95_s": 19.8, "non_live_ast_accuracy": 69.85, "non_live_simple_ast": 59.92, "non_live_multiple_ast": 69.0, "non_live_parallel_ast": 80.0, "non_live_parallel_multiple_ast": 70.5, "live_accuracy": 58.62, "live_simple_ast": 62.02, "live_multiple_ast": 58.02, "live_parallel_ast": 62.5, "live_parallel_multiple_ast": 45.83, "multi_turn_accuracy": 27.5, "multi_turn_base": 36.5, "multi_turn_miss_func": 17.0, "multi_turn_miss_param": 23.5, "multi_turn_long_context": 33.0, "web_search_accuracy": 82.0, "web_search_base": 87.0, "web_search_no_snippet": 77.0, "memory_accuracy": 44.3, "memory_kv": 36.77, "memory_vector": 43.87, "memory_recursive_summarization": 52.26, "relevance_detection": 62.5, "irrelevance_detection": 91.01, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/introducing-gpt-5/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 18, "model": "xLAM-2-32b-fc-r (FC)", "model_name": "xLAM-2-32b-fc-r (FC)", "overall_accuracy": 54.66, "cost": 6.0, "latency_mean_s": 6.94, "latency_std_s": 8.21, "latency_p95_s": 17.66, "non_live_ast_accuracy": 89.6, "non_live_simple_ast": 80.42, "non_live_multiple_ast": 94.0, "non_live_parallel_ast": 93.0, "non_live_parallel_multiple_ast": 91.0, "live_accuracy": 75.5, "live_simple_ast": 82.17, "live_multiple_ast": 74.64, "live_parallel_ast": 50.0, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 69.5, "multi_turn_base": 81.5, "multi_turn_miss_func": 72.5, "multi_turn_miss_param": 67.5, "multi_turn_long_context": 56.5, "web_search_accuracy": 25.5, "web_search_base": 37.0, "web_search_no_snippet": 14.0, "memory_accuracy": 20.86, "memory_kv": 6.45, "memory_vector": 10.32, "memory_recursive_summarization": 45.81, "relevance_detection": 81.25, "irrelevance_detection": 80.23, "organization": "Salesforce", "license": "cc-by-nc-4.0", "model_link": "https://huggingface.co/Salesforce/xLAM-2-32b-fc-r" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 19, "model": "DeepSeek-V3.2-Exp (FC)", "model_name": "DeepSeek-V3.2-Exp (FC)", "overall_accuracy": 54.12, "cost": 6.71, "latency_mean_s": 5.83, "latency_std_s": 11.71, "latency_p95_s": 10.59, "non_live_ast_accuracy": 34.85, "non_live_simple_ast": 37.92, "non_live_multiple_ast": 74.0, "non_live_parallel_ast": 15.0, "non_live_parallel_multiple_ast": 12.5, "live_accuracy": 53.66, "live_simple_ast": 66.28, "live_multiple_ast": 51.66, "live_parallel_ast": 25.0, "live_parallel_multiple_ast": 25.0, "multi_turn_accuracy": 37.38, "multi_turn_base": 41.5, "multi_turn_miss_func": 39.5, "multi_turn_miss_param": 33.5, "multi_turn_long_context": 35.0, "web_search_accuracy": 69.5, "web_search_base": 80.0, "web_search_no_snippet": 59.0, "memory_accuracy": 54.19, "memory_kv": 41.94, "memory_vector": 61.29, "memory_recursive_summarization": 59.35, "relevance_detection": 37.5, "irrelevance_detection": 93.18, "organization": "DeepSeek", "license": "MIT", "model_link": "https://api-docs.deepseek.com/news/news250528" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 20, "model": "GPT-4.1-2025-04-14 (FC)", "model_name": "GPT-4.1-2025-04-14 (FC)", "overall_accuracy": 53.96, "cost": 100.75, "latency_mean_s": 1.63, "latency_std_s": 3.05, "latency_p95_s": 4.01, "non_live_ast_accuracy": 82.79, "non_live_simple_ast": 72.67, "non_live_multiple_ast": 89.0, "non_live_parallel_ast": 88.0, "non_live_parallel_multiple_ast": 81.5, "live_accuracy": 69.95, "live_simple_ast": 69.38, "live_multiple_ast": 70.28, "live_parallel_ast": 56.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 38.88, "multi_turn_base": 47.5, "multi_turn_miss_func": 32.5, "multi_turn_miss_param": 32.5, "multi_turn_long_context": 43.0, "web_search_accuracy": 68.0, "web_search_base": 67.0, "web_search_no_snippet": 69.0, "memory_accuracy": 23.87, "memory_kv": 16.13, "memory_vector": 18.06, "memory_recursive_summarization": 37.42, "relevance_detection": 87.5, "irrelevance_detection": 86.52, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/gpt-4-1/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 21, "model": "o4-mini-2025-04-16 (FC)", "model_name": "o4-mini-2025-04-16 (FC)", "overall_accuracy": 53.24, "cost": 81.91, "latency_mean_s": 3.71, "latency_std_s": 7.18, "latency_p95_s": 9.33, "non_live_ast_accuracy": 37.73, "non_live_simple_ast": 66.92, "non_live_multiple_ast": 84.0, "non_live_parallel_ast": 0.0, "non_live_parallel_multiple_ast": 0.0, "live_accuracy": 66.1, "live_simple_ast": 69.38, "live_multiple_ast": 67.81, "live_parallel_ast": 0.0, "live_parallel_multiple_ast": 0.0, "multi_turn_accuracy": 41.75, "multi_turn_base": 51.0, "multi_turn_miss_func": 30.0, "multi_turn_miss_param": 40.5, "multi_turn_long_context": 45.5, "web_search_accuracy": 75.5, "web_search_base": 75.0, "web_search_no_snippet": 76.0, "memory_accuracy": 34.19, "memory_kv": 19.35, "memory_vector": 24.52, "memory_recursive_summarization": 58.71, "relevance_detection": 81.25, "irrelevance_detection": 83.91, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/introducing-o3-and-o4-mini/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 22, "model": "xLAM-2-70b-fc-r (FC)", "model_name": "xLAM-2-70b-fc-r (FC)", "overall_accuracy": 53.07, "cost": 25.1, "latency_mean_s": 28.06, "latency_std_s": 68.77, "latency_p95_s": 91.21, "non_live_ast_accuracy": 88.44, "non_live_simple_ast": 78.25, "non_live_multiple_ast": 94.0, "non_live_parallel_ast": 92.0, "non_live_parallel_multiple_ast": 89.5, "live_accuracy": 72.17, "live_simple_ast": 77.91, "live_multiple_ast": 71.13, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 77.38, "multi_turn_base": 82.5, "multi_turn_miss_func": 77.0, "multi_turn_miss_param": 74.0, "multi_turn_long_context": 76.0, "web_search_accuracy": 15.0, "web_search_base": 17.0, "web_search_no_snippet": 13.0, "memory_accuracy": 14.41, "memory_kv": 2.58, "memory_vector": 10.97, "memory_recursive_summarization": 29.68, "relevance_detection": 75.0, "irrelevance_detection": 79.11, "organization": "Salesforce", "license": "cc-by-nc-4.0", "model_link": "https://huggingface.co/Salesforce/Llama-xLAM-2-70b-fc-r" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 23, "model": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", "model_name": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", "overall_accuracy": 52.15, "cost": 3.12, "latency_mean_s": 2.56, "latency_std_s": 2.75, "latency_p95_s": 7.61, "non_live_ast_accuracy": 90.33, "non_live_simple_ast": 79.83, "non_live_multiple_ast": 95.0, "non_live_parallel_ast": 95.5, "non_live_parallel_multiple_ast": 91.0, "live_accuracy": 78.68, "live_simple_ast": 82.95, "live_multiple_ast": 77.78, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 44.62, "multi_turn_base": 54.0, "multi_turn_miss_func": 42.5, "multi_turn_miss_param": 31.5, "multi_turn_long_context": 50.5, "web_search_accuracy": 50.5, "web_search_base": 56.0, "web_search_no_snippet": 45.0, "memory_accuracy": 19.35, "memory_kv": 12.9, "memory_vector": 11.61, "memory_recursive_summarization": 33.55, "relevance_detection": 93.75, "irrelevance_detection": 78.89, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 24, "model": "GPT-5-nano-2025-08-07 (FC)", "model_name": "GPT-5-nano-2025-08-07 (FC)", "overall_accuracy": 51.45, "cost": 8.79, "latency_mean_s": 10.36, "latency_std_s": 10.37, "latency_p95_s": 23.56, "non_live_ast_accuracy": 68.0, "non_live_simple_ast": 57.0, "non_live_multiple_ast": 64.5, "non_live_parallel_ast": 79.0, "non_live_parallel_multiple_ast": 71.5, "live_accuracy": 59.44, "live_simple_ast": 58.91, "live_multiple_ast": 59.83, "live_parallel_ast": 50.0, "live_parallel_multiple_ast": 54.17, "multi_turn_accuracy": 34.5, "multi_turn_base": 44.0, "multi_turn_miss_func": 23.5, "multi_turn_miss_param": 32.5, "multi_turn_long_context": 38.0, "web_search_accuracy": 72.5, "web_search_base": 74.0, "web_search_no_snippet": 71.0, "memory_accuracy": 24.73, "memory_kv": 18.06, "memory_vector": 27.1, "memory_recursive_summarization": 29.03, "relevance_detection": 75.0, "irrelevance_detection": 89.1, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/introducing-gpt-5/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 25, "model": "Nanbeige4-3B-Thinking-2511 (FC)", "model_name": "Nanbeige4-3B-Thinking-2511 (FC)", "overall_accuracy": 51.4, "cost": 14.14, "latency_mean_s": 13.46, "latency_std_s": 26.41, "latency_p95_s": 37.45, "non_live_ast_accuracy": 81.58, "non_live_simple_ast": 63.83, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 84.5, "non_live_parallel_multiple_ast": 84.5, "live_accuracy": 79.42, "live_simple_ast": 86.05, "live_multiple_ast": 78.06, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 51.12, "multi_turn_base": 58.5, "multi_turn_miss_func": 54.0, "multi_turn_miss_param": 45.0, "multi_turn_long_context": 47.0, "web_search_accuracy": 21.5, "web_search_base": 31.0, "web_search_no_snippet": 12.0, "memory_accuracy": 36.77, "memory_kv": 31.61, "memory_vector": 34.19, "memory_recursive_summarization": 44.52, "relevance_detection": 75.0, "irrelevance_detection": 83.09, "organization": "Nanbeige", "license": "apache-2.0", "model_link": "https://huggingface.co/Nanbeige/Nanbeige4-3B-Thinking-2511" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 26, "model": "Gemini-2.5-Flash (Prompt)", "model_name": "Gemini-2.5-Flash (Prompt)", "overall_accuracy": 50.9, "cost": 33.45, "latency_mean_s": 3.18, "latency_std_s": 4.44, "latency_p95_s": 6.09, "non_live_ast_accuracy": 88.08, "non_live_simple_ast": 77.33, "non_live_multiple_ast": 91.5, "non_live_parallel_ast": 96.0, "non_live_parallel_multiple_ast": 87.5, "live_accuracy": 78.16, "live_simple_ast": 87.21, "live_multiple_ast": 75.97, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 16.75, "multi_turn_base": 14.5, "multi_turn_miss_func": 16.5, "multi_turn_miss_param": 17.5, "multi_turn_long_context": 18.5, "web_search_accuracy": 62.0, "web_search_base": 60.0, "web_search_no_snippet": 64.0, "memory_accuracy": 38.71, "memory_kv": 13.55, "memory_vector": 47.1, "memory_recursive_summarization": 55.48, "relevance_detection": 62.5, "irrelevance_detection": 91.09, "organization": "Google", "license": "Proprietary", "model_link": "https://deepmind.google/technologies/gemini/flash/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 27, "model": "GPT-4.1-mini-2025-04-14 (FC)", "model_name": "GPT-4.1-mini-2025-04-14 (FC)", "overall_accuracy": 50.45, "cost": 19.25, "latency_mean_s": 1.32, "latency_std_s": 3.65, "latency_p95_s": 2.4, "non_live_ast_accuracy": 83.83, "non_live_simple_ast": 73.33, "non_live_multiple_ast": 89.0, "non_live_parallel_ast": 91.0, "non_live_parallel_multiple_ast": 82.0, "live_accuracy": 68.84, "live_simple_ast": 67.05, "live_multiple_ast": 69.8, "live_parallel_ast": 43.75, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 34.13, "multi_turn_base": 43.5, "multi_turn_miss_func": 22.5, "multi_turn_miss_param": 30.5, "multi_turn_long_context": 40.0, "web_search_accuracy": 57.0, "web_search_base": 62.0, "web_search_no_snippet": 52.0, "memory_accuracy": 26.88, "memory_kv": 22.58, "memory_vector": 16.13, "memory_recursive_summarization": 41.94, "relevance_detection": 81.25, "irrelevance_detection": 81.69, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/gpt-4-1/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 28, "model": "o4-mini-2025-04-16 (Prompt)", "model_name": "o4-mini-2025-04-16 (Prompt)", "overall_accuracy": 50.26, "cost": 133.63, "latency_mean_s": 4.47, "latency_std_s": 5.19, "latency_p95_s": 10.19, "non_live_ast_accuracy": 81.29, "non_live_simple_ast": 72.67, "non_live_multiple_ast": 88.0, "non_live_parallel_ast": 84.5, "non_live_parallel_multiple_ast": 80.0, "live_accuracy": 70.76, "live_simple_ast": 79.46, "live_multiple_ast": 68.76, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 16.62, "multi_turn_base": 16.5, "multi_turn_miss_func": 18.0, "multi_turn_miss_param": 17.5, "multi_turn_long_context": 14.5, "web_search_accuracy": 71.5, "web_search_base": 73.0, "web_search_no_snippet": 70.0, "memory_accuracy": 35.27, "memory_kv": 22.58, "memory_vector": 25.16, "memory_recursive_summarization": 58.06, "relevance_detection": 81.25, "irrelevance_detection": 87.16, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/introducing-o3-and-o4-mini/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 29, "model": "Qwen3-32B (FC)", "model_name": "Qwen3-32B (FC)", "overall_accuracy": 48.71, "cost": 153.08, "latency_mean_s": 169.87, "latency_std_s": 164.27, "latency_p95_s": 473.49, "non_live_ast_accuracy": 88.77, "non_live_simple_ast": 75.58, "non_live_multiple_ast": 94.5, "non_live_parallel_ast": 93.5, "non_live_parallel_multiple_ast": 91.5, "live_accuracy": 82.01, "live_simple_ast": 89.53, "live_multiple_ast": 80.91, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 50.0, "multi_turn_accuracy": 47.87, "multi_turn_base": 56.0, "multi_turn_miss_func": 52.5, "multi_turn_miss_param": 40.0, "multi_turn_long_context": 43.0, "web_search_accuracy": 21.5, "web_search_base": 25.0, "web_search_no_snippet": 18.0, "memory_accuracy": 26.67, "memory_kv": 12.26, "memory_vector": 25.81, "memory_recursive_summarization": 41.94, "relevance_detection": 93.75, "irrelevance_detection": 76.37, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-32B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 30, "model": "o3-2025-04-16 (FC)", "model_name": "o3-2025-04-16 (FC)", "overall_accuracy": 48.56, "cost": 133.45, "latency_mean_s": 3.5, "latency_std_s": 8.69, "latency_p95_s": 8.39, "non_live_ast_accuracy": 40.38, "non_live_simple_ast": 74.5, "non_live_multiple_ast": 87.0, "non_live_parallel_ast": 0.0, "non_live_parallel_multiple_ast": 0.0, "live_accuracy": 66.17, "live_simple_ast": 70.54, "live_multiple_ast": 67.62, "live_parallel_ast": 0.0, "live_parallel_multiple_ast": 0.0, "multi_turn_accuracy": 14.75, "multi_turn_base": 16.5, "multi_turn_miss_func": 11.5, "multi_turn_miss_param": 14.5, "multi_turn_long_context": 16.5, "web_search_accuracy": 77.0, "web_search_base": 79.0, "web_search_no_snippet": 75.0, "memory_accuracy": 47.31, "memory_kv": 24.52, "memory_vector": 44.52, "memory_recursive_summarization": 72.9, "relevance_detection": 81.25, "irrelevance_detection": 86.13, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/introducing-o3-and-o4-mini/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 31, "model": "Qwen3-235B-A22B-Instruct-2507 (FC)", "model_name": "Qwen3-235B-A22B-Instruct-2507 (FC)", "overall_accuracy": 47.99, "cost": 2.5, "latency_mean_s": 2.57, "latency_std_s": 2.44, "latency_p95_s": 6.27, "non_live_ast_accuracy": 37.4, "non_live_simple_ast": 40.58, "non_live_multiple_ast": 36.5, "non_live_parallel_ast": 53.0, "non_live_parallel_multiple_ast": 19.5, "live_accuracy": 68.91, "live_simple_ast": 58.53, "live_multiple_ast": 71.6, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 45.38, "multi_turn_base": 57.5, "multi_turn_miss_func": 35.0, "multi_turn_miss_param": 33.5, "multi_turn_long_context": 55.5, "web_search_accuracy": 54.0, "web_search_base": 57.0, "web_search_no_snippet": 51.0, "memory_accuracy": 23.87, "memory_kv": 7.1, "memory_vector": 18.71, "memory_recursive_summarization": 45.81, "relevance_detection": 87.5, "irrelevance_detection": 81.73, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 32, "model": "Nanbeige3.5-Pro-Thinking (FC)", "model_name": "Nanbeige3.5-Pro-Thinking (FC)", "overall_accuracy": 47.68, "cost": 23.46, "latency_mean_s": 21.12, "latency_std_s": 28.61, "latency_p95_s": 63.29, "non_live_ast_accuracy": 38.35, "non_live_simple_ast": 43.92, "non_live_multiple_ast": 36.5, "non_live_parallel_ast": 53.0, "non_live_parallel_multiple_ast": 20.0, "live_accuracy": 69.95, "live_simple_ast": 63.18, "live_multiple_ast": 71.42, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 66.67, "multi_turn_accuracy": 40.0, "multi_turn_base": 56.0, "multi_turn_miss_func": 34.0, "multi_turn_miss_param": 29.0, "multi_turn_long_context": 41.0, "web_search_accuracy": 42.0, "web_search_base": 47.0, "web_search_no_snippet": 37.0, "memory_accuracy": 45.16, "memory_kv": 38.06, "memory_vector": 58.06, "memory_recursive_summarization": 39.35, "relevance_detection": 100.0, "irrelevance_detection": 74.2, "organization": "Nanbeige", "license": "apache-2.0", "model_link": "https://huggingface.co/Nanbeige" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 33, "model": "Qwen3-32B (Prompt)", "model_name": "Qwen3-32B (Prompt)", "overall_accuracy": 46.78, "cost": 199.47, "latency_mean_s": 167.54, "latency_std_s": 160.5, "latency_p95_s": 457.87, "non_live_ast_accuracy": 90.27, "non_live_simple_ast": 79.08, "non_live_multiple_ast": 97.0, "non_live_parallel_ast": 93.5, "non_live_parallel_multiple_ast": 91.5, "live_accuracy": 82.01, "live_simple_ast": 87.21, "live_multiple_ast": 81.2, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 43.25, "multi_turn_base": 54.0, "multi_turn_miss_func": 46.0, "multi_turn_miss_param": 36.5, "multi_turn_long_context": 36.5, "web_search_accuracy": 26.0, "web_search_base": 34.0, "web_search_no_snippet": 18.0, "memory_accuracy": 15.7, "memory_kv": 13.55, "memory_vector": 14.19, "memory_recursive_summarization": 19.35, "relevance_detection": 81.25, "irrelevance_detection": 82.39, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-32B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 34, "model": "xLAM-2-8b-fc-r (FC)", "model_name": "xLAM-2-8b-fc-r (FC)", "overall_accuracy": 46.68, "cost": 20.92, "latency_mean_s": 22.65, "latency_std_s": 46.92, "latency_p95_s": 108.81, "non_live_ast_accuracy": 84.58, "non_live_simple_ast": 73.83, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 87.5, "non_live_parallel_multiple_ast": 83.5, "live_accuracy": 67.95, "live_simple_ast": 75.58, "live_multiple_ast": 66.57, "live_parallel_ast": 56.25, "live_parallel_multiple_ast": 54.17, "multi_turn_accuracy": 70.0, "multi_turn_base": 76.0, "multi_turn_miss_func": 72.0, "multi_turn_miss_param": 65.0, "multi_turn_long_context": 67.0, "web_search_accuracy": 6.5, "web_search_base": 11.0, "web_search_no_snippet": 2.0, "memory_accuracy": 13.98, "memory_kv": 5.81, "memory_vector": 15.48, "memory_recursive_summarization": 20.65, "relevance_detection": 87.5, "irrelevance_detection": 63.28, "organization": "Salesforce", "license": "cc-by-nc-4.0", "model_link": "https://huggingface.co/Salesforce/Llama-xLAM-2-8b-fc-r" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 35, "model": "Command A (FC)", "model_name": "Command A (FC)", "overall_accuracy": 46.49, "cost": 91.37, "latency_mean_s": 2.09, "latency_std_s": 7.36, "latency_p95_s": 4.94, "non_live_ast_accuracy": 87.56, "non_live_simple_ast": 75.75, "non_live_multiple_ast": 93.0, "non_live_parallel_ast": 93.5, "non_live_parallel_multiple_ast": 88.0, "live_accuracy": 78.53, "live_simple_ast": 85.66, "live_multiple_ast": 76.92, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 29.5, "multi_turn_base": 38.0, "multi_turn_miss_func": 23.0, "multi_turn_miss_param": 32.0, "multi_turn_long_context": 25.0, "web_search_accuracy": 46.5, "web_search_base": 60.0, "web_search_no_snippet": 33.0, "memory_accuracy": 16.56, "memory_kv": 4.52, "memory_vector": 5.16, "memory_recursive_summarization": 40.0, "relevance_detection": 81.25, "irrelevance_detection": 84.19, "organization": "Cohere", "license": "CC-BY-NC 4.0 License (w/ Acceptable Use Addendum)", "model_link": "https://cohere.com/blog/command-a" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 36, "model": "BitAgent-Bounty-8B", "model_name": "BitAgent-Bounty-8B", "overall_accuracy": 46.23, "cost": 18.02, "latency_mean_s": 16.52, "latency_std_s": 30.73, "latency_p95_s": 77.12, "non_live_ast_accuracy": 81.6, "non_live_simple_ast": 72.42, "non_live_multiple_ast": 93.0, "non_live_parallel_ast": 83.0, "non_live_parallel_multiple_ast": 78.0, "live_accuracy": 93.12, "live_simple_ast": 90.31, "live_multiple_ast": 94.02, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 95.83, "multi_turn_accuracy": 62.38, "multi_turn_base": 75.0, "multi_turn_miss_func": 49.5, "multi_turn_miss_param": 68.0, "multi_turn_long_context": 57.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 1.51, "memory_kv": 1.29, "memory_vector": 1.29, "memory_recursive_summarization": 1.94, "relevance_detection": 68.75, "irrelevance_detection": 97.48, "organization": "Bittensor", "license": "Apache-2.0", "model_link": "https://huggingface.co/BitAgent/BitAgent-Bounty-8B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 37, "model": "Arch-Agent-32B", "model_name": "Arch-Agent-32B", "overall_accuracy": 45.37, "cost": 8.87, "latency_mean_s": 9.44, "latency_std_s": 21.44, "latency_p95_s": 24.87, "non_live_ast_accuracy": 88.92, "non_live_simple_ast": 76.67, "non_live_multiple_ast": 94.0, "non_live_parallel_ast": 96.0, "non_live_parallel_multiple_ast": 89.0, "live_accuracy": 80.68, "live_simple_ast": 86.43, "live_multiple_ast": 79.11, "live_parallel_ast": 93.75, "live_parallel_multiple_ast": 79.17, "multi_turn_accuracy": 54.25, "multi_turn_base": 64.5, "multi_turn_miss_func": 58.0, "multi_turn_miss_param": 53.0, "multi_turn_long_context": 41.5, "web_search_accuracy": 5.0, "web_search_base": 4.0, "web_search_no_snippet": 6.0, "memory_accuracy": 14.62, "memory_kv": 5.81, "memory_vector": 9.03, "memory_recursive_summarization": 29.03, "relevance_detection": 81.25, "irrelevance_detection": 82.15, "organization": "katanemo", "license": "katanemo-research", "model_link": "https://huggingface.co/katanemo/Arch-Agent-32B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 38, "model": "GPT-5.2-2025-12-11 (Prompt)", "model_name": "GPT-5.2-2025-12-11 (Prompt)", "overall_accuracy": 45.27, "cost": 164.58, "latency_mean_s": 4.21, "latency_std_s": 20.93, "latency_p95_s": 10.58, "non_live_ast_accuracy": 78.29, "non_live_simple_ast": 71.17, "non_live_multiple_ast": 83.5, "non_live_parallel_ast": 84.0, "non_live_parallel_multiple_ast": 74.5, "live_accuracy": 67.14, "live_simple_ast": 77.91, "live_multiple_ast": 64.58, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 43.75, "multi_turn_base": 54.5, "multi_turn_miss_func": 40.5, "multi_turn_miss_param": 33.5, "multi_turn_long_context": 46.5, "web_search_accuracy": 40.5, "web_search_base": 45.0, "web_search_no_snippet": 36.0, "memory_accuracy": 3.87, "memory_kv": 2.58, "memory_vector": 1.94, "memory_recursive_summarization": 7.1, "relevance_detection": 75.0, "irrelevance_detection": 87.26, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/zh-Hans-CN/index/introducing-gpt-5-2/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 39, "model": "Qwen3-8B (FC)", "model_name": "Qwen3-8B (FC)", "overall_accuracy": 42.57, "cost": 43.32, "latency_mean_s": 51.36, "latency_std_s": 76.14, "latency_p95_s": 188.98, "non_live_ast_accuracy": 87.58, "non_live_simple_ast": 72.83, "non_live_multiple_ast": 96.5, "non_live_parallel_ast": 92.0, "non_live_parallel_multiple_ast": 89.0, "live_accuracy": 80.53, "live_simple_ast": 84.5, "live_multiple_ast": 79.68, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 79.17, "multi_turn_accuracy": 41.75, "multi_turn_base": 50.5, "multi_turn_miss_func": 42.0, "multi_turn_miss_param": 40.0, "multi_turn_long_context": 34.5, "web_search_accuracy": 12.0, "web_search_base": 15.0, "web_search_no_snippet": 9.0, "memory_accuracy": 14.62, "memory_kv": 5.16, "memory_vector": 7.1, "memory_recursive_summarization": 31.61, "relevance_detection": 93.75, "irrelevance_detection": 79.07, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-8B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 40, "model": "ToolACE-2-8B (FC)", "model_name": "ToolACE-2-8B (FC)", "overall_accuracy": 42.44, "cost": 24.43, "latency_mean_s": 15.95, "latency_std_s": 40.06, "latency_p95_s": 65.26, "non_live_ast_accuracy": 87.1, "non_live_simple_ast": 73.42, "non_live_multiple_ast": 91.0, "non_live_parallel_ast": 93.0, "non_live_parallel_multiple_ast": 91.0, "live_accuracy": 77.42, "live_simple_ast": 71.32, "live_multiple_ast": 79.39, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 38.38, "multi_turn_base": 49.0, "multi_turn_miss_func": 28.0, "multi_turn_miss_param": 30.5, "multi_turn_long_context": 46.0, "web_search_accuracy": 8.5, "web_search_base": 13.0, "web_search_no_snippet": 4.0, "memory_accuracy": 18.49, "memory_kv": 5.81, "memory_vector": 16.13, "memory_recursive_summarization": 33.55, "relevance_detection": 75.0, "irrelevance_detection": 90.79, "organization": "Huawei Noah & USTC", "license": "Apache-2.0", "model_link": "https://huggingface.co/Team-ACE/ToolACE-2-8B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 41, "model": "Qwen3-30B-A3B-Instruct-2507 (FC)", "model_name": "Qwen3-30B-A3B-Instruct-2507 (FC)", "overall_accuracy": 41.39, "cost": 5.62, "latency_mean_s": 5.95, "latency_std_s": 25.48, "latency_p95_s": 12.7, "non_live_ast_accuracy": 85.77, "non_live_simple_ast": 68.58, "non_live_multiple_ast": 94.5, "non_live_parallel_ast": 91.5, "non_live_parallel_multiple_ast": 88.5, "live_accuracy": 77.94, "live_simple_ast": 83.33, "live_multiple_ast": 76.83, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 30.0, "multi_turn_base": 43.5, "multi_turn_miss_func": 10.5, "multi_turn_miss_param": 25.0, "multi_turn_long_context": 41.0, "web_search_accuracy": 22.5, "web_search_base": 21.0, "web_search_no_snippet": 24.0, "memory_accuracy": 17.63, "memory_kv": 9.03, "memory_vector": 9.03, "memory_recursive_summarization": 34.84, "relevance_detection": 81.25, "irrelevance_detection": 79.9, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 42, "model": "xLAM-2-3b-fc-r (FC)", "model_name": "xLAM-2-3b-fc-r (FC)", "overall_accuracy": 41.22, "cost": 3.36, "latency_mean_s": 3.8, "latency_std_s": 3.59, "latency_p95_s": 8.79, "non_live_ast_accuracy": 82.96, "non_live_simple_ast": 75.33, "non_live_multiple_ast": 91.0, "non_live_parallel_ast": 86.5, "non_live_parallel_multiple_ast": 79.0, "live_accuracy": 62.92, "live_simple_ast": 73.26, "live_multiple_ast": 60.68, "live_parallel_ast": 62.5, "live_parallel_multiple_ast": 50.0, "multi_turn_accuracy": 58.38, "multi_turn_base": 71.5, "multi_turn_miss_func": 59.0, "multi_turn_miss_param": 57.5, "multi_turn_long_context": 45.5, "web_search_accuracy": 2.5, "web_search_base": 3.0, "web_search_no_snippet": 2.0, "memory_accuracy": 11.4, "memory_kv": 5.81, "memory_vector": 5.81, "memory_recursive_summarization": 22.58, "relevance_detection": 87.5, "irrelevance_detection": 63.45, "organization": "Salesforce", "license": "cc-by-nc-4.0", "model_link": "https://huggingface.co/Salesforce/xLAM-2-3b-fc-r" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 43, "model": "Qwen3-14B (FC)", "model_name": "Qwen3-14B (FC)", "overall_accuracy": 41.03, "cost": 3.38, "latency_mean_s": 4.5, "latency_std_s": 18.84, "latency_p95_s": 13.34, "non_live_ast_accuracy": 84.94, "non_live_simple_ast": 74.75, "non_live_multiple_ast": 93.0, "non_live_parallel_ast": 80.0, "non_live_parallel_multiple_ast": 92.0, "live_accuracy": 80.01, "live_simple_ast": 85.66, "live_multiple_ast": 79.01, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 34.75, "multi_turn_base": 39.0, "multi_turn_miss_func": 34.0, "multi_turn_miss_param": 33.5, "multi_turn_long_context": 32.5, "web_search_accuracy": 10.0, "web_search_base": 8.0, "web_search_no_snippet": 12.0, "memory_accuracy": 19.57, "memory_kv": 7.1, "memory_vector": 16.77, "memory_recursive_summarization": 34.84, "relevance_detection": 87.5, "irrelevance_detection": 81.94, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-14B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 44, "model": "Qwen3-8B (Prompt)", "model_name": "Qwen3-8B (Prompt)", "overall_accuracy": 40.43, "cost": 63.95, "latency_mean_s": 54.17, "latency_std_s": 79.9, "latency_p95_s": 194.15, "non_live_ast_accuracy": 88.56, "non_live_simple_ast": 75.25, "non_live_multiple_ast": 95.0, "non_live_parallel_ast": 94.5, "non_live_parallel_multiple_ast": 89.5, "live_accuracy": 80.09, "live_simple_ast": 84.5, "live_multiple_ast": 78.92, "live_parallel_ast": 93.75, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 33.38, "multi_turn_base": 41.5, "multi_turn_miss_func": 38.5, "multi_turn_miss_param": 27.0, "multi_turn_long_context": 26.5, "web_search_accuracy": 13.5, "web_search_base": 19.0, "web_search_no_snippet": 8.0, "memory_accuracy": 13.12, "memory_kv": 3.87, "memory_vector": 10.32, "memory_recursive_summarization": 25.16, "relevance_detection": 75.0, "irrelevance_detection": 82.27, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-8B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 45, "model": "GPT-4.1-2025-04-14 (Prompt)", "model_name": "GPT-4.1-2025-04-14 (Prompt)", "overall_accuracy": 39.38, "cost": 145.85, "latency_mean_s": 1.2, "latency_std_s": 3.23, "latency_p95_s": 2.53, "non_live_ast_accuracy": 88.69, "non_live_simple_ast": 78.25, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 94.0, "non_live_parallel_multiple_ast": 89.0, "live_accuracy": 78.9, "live_simple_ast": 84.88, "live_multiple_ast": 77.4, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 9.75, "multi_turn_base": 10.5, "multi_turn_miss_func": 11.0, "multi_turn_miss_param": 8.0, "multi_turn_long_context": 9.5, "web_search_accuracy": 35.0, "web_search_base": 40.0, "web_search_no_snippet": 30.0, "memory_accuracy": 21.51, "memory_kv": 9.68, "memory_vector": 19.35, "memory_recursive_summarization": 35.48, "relevance_detection": 100.0, "irrelevance_detection": 83.99, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/gpt-4-1/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 46, "model": "mistral-large-2411 (FC)", "model_name": "mistral-large-2411 (FC)", "overall_accuracy": 38.37, "cost": 115.98, "latency_mean_s": 2.04, "latency_std_s": 4.02, "latency_p95_s": 4.68, "non_live_ast_accuracy": 84.65, "non_live_simple_ast": 72.08, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 89.5, "non_live_parallel_multiple_ast": 83.5, "live_accuracy": 81.87, "live_simple_ast": 87.21, "live_multiple_ast": 80.72, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 14.12, "multi_turn_base": 18.5, "multi_turn_miss_func": 11.5, "multi_turn_miss_param": 13.0, "multi_turn_long_context": 13.5, "web_search_accuracy": 28.0, "web_search_base": 41.0, "web_search_no_snippet": 15.0, "memory_accuracy": 24.95, "memory_kv": 18.71, "memory_vector": 29.03, "memory_recursive_summarization": 27.1, "relevance_detection": 93.75, "irrelevance_detection": 68.92, "organization": "Mistral AI", "license": "Proprietary", "model_link": "https://docs.mistral.ai/guides/model-selection/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 47, "model": "Qwen3-14B (Prompt)", "model_name": "Qwen3-14B (Prompt)", "overall_accuracy": 37.77, "cost": 1.35, "latency_mean_s": 1.2, "latency_std_s": 8.5, "latency_p95_s": 2.3, "non_live_ast_accuracy": 89.46, "non_live_simple_ast": 76.83, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 95.5, "non_live_parallel_multiple_ast": 92.0, "live_accuracy": 79.35, "live_simple_ast": 84.5, "live_multiple_ast": 78.06, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 26.13, "multi_turn_base": 16.5, "multi_turn_miss_func": 37.5, "multi_turn_miss_param": 31.0, "multi_turn_long_context": 19.5, "web_search_accuracy": 10.5, "web_search_base": 6.0, "web_search_no_snippet": 15.0, "memory_accuracy": 11.18, "memory_kv": 4.52, "memory_vector": 6.45, "memory_recursive_summarization": 22.58, "relevance_detection": 81.25, "irrelevance_detection": 87.18, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-14B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 48, "model": "Mistral-Medium-2505", "model_name": "Mistral-Medium-2505", "overall_accuracy": 37.69, "cost": 36.51, "latency_mean_s": 1.21, "latency_std_s": 3.5, "latency_p95_s": 2.86, "non_live_ast_accuracy": 85.33, "non_live_simple_ast": 76.33, "non_live_multiple_ast": 91.0, "non_live_parallel_ast": 88.5, "non_live_parallel_multiple_ast": 85.5, "live_accuracy": 66.03, "live_simple_ast": 80.23, "live_multiple_ast": 62.39, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 9.88, "multi_turn_base": 13.5, "multi_turn_miss_func": 6.5, "multi_turn_miss_param": 6.0, "multi_turn_long_context": 13.5, "web_search_accuracy": 39.0, "web_search_base": 41.0, "web_search_no_snippet": 37.0, "memory_accuracy": 21.72, "memory_kv": 16.13, "memory_vector": 14.84, "memory_recursive_summarization": 34.19, "relevance_detection": 75.0, "irrelevance_detection": 74.49, "organization": "Mistral AI", "license": "Proprietary", "model_link": "https://docs.mistral.ai/guides/model-selection/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 49, "model": "Mistral-Medium-2505 (FC)", "model_name": "Mistral-Medium-2505 (FC)", "overall_accuracy": 37.56, "cost": 18.8, "latency_mean_s": 1.6, "latency_std_s": 4.44, "latency_p95_s": 4.19, "non_live_ast_accuracy": 67.44, "non_live_simple_ast": 39.75, "non_live_multiple_ast": 78.0, "non_live_parallel_ast": 83.0, "non_live_parallel_multiple_ast": 69.0, "live_accuracy": 67.95, "live_simple_ast": 67.05, "live_multiple_ast": 68.09, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 10.75, "multi_turn_base": 15.5, "multi_turn_miss_func": 7.0, "multi_turn_miss_param": 7.5, "multi_turn_long_context": 13.0, "web_search_accuracy": 35.0, "web_search_base": 36.0, "web_search_no_snippet": 34.0, "memory_accuracy": 23.01, "memory_kv": 15.48, "memory_vector": 20.0, "memory_recursive_summarization": 33.55, "relevance_detection": 62.5, "irrelevance_detection": 91.95, "organization": "Mistral AI", "license": "Proprietary", "model_link": "https://docs.mistral.ai/guides/model-selection/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 50, "model": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", "model_name": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", "overall_accuracy": 37.29, "cost": 18.25, "latency_mean_s": 18.43, "latency_std_s": 34.11, "latency_p95_s": 102.75, "non_live_ast_accuracy": 88.65, "non_live_simple_ast": 77.08, "non_live_multiple_ast": 95.0, "non_live_parallel_ast": 94.0, "non_live_parallel_multiple_ast": 88.5, "live_accuracy": 73.65, "live_simple_ast": 84.5, "live_multiple_ast": 71.04, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 20.25, "multi_turn_base": 27.0, "multi_turn_miss_func": 22.0, "multi_turn_miss_param": 14.0, "multi_turn_long_context": 18.0, "web_search_accuracy": 28.0, "web_search_base": 39.0, "web_search_no_snippet": 17.0, "memory_accuracy": 18.92, "memory_kv": 8.39, "memory_vector": 32.9, "memory_recursive_summarization": 15.48, "relevance_detection": 100.0, "irrelevance_detection": 55.97, "organization": "Meta", "license": "Meta Llama 4 Community", "model_link": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 51, "model": "Mistral-small-2506 (FC)", "model_name": "Mistral-small-2506 (FC)", "overall_accuracy": 37.15, "cost": 5.2, "latency_mean_s": 1.48, "latency_std_s": 18.25, "latency_p95_s": 2.5, "non_live_ast_accuracy": 73.6, "non_live_simple_ast": 38.92, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 83.5, "non_live_parallel_multiple_ast": 78.5, "live_accuracy": 77.28, "live_simple_ast": 69.38, "live_multiple_ast": 79.39, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 11.5, "multi_turn_base": 17.5, "multi_turn_miss_func": 6.0, "multi_turn_miss_param": 10.5, "multi_turn_long_context": 12.0, "web_search_accuracy": 31.0, "web_search_base": 37.0, "web_search_no_snippet": 25.0, "memory_accuracy": 18.06, "memory_kv": 8.39, "memory_vector": 14.19, "memory_recursive_summarization": 31.61, "relevance_detection": 87.5, "irrelevance_detection": 87.94, "organization": "Mistral AI", "license": "Proprietary", "model_link": "https://docs.mistral.ai/guides/model-selection/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 52, "model": "Gemini-2.5-Flash-Lite (FC)", "model_name": "Gemini-2.5-Flash-Lite (FC)", "overall_accuracy": 36.87, "cost": 7.55, "latency_mean_s": 1.18, "latency_std_s": 8.06, "latency_p95_s": 1.67, "non_live_ast_accuracy": 86.6, "non_live_simple_ast": 70.92, "non_live_multiple_ast": 90.0, "non_live_parallel_ast": 93.5, "non_live_parallel_multiple_ast": 92.0, "live_accuracy": 65.8, "live_simple_ast": 73.26, "live_multiple_ast": 63.82, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 66.67, "multi_turn_accuracy": 13.5, "multi_turn_base": 20.0, "multi_turn_miss_func": 1.5, "multi_turn_miss_param": 15.0, "multi_turn_long_context": 17.5, "web_search_accuracy": 21.0, "web_search_base": 26.0, "web_search_no_snippet": 16.0, "memory_accuracy": 20.65, "memory_kv": 3.87, "memory_vector": 6.45, "memory_recursive_summarization": 51.61, "relevance_detection": 43.75, "irrelevance_detection": 92.5, "organization": "Google", "license": "Proprietary", "model_link": "https://deepmind.google/technologies/gemini/flash-lite/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 53, "model": "Qwen3-30B-A3B-Instruct-2507 (Prompt)", "model_name": "Qwen3-30B-A3B-Instruct-2507 (Prompt)", "overall_accuracy": 36.7, "cost": 1.56, "latency_mean_s": 1.24, "latency_std_s": 7.9, "latency_p95_s": 2.84, "non_live_ast_accuracy": 88.92, "non_live_simple_ast": 80.67, "non_live_multiple_ast": 93.0, "non_live_parallel_ast": 94.0, "non_live_parallel_multiple_ast": 88.0, "live_accuracy": 78.39, "live_simple_ast": 82.56, "live_multiple_ast": 77.49, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 66.67, "multi_turn_accuracy": 23.5, "multi_turn_base": 33.0, "multi_turn_miss_func": 16.0, "multi_turn_miss_param": 16.0, "multi_turn_long_context": 29.0, "web_search_accuracy": 17.5, "web_search_base": 15.0, "web_search_no_snippet": 20.0, "memory_accuracy": 9.68, "memory_kv": 5.81, "memory_vector": 6.45, "memory_recursive_summarization": 16.77, "relevance_detection": 93.75, "irrelevance_detection": 74.85, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 54, "model": "Qwen3-4B-Instruct-2507 (FC)", "model_name": "Qwen3-4B-Instruct-2507 (FC)", "overall_accuracy": 35.68, "cost": 6.37, "latency_mean_s": 7.61, "latency_std_s": 20.36, "latency_p95_s": 49.18, "non_live_ast_accuracy": 87.88, "non_live_simple_ast": 75.5, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 92.5, "non_live_parallel_multiple_ast": 90.0, "live_accuracy": 76.39, "live_simple_ast": 79.07, "live_multiple_ast": 76.16, "live_parallel_ast": 62.5, "live_parallel_multiple_ast": 66.67, "multi_turn_accuracy": 22.12, "multi_turn_base": 26.5, "multi_turn_miss_func": 21.0, "multi_turn_miss_param": 15.5, "multi_turn_long_context": 25.5, "web_search_accuracy": 3.0, "web_search_base": 4.0, "web_search_no_snippet": 2.0, "memory_accuracy": 17.63, "memory_kv": 16.13, "memory_vector": 12.26, "memory_recursive_summarization": 24.52, "relevance_detection": 87.5, "irrelevance_detection": 84.93, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 55, "model": "Qwen3-4B-Instruct-2507 (Prompt)", "model_name": "Qwen3-4B-Instruct-2507 (Prompt)", "overall_accuracy": 35.52, "cost": 53.66, "latency_mean_s": 44.7, "latency_std_s": 163.79, "latency_p95_s": 208.06, "non_live_ast_accuracy": 86.44, "non_live_simple_ast": 77.25, "non_live_multiple_ast": 91.0, "non_live_parallel_ast": 88.0, "non_live_parallel_multiple_ast": 89.5, "live_accuracy": 74.69, "live_simple_ast": 77.91, "live_multiple_ast": 74.17, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 20.5, "multi_turn_base": 24.5, "multi_turn_miss_func": 21.5, "multi_turn_miss_param": 16.0, "multi_turn_long_context": 20.0, "web_search_accuracy": 4.5, "web_search_base": 4.0, "web_search_no_snippet": 5.0, "memory_accuracy": 23.87, "memory_kv": 12.9, "memory_vector": 14.19, "memory_recursive_summarization": 44.52, "relevance_detection": 87.5, "irrelevance_detection": 75.87, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 56, "model": "Arch-Agent-3B", "model_name": "Arch-Agent-3B", "overall_accuracy": 35.36, "cost": 3.7, "latency_mean_s": 3.56, "latency_std_s": 6.65, "latency_p95_s": 8.19, "non_live_ast_accuracy": 86.67, "non_live_simple_ast": 78.67, "non_live_multiple_ast": 94.5, "non_live_parallel_ast": 91.0, "non_live_parallel_multiple_ast": 82.5, "live_accuracy": 72.91, "live_simple_ast": 75.58, "live_multiple_ast": 72.27, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 34.88, "multi_turn_base": 42.0, "multi_turn_miss_func": 37.5, "multi_turn_miss_param": 31.0, "multi_turn_long_context": 29.0, "web_search_accuracy": 0.5, "web_search_base": 1.0, "web_search_no_snippet": 0.0, "memory_accuracy": 6.88, "memory_kv": 5.16, "memory_vector": 5.81, "memory_recursive_summarization": 9.68, "relevance_detection": 68.75, "irrelevance_detection": 74.67, "organization": "katanemo", "license": "katanemo-research", "model_link": "https://huggingface.co/katanemo/Arch-Agent-3B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 57, "model": "Claude-Opus-4-5-20251101 (Prompt)", "model_name": "Claude-Opus-4-5-20251101 (Prompt)", "overall_accuracy": 33.47, "cost": 88.33, "latency_mean_s": 3.76, "latency_std_s": 13.19, "latency_p95_s": 5.52, "non_live_ast_accuracy": 89.65, "non_live_simple_ast": 79.58, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 93.0, "non_live_parallel_multiple_ast": 92.5, "live_accuracy": 76.02, "live_simple_ast": 84.5, "live_multiple_ast": 74.17, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 16.12, "multi_turn_base": 20.5, "multi_turn_miss_func": 9.0, "multi_turn_miss_param": 21.5, "multi_turn_long_context": 13.5, "web_search_accuracy": 13.0, "web_search_base": 13.0, "web_search_no_snippet": 13.0, "memory_accuracy": 1.94, "memory_kv": 1.29, "memory_vector": 1.94, "memory_recursive_summarization": 2.58, "relevance_detection": 68.75, "irrelevance_detection": 90.75, "organization": "Anthropic", "license": "Proprietary", "model_link": "https://www.anthropic.com/news/claude-4" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 58, "model": "GPT-4.1-nano-2025-04-14 (FC)", "model_name": "GPT-4.1-nano-2025-04-14 (FC)", "overall_accuracy": 33.05, "cost": 5.66, "latency_mean_s": 1.44, "latency_std_s": 10.84, "latency_p95_s": 2.26, "non_live_ast_accuracy": 72.98, "non_live_simple_ast": 59.92, "non_live_multiple_ast": 79.5, "non_live_parallel_ast": 84.0, "non_live_parallel_multiple_ast": 68.5, "live_accuracy": 60.77, "live_simple_ast": 58.14, "live_multiple_ast": 61.44, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 54.17, "multi_turn_accuracy": 23.62, "multi_turn_base": 39.5, "multi_turn_miss_func": 7.5, "multi_turn_miss_param": 17.5, "multi_turn_long_context": 30.0, "web_search_accuracy": 11.0, "web_search_base": 13.0, "web_search_no_snippet": 9.0, "memory_accuracy": 18.92, "memory_kv": 10.32, "memory_vector": 19.35, "memory_recursive_summarization": 27.1, "relevance_detection": 93.75, "irrelevance_detection": 66.0, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/gpt-4-1/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 59, "model": "Mistral-Small-2506 (Prompt)", "model_name": "Mistral-Small-2506 (Prompt)", "overall_accuracy": 32.38, "cost": 6.91, "latency_mean_s": 0.92, "latency_std_s": 6.79, "latency_p95_s": 2.02, "non_live_ast_accuracy": 89.69, "non_live_simple_ast": 78.75, "non_live_multiple_ast": 96.0, "non_live_parallel_ast": 92.5, "non_live_parallel_multiple_ast": 91.5, "live_accuracy": 79.05, "live_simple_ast": 81.4, "live_multiple_ast": 78.54, "live_parallel_ast": 93.75, "live_parallel_multiple_ast": 66.67, "multi_turn_accuracy": 14.75, "multi_turn_base": 20.5, "multi_turn_miss_func": 17.0, "multi_turn_miss_param": 9.5, "multi_turn_long_context": 12.0, "web_search_accuracy": 7.5, "web_search_base": 9.0, "web_search_no_snippet": 6.0, "memory_accuracy": 15.05, "memory_kv": 2.58, "memory_vector": 11.61, "memory_recursive_summarization": 30.97, "relevance_detection": 93.75, "irrelevance_detection": 65.73, "organization": "Mistral AI", "license": "Proprietary", "model_link": "https://docs.mistral.ai/guides/model-selection/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 60, "model": "Arch-Agent-1.5B", "model_name": "Arch-Agent-1.5B", "overall_accuracy": 32.14, "cost": 2.45, "latency_mean_s": 2.38, "latency_std_s": 4.01, "latency_p95_s": 5.3, "non_live_ast_accuracy": 82.67, "non_live_simple_ast": 72.17, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 85.5, "non_live_parallel_multiple_ast": 81.0, "live_accuracy": 67.73, "live_simple_ast": 70.54, "live_multiple_ast": 67.81, "live_parallel_ast": 31.25, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 26.62, "multi_turn_base": 35.5, "multi_turn_miss_func": 27.5, "multi_turn_miss_param": 21.5, "multi_turn_long_context": 22.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 8.17, "memory_kv": 5.81, "memory_vector": 5.81, "memory_recursive_summarization": 12.9, "relevance_detection": 75.0, "irrelevance_detection": 74.83, "organization": "katanemo", "license": "katanemo-research", "model_link": "https://huggingface.co/katanemo/Arch-Agent-1.5B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 61, "model": "Command R7B (FC)", "model_name": "Command R7B (FC)", "overall_accuracy": 32.07, "cost": 1.5, "latency_mean_s": 1.38, "latency_std_s": 2.87, "latency_p95_s": 2.69, "non_live_ast_accuracy": 80.96, "non_live_simple_ast": 67.33, "non_live_multiple_ast": 89.5, "non_live_parallel_ast": 85.5, "non_live_parallel_multiple_ast": 81.5, "live_accuracy": 69.06, "live_simple_ast": 62.79, "live_multiple_ast": 70.94, "live_parallel_ast": 43.75, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 8.25, "multi_turn_base": 12.0, "multi_turn_miss_func": 0.5, "multi_turn_miss_param": 10.5, "multi_turn_long_context": 10.0, "web_search_accuracy": 27.0, "web_search_base": 43.0, "web_search_no_snippet": 11.0, "memory_accuracy": 5.16, "memory_kv": 2.58, "memory_vector": 9.68, "memory_recursive_summarization": 3.23, "relevance_detection": 68.75, "irrelevance_detection": 81.65, "organization": "Cohere", "license": "cc-by-nc-4.0", "model_link": "https://cohere.com/blog/command-r7b" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 62, "model": "Llama-3.3-70B-Instruct (FC)", "model_name": "Llama-3.3-70B-Instruct (FC)", "overall_accuracy": 31.9, "cost": 29.54, "latency_mean_s": 26.11, "latency_std_s": 93.22, "latency_p95_s": 187.93, "non_live_ast_accuracy": 88.02, "non_live_simple_ast": 76.08, "non_live_multiple_ast": 95.0, "non_live_parallel_ast": 90.0, "non_live_parallel_multiple_ast": 91.0, "live_accuracy": 76.61, "live_simple_ast": 81.4, "live_multiple_ast": 75.5, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 21.5, "multi_turn_base": 26.0, "multi_turn_miss_func": 19.0, "multi_turn_miss_param": 14.5, "multi_turn_long_context": 26.5, "web_search_accuracy": 10.0, "web_search_base": 14.0, "web_search_no_snippet": 6.0, "memory_accuracy": 8.17, "memory_kv": 4.52, "memory_vector": 8.39, "memory_recursive_summarization": 11.61, "relevance_detection": 100.0, "irrelevance_detection": 53.53, "organization": "Meta", "license": "Meta Llama 3 Community", "model_link": "https://llama.meta.com/llama3" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 63, "model": "mistral-large-2411 (Prompt)", "model_name": "mistral-large-2411 (Prompt)", "overall_accuracy": 31.84, "cost": 232.42, "latency_mean_s": 1.82, "latency_std_s": 7.15, "latency_p95_s": 4.08, "non_live_ast_accuracy": 83.0, "non_live_simple_ast": 75.5, "non_live_multiple_ast": 89.5, "non_live_parallel_ast": 87.0, "non_live_parallel_multiple_ast": 80.0, "live_accuracy": 68.1, "live_simple_ast": 83.72, "live_multiple_ast": 64.01, "live_parallel_ast": 93.75, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 13.75, "multi_turn_base": 20.0, "multi_turn_miss_func": 5.0, "multi_turn_miss_param": 11.0, "multi_turn_long_context": 19.0, "web_search_accuracy": 20.0, "web_search_base": 28.0, "web_search_no_snippet": 12.0, "memory_accuracy": 23.66, "memory_kv": 16.77, "memory_vector": 30.97, "memory_recursive_summarization": 23.23, "relevance_detection": 93.75, "irrelevance_detection": 38.77, "organization": "Mistral AI", "license": "Proprietary", "model_link": "https://docs.mistral.ai/guides/model-selection/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 64, "model": "Hammer2.1-7b (FC)", "model_name": "Hammer2.1-7b (FC)", "overall_accuracy": 31.67, "cost": 4.99, "latency_mean_s": 5.77, "latency_std_s": 10.29, "latency_p95_s": 31.26, "non_live_ast_accuracy": 85.5, "non_live_simple_ast": 72.5, "non_live_multiple_ast": 92.5, "non_live_parallel_ast": 91.0, "non_live_parallel_multiple_ast": 86.0, "live_accuracy": 69.5, "live_simple_ast": 66.67, "live_multiple_ast": 69.99, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 23.87, "multi_turn_base": 24.0, "multi_turn_miss_func": 28.5, "multi_turn_miss_param": 21.5, "multi_turn_long_context": 21.5, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 0.0, "memory_kv": 0.0, "memory_vector": 0.0, "memory_recursive_summarization": 0.0, "relevance_detection": 50.0, "irrelevance_detection": 90.12, "organization": "MadeAgents", "license": "cc-by-nc-4.0", "model_link": "https://huggingface.co/MadeAgents/Hammer2.1-7b" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 65, "model": "xLAM-2-1b-fc-r (FC)", "model_name": "xLAM-2-1b-fc-r (FC)", "overall_accuracy": 30.44, "cost": 2.79, "latency_mean_s": 2.84, "latency_std_s": 2.35, "latency_p95_s": 6.52, "non_live_ast_accuracy": 69.04, "non_live_simple_ast": 64.17, "non_live_multiple_ast": 82.5, "non_live_parallel_ast": 73.5, "non_live_parallel_multiple_ast": 56.0, "live_accuracy": 55.14, "live_simple_ast": 68.22, "live_multiple_ast": 52.8, "live_parallel_ast": 43.75, "live_parallel_multiple_ast": 25.0, "multi_turn_accuracy": 36.0, "multi_turn_base": 45.5, "multi_turn_miss_func": 36.0, "multi_turn_miss_param": 37.0, "multi_turn_long_context": 25.5, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 3.87, "memory_kv": 3.87, "memory_vector": 3.87, "memory_recursive_summarization": 3.87, "relevance_detection": 87.5, "irrelevance_detection": 64.47, "organization": "Salesforce", "license": "cc-by-nc-4.0", "model_link": "https://huggingface.co/Salesforce/xLAM-2-1b-fc-r" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 66, "model": "Gemma-3-12b-it (Prompt)", "model_name": "Gemma-3-12b-it (Prompt)", "overall_accuracy": 30.43, "cost": 10.77, "latency_mean_s": 11.1, "latency_std_s": 17.17, "latency_p95_s": 34.66, "non_live_ast_accuracy": 79.44, "non_live_simple_ast": 76.25, "non_live_multiple_ast": 94.0, "non_live_parallel_ast": 91.0, "non_live_parallel_multiple_ast": 56.5, "live_accuracy": 74.24, "live_simple_ast": 85.66, "live_multiple_ast": 71.89, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 45.83, "multi_turn_accuracy": 5.75, "multi_turn_base": 6.5, "multi_turn_miss_func": 7.5, "multi_turn_miss_param": 5.0, "multi_turn_long_context": 4.0, "web_search_accuracy": 4.0, "web_search_base": 4.0, "web_search_no_snippet": 4.0, "memory_accuracy": 27.53, "memory_kv": 8.39, "memory_vector": 25.16, "memory_recursive_summarization": 49.03, "relevance_detection": 93.75, "irrelevance_detection": 70.29, "organization": "Google", "license": "gemma-terms-of-use", "model_link": "https://blog.google/technology/developers/gemma-3/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 67, "model": "GPT-4.1-mini-2025-04-14 (Prompt)", "model_name": "GPT-4.1-mini-2025-04-14 (Prompt)", "overall_accuracy": 29.73, "cost": 20.52, "latency_mean_s": 1.36, "latency_std_s": 4.5, "latency_p95_s": 3.38, "non_live_ast_accuracy": 84.6, "non_live_simple_ast": 74.92, "non_live_multiple_ast": 92.5, "non_live_parallel_ast": 87.5, "non_live_parallel_multiple_ast": 83.5, "live_accuracy": 74.76, "live_simple_ast": 80.62, "live_multiple_ast": 73.31, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 2.5, "multi_turn_base": 1.5, "multi_turn_miss_func": 4.5, "multi_turn_miss_param": 2.5, "multi_turn_long_context": 1.5, "web_search_accuracy": 4.0, "web_search_base": 7.0, "web_search_no_snippet": 1.0, "memory_accuracy": 24.3, "memory_kv": 20.65, "memory_vector": 13.55, "memory_recursive_summarization": 38.71, "relevance_detection": 87.5, "irrelevance_detection": 73.88, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/gpt-4-1/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 68, "model": "Hammer2.1-3b (FC)", "model_name": "Hammer2.1-3b (FC)", "overall_accuracy": 29.71, "cost": 10.89, "latency_mean_s": 11.24, "latency_std_s": 15.81, "latency_p95_s": 47.44, "non_live_ast_accuracy": 84.96, "non_live_simple_ast": 79.33, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 86.5, "non_live_parallel_multiple_ast": 80.5, "live_accuracy": 70.54, "live_simple_ast": 68.22, "live_multiple_ast": 71.32, "live_parallel_ast": 62.5, "live_parallel_multiple_ast": 66.67, "multi_turn_accuracy": 16.5, "multi_turn_base": 22.0, "multi_turn_miss_func": 12.5, "multi_turn_miss_param": 16.0, "multi_turn_long_context": 15.5, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 3.01, "memory_kv": 2.58, "memory_vector": 3.87, "memory_recursive_summarization": 2.58, "relevance_detection": 56.25, "irrelevance_detection": 86.12, "organization": "MadeAgents", "license": "qwen-research", "model_link": "https://huggingface.co/MadeAgents/Hammer2.1-3b" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 69, "model": "Gemma-3-27b-it (Prompt)", "model_name": "Gemma-3-27b-it (Prompt)", "overall_accuracy": 29.47, "cost": 11.82, "latency_mean_s": 10.88, "latency_std_s": 19.67, "latency_p95_s": 55.5, "non_live_ast_accuracy": 87.17, "non_live_simple_ast": 77.67, "non_live_multiple_ast": 92.5, "non_live_parallel_ast": 89.0, "non_live_parallel_multiple_ast": 89.5, "live_accuracy": 74.54, "live_simple_ast": 84.5, "live_multiple_ast": 72.46, "live_parallel_ast": 93.75, "live_parallel_multiple_ast": 45.83, "multi_turn_accuracy": 10.75, "multi_turn_base": 16.5, "multi_turn_miss_func": 4.5, "multi_turn_miss_param": 8.0, "multi_turn_long_context": 14.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 13.55, "memory_kv": 1.94, "memory_vector": 3.23, "memory_recursive_summarization": 35.48, "relevance_detection": 81.25, "irrelevance_detection": 73.67, "organization": "Google", "license": "gemma-terms-of-use", "model_link": "https://blog.google/technology/developers/gemma-3/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 70, "model": "Phi-4 (Prompt)", "model_name": "Phi-4 (Prompt)", "overall_accuracy": 28.79, "cost": 8.72, "latency_mean_s": 9.49, "latency_std_s": 26.73, "latency_p95_s": 23.02, "non_live_ast_accuracy": 69.56, "non_live_simple_ast": 74.25, "non_live_multiple_ast": 89.5, "non_live_parallel_ast": 65.0, "non_live_parallel_multiple_ast": 49.5, "live_accuracy": 60.7, "live_simple_ast": 65.5, "live_multiple_ast": 59.64, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 41.67, "multi_turn_accuracy": 3.88, "multi_turn_base": 9.0, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 3.5, "multi_turn_long_context": 3.0, "web_search_accuracy": 4.5, "web_search_base": 4.0, "web_search_no_snippet": 5.0, "memory_accuracy": 24.73, "memory_kv": 17.42, "memory_vector": 25.16, "memory_recursive_summarization": 31.61, "relevance_detection": 50.0, "irrelevance_detection": 87.55, "organization": "Microsoft", "license": "MIT", "model_link": "https://huggingface.co/microsoft/phi-4" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 71, "model": "Qwen3-1.7B (FC)", "model_name": "Qwen3-1.7B (FC)", "overall_accuracy": 28.41, "cost": 4.33, "latency_mean_s": 5.12, "latency_std_s": 7.37, "latency_p95_s": 13.35, "non_live_ast_accuracy": 82.92, "non_live_simple_ast": 70.67, "non_live_multiple_ast": 92.5, "non_live_parallel_ast": 88.5, "non_live_parallel_multiple_ast": 80.0, "live_accuracy": 74.61, "live_simple_ast": 76.74, "live_multiple_ast": 74.26, "live_parallel_ast": 62.5, "live_parallel_multiple_ast": 75.0, "multi_turn_accuracy": 11.0, "multi_turn_base": 15.0, "multi_turn_miss_func": 6.0, "multi_turn_miss_param": 12.0, "multi_turn_long_context": 11.0, "web_search_accuracy": 2.5, "web_search_base": 3.0, "web_search_no_snippet": 2.0, "memory_accuracy": 6.02, "memory_kv": 4.52, "memory_vector": 7.74, "memory_recursive_summarization": 5.81, "relevance_detection": 81.25, "irrelevance_detection": 76.54, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-1.7B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 72, "model": "Llama-4-Scout-17B-16E-Instruct (FC)", "model_name": "Llama-4-Scout-17B-16E-Instruct (FC)", "overall_accuracy": 28.13, "cost": 24.68, "latency_mean_s": 17.86, "latency_std_s": 50.68, "latency_p95_s": 166.2, "non_live_ast_accuracy": 89.38, "non_live_simple_ast": 79.0, "non_live_multiple_ast": 94.0, "non_live_parallel_ast": 94.0, "non_live_parallel_multiple_ast": 90.5, "live_accuracy": 74.69, "live_simple_ast": 81.78, "live_multiple_ast": 72.74, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 79.17, "multi_turn_accuracy": 9.0, "multi_turn_base": 12.0, "multi_turn_miss_func": 7.0, "multi_turn_miss_param": 7.5, "multi_turn_long_context": 9.5, "web_search_accuracy": 14.5, "web_search_base": 18.0, "web_search_no_snippet": 11.0, "memory_accuracy": 8.17, "memory_kv": 2.58, "memory_vector": 2.58, "memory_recursive_summarization": 19.35, "relevance_detection": 100.0, "irrelevance_detection": 44.92, "organization": "Meta", "license": "Meta Llama 4 Community", "model_link": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 73, "model": "Gemini-2.5-Flash-Lite (Prompt)", "model_name": "Gemini-2.5-Flash-Lite (Prompt)", "overall_accuracy": 28.03, "cost": 7.05, "latency_mean_s": 1.0, "latency_std_s": 4.75, "latency_p95_s": 1.4, "non_live_ast_accuracy": 83.9, "non_live_simple_ast": 70.08, "non_live_multiple_ast": 86.0, "non_live_parallel_ast": 90.0, "non_live_parallel_multiple_ast": 89.5, "live_accuracy": 54.85, "live_simple_ast": 67.05, "live_multiple_ast": 51.66, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 50.0, "multi_turn_accuracy": 7.63, "multi_turn_base": 10.0, "multi_turn_miss_func": 5.0, "multi_turn_miss_param": 6.5, "multi_turn_long_context": 9.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 12.69, "memory_kv": 1.94, "memory_vector": 6.45, "memory_recursive_summarization": 29.68, "relevance_detection": 50.0, "irrelevance_detection": 93.33, "organization": "Google", "license": "Proprietary", "model_link": "https://deepmind.google/technologies/gemini/flash-lite/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 74, "model": "CoALM-70B", "model_name": "CoALM-70B", "overall_accuracy": 27.99, "cost": 19.89, "latency_mean_s": 16.22, "latency_std_s": 59.91, "latency_p95_s": 36.0, "non_live_ast_accuracy": 83.44, "non_live_simple_ast": 70.25, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 88.5, "non_live_parallel_multiple_ast": 83.0, "live_accuracy": 67.28, "live_simple_ast": 70.54, "live_multiple_ast": 66.57, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 10.62, "multi_turn_base": 11.0, "multi_turn_miss_func": 14.0, "multi_turn_miss_param": 9.0, "multi_turn_long_context": 8.5, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 5.81, "memory_kv": 9.03, "memory_vector": 5.16, "memory_recursive_summarization": 3.23, "relevance_detection": 93.75, "irrelevance_detection": 85.65, "organization": "UIUC + Oumi", "license": "Meta Llama 3 Community", "model_link": "https://huggingface.co/uiuc-convai/CoALM-70B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 75, "model": "Hammer2.1-1.5b (FC)", "model_name": "Hammer2.1-1.5b (FC)", "overall_accuracy": 27.88, "cost": 6.83, "latency_mean_s": 6.28, "latency_std_s": 8.79, "latency_p95_s": 30.72, "non_live_ast_accuracy": 82.98, "non_live_simple_ast": 73.92, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 85.5, "non_live_parallel_multiple_ast": 80.5, "live_accuracy": 69.5, "live_simple_ast": 72.09, "live_multiple_ast": 69.33, "live_parallel_ast": 50.0, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 15.62, "multi_turn_base": 20.0, "multi_turn_miss_func": 16.5, "multi_turn_miss_param": 9.5, "multi_turn_long_context": 16.5, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 0.0, "memory_kv": 0.0, "memory_vector": 0.0, "memory_recursive_summarization": 0.0, "relevance_detection": 75.0, "irrelevance_detection": 79.4, "organization": "MadeAgents", "license": "cc-by-nc-4.0", "model_link": "https://huggingface.co/MadeAgents/Hammer2.1-1.5b" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 76, "model": "palmyra-x-004 (FC)", "model_name": "palmyra-x-004 (FC)", "overall_accuracy": 27.87, "cost": 178.15, "latency_mean_s": 3.71, "latency_std_s": 7.62, "latency_p95_s": 8.04, "non_live_ast_accuracy": 87.46, "non_live_simple_ast": 71.33, "non_live_multiple_ast": 96.0, "non_live_parallel_ast": 93.0, "non_live_parallel_multiple_ast": 89.5, "live_accuracy": 77.87, "live_simple_ast": 79.46, "live_multiple_ast": 77.97, "live_parallel_ast": 56.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 0.38, "multi_turn_base": 0.5, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 0.5, "multi_turn_long_context": 0.5, "web_search_accuracy": 2.5, "web_search_base": 4.0, "web_search_no_snippet": 1.0, "memory_accuracy": 13.12, "memory_kv": 6.45, "memory_vector": 14.19, "memory_recursive_summarization": 18.71, "relevance_detection": 81.25, "irrelevance_detection": 80.99, "organization": "Writer", "license": "Proprietary", "model_link": "https://writer.com/engineering/actions-with-palmyra-x-004/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 77, "model": "GPT-5-mini-2025-08-07 (Prompt)", "model_name": "GPT-5-mini-2025-08-07 (Prompt)", "overall_accuracy": 27.83, "cost": 82.74, "latency_mean_s": 8.89, "latency_std_s": 11.08, "latency_p95_s": 19.72, "non_live_ast_accuracy": 68.04, "non_live_simple_ast": 59.17, "non_live_multiple_ast": 72.5, "non_live_parallel_ast": 71.5, "non_live_parallel_multiple_ast": 69.0, "live_accuracy": 62.55, "live_simple_ast": 69.77, "live_multiple_ast": 61.16, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 37.5, "multi_turn_accuracy": 5.5, "multi_turn_base": 5.5, "multi_turn_miss_func": 5.0, "multi_turn_miss_param": 4.5, "multi_turn_long_context": 7.0, "web_search_accuracy": 8.5, "web_search_base": 11.0, "web_search_no_snippet": 6.0, "memory_accuracy": 29.25, "memory_kv": 19.35, "memory_vector": 29.68, "memory_recursive_summarization": 38.71, "relevance_detection": 93.75, "irrelevance_detection": 55.71, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/introducing-gpt-5/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 78, "model": "Open-Mistral-Nemo-2407 (FC)", "model_name": "Open-Mistral-Nemo-2407 (FC)", "overall_accuracy": 27.63, "cost": 8.12, "latency_mean_s": 1.07, "latency_std_s": 11.93, "latency_p95_s": 1.39, "non_live_ast_accuracy": 82.81, "non_live_simple_ast": 65.75, "non_live_multiple_ast": 92.5, "non_live_parallel_ast": 88.5, "non_live_parallel_multiple_ast": 84.5, "live_accuracy": 73.8, "live_simple_ast": 78.68, "live_multiple_ast": 72.84, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 66.67, "multi_turn_accuracy": 7.75, "multi_turn_base": 12.5, "multi_turn_miss_func": 6.5, "multi_turn_miss_param": 7.5, "multi_turn_long_context": 4.5, "web_search_accuracy": 7.0, "web_search_base": 9.0, "web_search_no_snippet": 5.0, "memory_accuracy": 10.32, "memory_kv": 8.39, "memory_vector": 9.68, "memory_recursive_summarization": 12.9, "relevance_detection": 81.25, "irrelevance_detection": 61.77, "organization": "Mistral AI", "license": "Proprietary", "model_link": "https://mistral.ai/news/mistral-nemo/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 79, "model": "GPT-5-nano-2025-08-07 (Prompt)", "model_name": "GPT-5-nano-2025-08-07 (Prompt)", "overall_accuracy": 27.55, "cost": 21.47, "latency_mean_s": 10.67, "latency_std_s": 7.68, "latency_p95_s": 23.28, "non_live_ast_accuracy": 80.81, "non_live_simple_ast": 69.25, "non_live_multiple_ast": 86.0, "non_live_parallel_ast": 87.5, "non_live_parallel_multiple_ast": 80.5, "live_accuracy": 70.69, "live_simple_ast": 76.36, "live_multiple_ast": 69.71, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 54.17, "multi_turn_accuracy": 0.75, "multi_turn_base": 1.0, "multi_turn_miss_func": 1.0, "multi_turn_miss_param": 0.0, "multi_turn_long_context": 1.0, "web_search_accuracy": 13.5, "web_search_base": 10.0, "web_search_no_snippet": 17.0, "memory_accuracy": 24.52, "memory_kv": 20.65, "memory_vector": 31.61, "memory_recursive_summarization": 21.29, "relevance_detection": 93.75, "irrelevance_detection": 45.75, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/introducing-gpt-5/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 80, "model": "Amazon-Nova-2-Lite-v1:0 (FC)", "model_name": "Amazon-Nova-2-Lite-v1:0 (FC)", "overall_accuracy": 27.1, "cost": 78.19, "latency_mean_s": 8.55, "latency_std_s": 9.85, "latency_p95_s": 27.62, "non_live_ast_accuracy": 86.96, "non_live_simple_ast": 76.33, "non_live_multiple_ast": 94.0, "non_live_parallel_ast": 91.5, "non_live_parallel_multiple_ast": 86.0, "live_accuracy": 80.83, "live_simple_ast": 83.33, "live_multiple_ast": 80.15, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 79.17, "multi_turn_accuracy": 2.12, "multi_turn_base": 2.5, "multi_turn_miss_func": 1.5, "multi_turn_miss_param": 2.0, "multi_turn_long_context": 2.5, "web_search_accuracy": 5.0, "web_search_base": 4.0, "web_search_no_snippet": 6.0, "memory_accuracy": 2.37, "memory_kv": 1.94, "memory_vector": 3.23, "memory_recursive_summarization": 1.94, "relevance_detection": 75.0, "irrelevance_detection": 82.11, "organization": "Amazon", "license": "Proprietary", "model_link": "https://aws.amazon.com/cn/ai/generative-ai/nova/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 81, "model": "Granite-3.1-8B-Instruct (FC)", "model_name": "Granite-3.1-8B-Instruct (FC)", "overall_accuracy": 27.1, "cost": 9.32, "latency_mean_s": 13.23, "latency_std_s": 31.28, "latency_p95_s": 65.19, "non_live_ast_accuracy": 78.33, "non_live_simple_ast": 67.33, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 84.0, "non_live_parallel_multiple_ast": 70.0, "live_accuracy": 60.33, "live_simple_ast": 58.53, "live_multiple_ast": 61.82, "live_parallel_ast": 18.75, "live_parallel_multiple_ast": 41.67, "multi_turn_accuracy": 7.5, "multi_turn_base": 11.5, "multi_turn_miss_func": 2.0, "multi_turn_miss_param": 7.5, "multi_turn_long_context": 9.0, "web_search_accuracy": 0.5, "web_search_base": 1.0, "web_search_no_snippet": 0.0, "memory_accuracy": 14.41, "memory_kv": 9.68, "memory_vector": 7.1, "memory_recursive_summarization": 26.45, "relevance_detection": 68.75, "irrelevance_detection": 79.98, "organization": "IBM", "license": "Apache-2.0", "model_link": "https://huggingface.co/ibm-granite/granite-3.1-8b-instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 82, "model": "Falcon3-10B-Instruct (FC)", "model_name": "Falcon3-10B-Instruct (FC)", "overall_accuracy": 27.01, "cost": 52.59, "latency_mean_s": 69.27, "latency_std_s": 92.22, "latency_p95_s": 190.96, "non_live_ast_accuracy": 85.0, "non_live_simple_ast": 70.5, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 88.5, "non_live_parallel_multiple_ast": 87.5, "live_accuracy": 75.43, "live_simple_ast": 77.13, "live_multiple_ast": 76.16, "live_parallel_ast": 50.0, "live_parallel_multiple_ast": 41.67, "multi_turn_accuracy": 6.5, "multi_turn_base": 6.5, "multi_turn_miss_func": 9.5, "multi_turn_miss_param": 5.0, "multi_turn_long_context": 5.0, "web_search_accuracy": 1.5, "web_search_base": 2.0, "web_search_no_snippet": 1.0, "memory_accuracy": 27.53, "memory_kv": 12.26, "memory_vector": 19.35, "memory_recursive_summarization": 50.97, "relevance_detection": 93.75, "irrelevance_detection": 32.09, "organization": "TII UAE", "license": "falcon-llm-license", "model_link": "https://huggingface.co/tiiuae/Falcon3-10B-Instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 83, "model": "Granite-3.2-8B-Instruct (FC)", "model_name": "Granite-3.2-8B-Instruct (FC)", "overall_accuracy": 26.87, "cost": 25.02, "latency_mean_s": 36.13, "latency_std_s": 81.76, "latency_p95_s": 216.28, "non_live_ast_accuracy": 79.77, "non_live_simple_ast": 69.58, "non_live_multiple_ast": 88.5, "non_live_parallel_ast": 88.5, "non_live_parallel_multiple_ast": 72.5, "live_accuracy": 60.33, "live_simple_ast": 60.47, "live_multiple_ast": 61.16, "live_parallel_ast": 25.0, "live_parallel_multiple_ast": 45.83, "multi_turn_accuracy": 7.38, "multi_turn_base": 9.5, "multi_turn_miss_func": 3.0, "multi_turn_miss_param": 8.0, "multi_turn_long_context": 9.0, "web_search_accuracy": 0.5, "web_search_base": 1.0, "web_search_no_snippet": 0.0, "memory_accuracy": 12.47, "memory_kv": 6.45, "memory_vector": 9.68, "memory_recursive_summarization": 21.29, "relevance_detection": 75.0, "irrelevance_detection": 80.53, "organization": "IBM", "license": "Apache-2.0", "model_link": "https://huggingface.co/ibm-granite/granite-3.2-8b-instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 84, "model": "CoALM-8B", "model_name": "CoALM-8B", "overall_accuracy": 26.81, "cost": 25.33, "latency_mean_s": 20.36, "latency_std_s": 73.74, "latency_p95_s": 138.04, "non_live_ast_accuracy": 84.87, "non_live_simple_ast": 69.5, "non_live_multiple_ast": 93.5, "non_live_parallel_ast": 88.0, "non_live_parallel_multiple_ast": 88.5, "live_accuracy": 66.77, "live_simple_ast": 70.54, "live_multiple_ast": 66.19, "live_parallel_ast": 62.5, "live_parallel_multiple_ast": 54.17, "multi_turn_accuracy": 8.0, "multi_turn_base": 10.0, "multi_turn_miss_func": 7.0, "multi_turn_miss_param": 8.0, "multi_turn_long_context": 7.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 2.8, "memory_kv": 3.23, "memory_vector": 3.87, "memory_recursive_summarization": 1.29, "relevance_detection": 87.5, "irrelevance_detection": 86.9, "organization": "UIUC + Oumi", "license": "Meta Llama 3 Community", "model_link": "https://huggingface.co/uiuc-convai/CoALM-8B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 85, "model": "Llama-3.1-8B-Instruct (Prompt)", "model_name": "Llama-3.1-8B-Instruct (Prompt)", "overall_accuracy": 25.83, "cost": 7.49, "latency_mean_s": 5.6, "latency_std_s": 19.37, "latency_p95_s": 22.6, "non_live_ast_accuracy": 84.0, "non_live_simple_ast": 71.0, "non_live_multiple_ast": 95.0, "non_live_parallel_ast": 87.5, "non_live_parallel_multiple_ast": 82.5, "live_accuracy": 70.76, "live_simple_ast": 72.87, "live_multiple_ast": 71.13, "live_parallel_ast": 50.0, "live_parallel_multiple_ast": 45.83, "multi_turn_accuracy": 11.12, "multi_turn_base": 13.0, "multi_turn_miss_func": 9.0, "multi_turn_miss_param": 9.5, "multi_turn_long_context": 13.0, "web_search_accuracy": 3.0, "web_search_base": 6.0, "web_search_no_snippet": 0.0, "memory_accuracy": 10.75, "memory_kv": 7.74, "memory_vector": 5.81, "memory_recursive_summarization": 18.71, "relevance_detection": 93.75, "irrelevance_detection": 42.7, "organization": "Meta", "license": "Meta Llama 3 Community", "model_link": "https://llama.meta.com/llama3" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 86, "model": "MiniCPM3-4B-FC (FC)", "model_name": "MiniCPM3-4B-FC (FC)", "overall_accuracy": 25.55, "cost": 54.05, "latency_mean_s": 118.62, "latency_std_s": 143.98, "latency_p95_s": 388.67, "non_live_ast_accuracy": 81.75, "non_live_simple_ast": 70.5, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 84.0, "non_live_parallel_multiple_ast": 80.5, "live_accuracy": 65.21, "live_simple_ast": 73.26, "live_multiple_ast": 63.53, "live_parallel_ast": 50.0, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 3.88, "multi_turn_base": 6.5, "multi_turn_miss_func": 2.0, "multi_turn_miss_param": 4.5, "multi_turn_long_context": 2.5, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 12.04, "memory_kv": 9.68, "memory_vector": 15.48, "memory_recursive_summarization": 10.97, "relevance_detection": 68.75, "irrelevance_detection": 72.84, "organization": "openbmb", "license": "Apache-2.0", "model_link": "https://huggingface.co/openbmb/MiniCPM3-4B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 87, "model": "Claude-Haiku-4-5-20251001 (Prompt)", "model_name": "Claude-Haiku-4-5-20251001 (Prompt)", "overall_accuracy": 25.26, "cost": 45.13, "latency_mean_s": 3.75, "latency_std_s": 19.96, "latency_p95_s": 3.77, "non_live_ast_accuracy": 55.42, "non_live_simple_ast": 55.67, "non_live_multiple_ast": 84.0, "non_live_parallel_ast": 38.0, "non_live_parallel_multiple_ast": 44.0, "live_accuracy": 52.48, "live_simple_ast": 66.67, "live_multiple_ast": 49.76, "live_parallel_ast": 56.25, "live_parallel_multiple_ast": 16.67, "multi_turn_accuracy": 1.75, "multi_turn_base": 1.5, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 4.0, "multi_turn_long_context": 1.5, "web_search_accuracy": 19.5, "web_search_base": 20.0, "web_search_no_snippet": 19.0, "memory_accuracy": 2.58, "memory_kv": 2.58, "memory_vector": 1.94, "memory_recursive_summarization": 3.23, "relevance_detection": 31.25, "irrelevance_detection": 95.29, "organization": "Anthropic", "license": "Proprietary", "model_link": "https://www.anthropic.com/news/claude-haiku-4-5" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 88, "model": "Amazon-Nova-Pro-v1:0 (FC)", "model_name": "Amazon-Nova-Pro-v1:0 (FC)", "overall_accuracy": 24.97, "cost": 48.44, "latency_mean_s": 2.25, "latency_std_s": 1.91, "latency_p95_s": 3.29, "non_live_ast_accuracy": 86.58, "non_live_simple_ast": 75.83, "non_live_multiple_ast": 93.0, "non_live_parallel_ast": 93.5, "non_live_parallel_multiple_ast": 84.0, "live_accuracy": 78.53, "live_simple_ast": 81.4, "live_multiple_ast": 77.97, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 70.83, "multi_turn_accuracy": 1.88, "multi_turn_base": 1.5, "multi_turn_miss_func": 0.5, "multi_turn_miss_param": 2.5, "multi_turn_long_context": 3.0, "web_search_accuracy": 2.5, "web_search_base": 4.0, "web_search_no_snippet": 1.0, "memory_accuracy": 1.94, "memory_kv": 2.58, "memory_vector": 1.29, "memory_recursive_summarization": 1.94, "relevance_detection": 93.75, "irrelevance_detection": 70.06, "organization": "Amazon", "license": "Proprietary", "model_link": "https://aws.amazon.com/cn/ai/generative-ai/nova/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 89, "model": "Claude-Sonnet-4-5-20250929 (Prompt)", "model_name": "Claude-Sonnet-4-5-20250929 (Prompt)", "overall_accuracy": 24.9, "cost": 47.82, "latency_mean_s": 3.84, "latency_std_s": 1.53, "latency_p95_s": 6.66, "non_live_ast_accuracy": 59.81, "non_live_simple_ast": 47.25, "non_live_multiple_ast": 79.5, "non_live_parallel_ast": 53.5, "non_live_parallel_multiple_ast": 59.0, "live_accuracy": 46.56, "live_simple_ast": 73.26, "live_multiple_ast": 40.17, "live_parallel_ast": 56.25, "live_parallel_multiple_ast": 33.33, "multi_turn_accuracy": 1.62, "multi_turn_base": 2.0, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 3.0, "multi_turn_long_context": 1.5, "web_search_accuracy": 16.0, "web_search_base": 16.0, "web_search_no_snippet": 16.0, "memory_accuracy": 5.38, "memory_kv": 4.52, "memory_vector": 9.68, "memory_recursive_summarization": 1.94, "relevance_detection": 37.5, "irrelevance_detection": 95.03, "organization": "Anthropic", "license": "Proprietary", "model_link": "https://www.anthropic.com/news/claude-sonnet-4-5" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 90, "model": "GPT-4.1-nano-2025-04-14 (Prompt)", "model_name": "GPT-4.1-nano-2025-04-14 (Prompt)", "overall_accuracy": 24.88, "cost": 7.42, "latency_mean_s": 1.02, "latency_std_s": 7.3, "latency_p95_s": 1.88, "non_live_ast_accuracy": 72.44, "non_live_simple_ast": 68.75, "non_live_multiple_ast": 63.5, "non_live_parallel_ast": 85.0, "non_live_parallel_multiple_ast": 72.5, "live_accuracy": 50.33, "live_simple_ast": 63.18, "live_multiple_ast": 46.53, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 54.17, "multi_turn_accuracy": 2.0, "multi_turn_base": 2.5, "multi_turn_miss_func": 1.0, "multi_turn_miss_param": 2.5, "multi_turn_long_context": 2.0, "web_search_accuracy": 1.5, "web_search_base": 2.0, "web_search_no_snippet": 1.0, "memory_accuracy": 16.77, "memory_kv": 9.03, "memory_vector": 14.19, "memory_recursive_summarization": 27.1, "relevance_detection": 68.75, "irrelevance_detection": 83.44, "organization": "OpenAI", "license": "Proprietary", "model_link": "https://openai.com/index/gpt-4-1/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 91, "model": "Falcon3-7B-Instruct (FC)", "model_name": "Falcon3-7B-Instruct (FC)", "overall_accuracy": 24.03, "cost": 73.61, "latency_mean_s": 93.11, "latency_std_s": 117.8, "latency_p95_s": 315.7, "non_live_ast_accuracy": 82.69, "non_live_simple_ast": 65.75, "non_live_multiple_ast": 89.0, "non_live_parallel_ast": 87.0, "non_live_parallel_multiple_ast": 89.0, "live_accuracy": 68.32, "live_simple_ast": 74.81, "live_multiple_ast": 66.76, "live_parallel_ast": 75.0, "live_parallel_multiple_ast": 62.5, "multi_turn_accuracy": 5.0, "multi_turn_base": 7.0, "multi_turn_miss_func": 4.0, "multi_turn_miss_param": 5.0, "multi_turn_long_context": 4.0, "web_search_accuracy": 0.5, "web_search_base": 1.0, "web_search_no_snippet": 0.0, "memory_accuracy": 20.65, "memory_kv": 10.32, "memory_vector": 12.9, "memory_recursive_summarization": 38.71, "relevance_detection": 100.0, "irrelevance_detection": 31.99, "organization": "TII UAE", "license": "falcon-llm-license", "model_link": "https://huggingface.co/tiiuae/Falcon3-7B-Instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 92, "model": "Qwen3-0.6B (FC)", "model_name": "Qwen3-0.6B (FC)", "overall_accuracy": 23.93, "cost": 0.46, "latency_mean_s": 0.68, "latency_std_s": 8.45, "latency_p95_s": 0.96, "non_live_ast_accuracy": 71.79, "non_live_simple_ast": 64.17, "non_live_multiple_ast": 86.0, "non_live_parallel_ast": 67.5, "non_live_parallel_multiple_ast": 69.5, "live_accuracy": 56.62, "live_simple_ast": 61.24, "live_multiple_ast": 56.13, "live_parallel_ast": 43.75, "live_parallel_multiple_ast": 37.5, "multi_turn_accuracy": 3.62, "multi_turn_base": 5.5, "multi_turn_miss_func": 2.0, "multi_turn_miss_param": 3.0, "multi_turn_long_context": 4.0, "web_search_accuracy": 1.0, "web_search_base": 1.0, "web_search_no_snippet": 1.0, "memory_accuracy": 8.6, "memory_kv": 2.58, "memory_vector": 1.94, "memory_recursive_summarization": 21.29, "relevance_detection": 75.0, "irrelevance_detection": 80.84, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-0.6B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 93, "model": "Granite-20b-FunctionCalling (FC)", "model_name": "Granite-20b-FunctionCalling (FC)", "overall_accuracy": 23.23, "cost": 5.23, "latency_mean_s": 3.2, "latency_std_s": 3.43, "latency_p95_s": 9.97, "non_live_ast_accuracy": 82.35, "non_live_simple_ast": 72.92, "non_live_multiple_ast": 91.5, "non_live_parallel_ast": 83.5, "non_live_parallel_multiple_ast": 81.5, "live_accuracy": 58.7, "live_simple_ast": 67.83, "live_multiple_ast": 56.7, "live_parallel_ast": 43.75, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 5.38, "multi_turn_base": 9.0, "multi_turn_miss_func": 3.0, "multi_turn_miss_param": 6.5, "multi_turn_long_context": 3.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 0.0, "memory_kv": 0.0, "memory_vector": 0.0, "memory_recursive_summarization": 0.0, "relevance_detection": 87.5, "irrelevance_detection": 75.13, "organization": "IBM", "license": "Apache-2.0", "model_link": "https://huggingface.co/ibm-granite/granite-20b-functioncalling" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 94, "model": "Qwen3-0.6B (Prompt)", "model_name": "Qwen3-0.6B (Prompt)", "overall_accuracy": 22.38, "cost": 3.65, "latency_mean_s": 3.1, "latency_std_s": 4.32, "latency_p95_s": 10.31, "non_live_ast_accuracy": 70.0, "non_live_simple_ast": 64.0, "non_live_multiple_ast": 78.0, "non_live_parallel_ast": 75.0, "non_live_parallel_multiple_ast": 63.0, "live_accuracy": 49.37, "live_simple_ast": 57.75, "live_multiple_ast": 47.77, "live_parallel_ast": 37.5, "live_parallel_multiple_ast": 37.5, "multi_turn_accuracy": 1.38, "multi_turn_base": 1.5, "multi_turn_miss_func": 1.5, "multi_turn_miss_param": 1.5, "multi_turn_long_context": 1.0, "web_search_accuracy": 0.5, "web_search_base": 1.0, "web_search_no_snippet": 0.0, "memory_accuracy": 8.39, "memory_kv": 1.29, "memory_vector": 2.58, "memory_recursive_summarization": 21.29, "relevance_detection": 75.0, "irrelevance_detection": 82.5, "organization": "Qwen", "license": "apache-2.0", "model_link": "https://huggingface.co/Qwen/Qwen3-0.6B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 95, "model": "Amazon-Nova-Micro-v1:0 (FC)", "model_name": "Amazon-Nova-Micro-v1:0 (FC)", "overall_accuracy": 22.29, "cost": 1.81, "latency_mean_s": 1.12, "latency_std_s": 0.45, "latency_p95_s": 1.79, "non_live_ast_accuracy": 74.1, "non_live_simple_ast": 70.92, "non_live_multiple_ast": 87.5, "non_live_parallel_ast": 75.5, "non_live_parallel_multiple_ast": 62.5, "live_accuracy": 66.32, "live_simple_ast": 72.09, "live_multiple_ast": 64.96, "live_parallel_ast": 81.25, "live_parallel_multiple_ast": 54.17, "multi_turn_accuracy": 1.38, "multi_turn_base": 1.5, "multi_turn_miss_func": 1.0, "multi_turn_miss_param": 2.0, "multi_turn_long_context": 1.0, "web_search_accuracy": 1.5, "web_search_base": 3.0, "web_search_no_snippet": 0.0, "memory_accuracy": 2.37, "memory_kv": 1.94, "memory_vector": 2.58, "memory_recursive_summarization": 2.58, "relevance_detection": 81.25, "irrelevance_detection": 70.65, "organization": "Amazon", "license": "Proprietary", "model_link": "https://aws.amazon.com/cn/ai/generative-ai/nova/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 96, "model": "RZN-T (Prompt)", "model_name": "RZN-T (Prompt)", "overall_accuracy": 22.25, "cost": 12.31, "latency_mean_s": 12.32, "latency_std_s": 27.53, "latency_p95_s": 39.84, "non_live_ast_accuracy": 67.94, "non_live_simple_ast": 63.75, "non_live_multiple_ast": 75.0, "non_live_parallel_ast": 69.5, "non_live_parallel_multiple_ast": 63.5, "live_accuracy": 49.74, "live_simple_ast": 61.24, "live_multiple_ast": 47.2, "live_parallel_ast": 43.75, "live_parallel_multiple_ast": 41.67, "multi_turn_accuracy": 2.88, "multi_turn_base": 4.5, "multi_turn_miss_func": 2.0, "multi_turn_miss_param": 2.5, "multi_turn_long_context": 2.5, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 6.88, "memory_kv": 9.03, "memory_vector": 6.45, "memory_recursive_summarization": 5.16, "relevance_detection": 81.25, "irrelevance_detection": 82.41, "organization": "Phronetic AI", "license": "apache-2.0", "model_link": "https://huggingface.co/phronetic-ai/RZN-T" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 97, "model": "MiniCPM3-4B (Prompt)", "model_name": "MiniCPM3-4B (Prompt)", "overall_accuracy": 22.08, "cost": 29.83, "latency_mean_s": 31.18, "latency_std_s": 35.61, "latency_p95_s": 102.02, "non_live_ast_accuracy": 70.54, "non_live_simple_ast": 66.17, "non_live_multiple_ast": 77.0, "non_live_parallel_ast": 70.0, "non_live_parallel_multiple_ast": 69.0, "live_accuracy": 43.15, "live_simple_ast": 47.67, "live_multiple_ast": 42.17, "live_parallel_ast": 43.75, "live_parallel_multiple_ast": 37.5, "multi_turn_accuracy": 3.5, "multi_turn_base": 4.5, "multi_turn_miss_func": 4.5, "multi_turn_miss_param": 2.0, "multi_turn_long_context": 3.0, "web_search_accuracy": 2.0, "web_search_base": 2.0, "web_search_no_snippet": 2.0, "memory_accuracy": 9.46, "memory_kv": 8.39, "memory_vector": 10.32, "memory_recursive_summarization": 9.68, "relevance_detection": 56.25, "irrelevance_detection": 73.71, "organization": "openbmb", "license": "Apache-2.0", "model_link": "https://huggingface.co/openbmb/MiniCPM3-4B" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 98, "model": "Llama-3.2-3B-Instruct (FC)", "model_name": "Llama-3.2-3B-Instruct (FC)", "overall_accuracy": 21.95, "cost": 6.2, "latency_mean_s": 6.1, "latency_std_s": 20.07, "latency_p95_s": 17.27, "non_live_ast_accuracy": 82.67, "non_live_simple_ast": 70.67, "non_live_multiple_ast": 92.5, "non_live_parallel_ast": 88.5, "non_live_parallel_multiple_ast": 79.0, "live_accuracy": 58.33, "live_simple_ast": 65.12, "live_multiple_ast": 57.64, "live_parallel_ast": 25.0, "live_parallel_multiple_ast": 37.5, "multi_turn_accuracy": 4.0, "multi_turn_base": 5.0, "multi_turn_miss_func": 3.5, "multi_turn_miss_param": 4.0, "multi_turn_long_context": 3.5, "web_search_accuracy": 1.0, "web_search_base": 2.0, "web_search_no_snippet": 0.0, "memory_accuracy": 6.24, "memory_kv": 3.23, "memory_vector": 3.23, "memory_recursive_summarization": 12.26, "relevance_detection": 87.5, "irrelevance_detection": 52.06, "organization": "Meta", "license": "Meta Llama 3 Community", "model_link": "https://llama.meta.com/llama3" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 99, "model": "Bielik-11B-v2.3-Instruct (Prompt)", "model_name": "Bielik-11B-v2.3-Instruct (Prompt)", "overall_accuracy": 21.9, "cost": 22.44, "latency_mean_s": 23.75, "latency_std_s": 61.76, "latency_p95_s": 72.8, "non_live_ast_accuracy": 81.5, "non_live_simple_ast": 73.0, "non_live_multiple_ast": 92.0, "non_live_parallel_ast": 85.5, "non_live_parallel_multiple_ast": 75.5, "live_accuracy": 67.8, "live_simple_ast": 75.58, "live_multiple_ast": 66.19, "live_parallel_ast": 62.5, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 2.62, "multi_turn_base": 4.5, "multi_turn_miss_func": 0.5, "multi_turn_miss_param": 3.0, "multi_turn_long_context": 2.5, "web_search_accuracy": 1.5, "web_search_base": 1.0, "web_search_no_snippet": 2.0, "memory_accuracy": 11.4, "memory_kv": 7.1, "memory_vector": 4.52, "memory_recursive_summarization": 22.58, "relevance_detection": 93.75, "irrelevance_detection": 36.01, "organization": "SpeakLeash & ACK Cyfronet AGH", "license": "Apache 2.0", "model_link": "https://huggingface.co/speakleash/Bielik-11B-v2.3-Instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 100, "model": "Hammer2.1-0.5b (FC)", "model_name": "Hammer2.1-0.5b (FC)", "overall_accuracy": 21.22, "cost": 2.82, "latency_mean_s": 2.79, "latency_std_s": 3.17, "latency_p95_s": 9.86, "non_live_ast_accuracy": 65.98, "non_live_simple_ast": 62.42, "non_live_multiple_ast": 81.0, "non_live_parallel_ast": 69.0, "non_live_parallel_multiple_ast": 51.5, "live_accuracy": 54.63, "live_simple_ast": 56.59, "live_multiple_ast": 54.42, "live_parallel_ast": 62.5, "live_parallel_multiple_ast": 37.5, "multi_turn_accuracy": 2.88, "multi_turn_base": 4.5, "multi_turn_miss_func": 0.5, "multi_turn_miss_param": 4.0, "multi_turn_long_context": 2.5, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 1.08, "memory_kv": 0.65, "memory_vector": 1.94, "memory_recursive_summarization": 0.65, "relevance_detection": 68.75, "irrelevance_detection": 80.79, "organization": "MadeAgents", "license": "cc-by-nc-4.0", "model_link": "https://huggingface.co/MadeAgents/Hammer2.1-0.5b" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 101, "model": "Gemma-3-4b-it (Prompt)", "model_name": "Gemma-3-4b-it (Prompt)", "overall_accuracy": 19.62, "cost": 4.14, "latency_mean_s": 4.69, "latency_std_s": 9.53, "latency_p95_s": 11.42, "non_live_ast_accuracy": 61.12, "non_live_simple_ast": 64.5, "non_live_multiple_ast": 88.0, "non_live_parallel_ast": 56.0, "non_live_parallel_multiple_ast": 36.0, "live_accuracy": 60.84, "live_simple_ast": 70.93, "live_multiple_ast": 59.35, "live_parallel_ast": 25.0, "live_parallel_multiple_ast": 41.67, "multi_turn_accuracy": 0.38, "multi_turn_base": 0.5, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 0.5, "multi_turn_long_context": 0.5, "web_search_accuracy": 1.0, "web_search_base": 1.0, "web_search_no_snippet": 1.0, "memory_accuracy": 8.6, "memory_kv": 9.68, "memory_vector": 9.68, "memory_recursive_summarization": 6.45, "relevance_detection": 100.0, "irrelevance_detection": 53.94, "organization": "Google", "license": "gemma-terms-of-use", "model_link": "https://blog.google/technology/developers/gemma-3/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 102, "model": "Open-Mistral-Nemo-2407 (Prompt)", "model_name": "Open-Mistral-Nemo-2407 (Prompt)", "overall_accuracy": 19.31, "cost": 13.8, "latency_mean_s": 0.84, "latency_std_s": 7.05, "latency_p95_s": 1.32, "non_live_ast_accuracy": 88.46, "non_live_simple_ast": 79.33, "non_live_multiple_ast": 92.5, "non_live_parallel_ast": 90.5, "non_live_parallel_multiple_ast": 91.5, "live_accuracy": 73.95, "live_simple_ast": 78.29, "live_multiple_ast": 73.03, "live_parallel_ast": 87.5, "live_parallel_multiple_ast": 58.33, "multi_turn_accuracy": 0.75, "multi_turn_base": 0.5, "multi_turn_miss_func": 1.0, "multi_turn_miss_param": 0.0, "multi_turn_long_context": 1.5, "web_search_accuracy": 2.5, "web_search_base": 3.0, "web_search_no_snippet": 2.0, "memory_accuracy": 8.6, "memory_kv": 9.68, "memory_vector": 9.68, "memory_recursive_summarization": 6.45, "relevance_detection": 93.75, "irrelevance_detection": 6.28, "organization": "Mistral AI", "license": "Proprietary", "model_link": "https://mistral.ai/news/mistral-nemo/" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 103, "model": "Granite-4.0-350m (FC)", "model_name": "Granite-4.0-350m (FC)", "overall_accuracy": 18.98, "cost": 1.44, "latency_mean_s": 1.74, "latency_std_s": 4.85, "latency_p95_s": 3.44, "non_live_ast_accuracy": 67.92, "non_live_simple_ast": 61.67, "non_live_multiple_ast": 84.5, "non_live_parallel_ast": 70.0, "non_live_parallel_multiple_ast": 55.5, "live_accuracy": 46.11, "live_simple_ast": 61.24, "live_multiple_ast": 42.36, "live_parallel_ast": 68.75, "live_parallel_multiple_ast": 33.33, "multi_turn_accuracy": 2.5, "multi_turn_base": 5.0, "multi_turn_miss_func": 0.5, "multi_turn_miss_param": 2.5, "multi_turn_long_context": 2.0, "web_search_accuracy": 0.5, "web_search_base": 0.0, "web_search_no_snippet": 1.0, "memory_accuracy": 3.23, "memory_kv": 1.94, "memory_vector": 1.29, "memory_recursive_summarization": 6.45, "relevance_detection": 81.25, "irrelevance_detection": 60.84, "organization": "IBM", "license": "Apache-2.0", "model_link": "https://huggingface.co/ibm-granite/granite-4.0-350m" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 104, "model": "Falcon3-3B-Instruct (FC)", "model_name": "Falcon3-3B-Instruct (FC)", "overall_accuracy": 16.25, "cost": 36.7, "latency_mean_s": 38.52, "latency_std_s": 107.47, "latency_p95_s": 103.62, "non_live_ast_accuracy": 54.62, "non_live_simple_ast": 56.5, "non_live_multiple_ast": 69.5, "non_live_parallel_ast": 67.0, "non_live_parallel_multiple_ast": 25.5, "live_accuracy": 54.48, "live_simple_ast": 57.36, "live_multiple_ast": 54.7, "live_parallel_ast": 25.0, "live_parallel_multiple_ast": 33.33, "multi_turn_accuracy": 1.0, "multi_turn_base": 1.5, "multi_turn_miss_func": 0.5, "multi_turn_miss_param": 0.5, "multi_turn_long_context": 1.5, "web_search_accuracy": 1.0, "web_search_base": 1.0, "web_search_no_snippet": 1.0, "memory_accuracy": 7.74, "memory_kv": 6.45, "memory_vector": 8.39, "memory_recursive_summarization": 8.39, "relevance_detection": 81.25, "irrelevance_detection": 32.92, "organization": "TII UAE", "license": "falcon-llm-license", "model_link": "https://huggingface.co/tiiuae/Falcon3-3B-Instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 105, "model": "Ministral-8B-Instruct-2410 (FC)", "model_name": "Ministral-8B-Instruct-2410 (FC)", "overall_accuracy": 11.1, "cost": 70.01, "latency_mean_s": 82.07, "latency_std_s": 212.99, "latency_p95_s": 568.59, "non_live_ast_accuracy": 0.0, "non_live_simple_ast": 0.0, "non_live_multiple_ast": 0.0, "non_live_parallel_ast": 0.0, "non_live_parallel_multiple_ast": 0.0, "live_accuracy": 0.0, "live_simple_ast": 0.0, "live_multiple_ast": 0.0, "live_parallel_ast": 0.0, "live_parallel_multiple_ast": 0.0, "multi_turn_accuracy": 0.0, "multi_turn_base": 0.0, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 0.0, "multi_turn_long_context": 0.0, "web_search_accuracy": 1.0, "web_search_base": 2.0, "web_search_no_snippet": 0.0, "memory_accuracy": 4.52, "memory_kv": 3.87, "memory_vector": 7.1, "memory_recursive_summarization": 2.58, "relevance_detection": 0.0, "irrelevance_detection": 100.0, "organization": "Mistral AI", "license": "Mistral AI Research License", "model_link": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 106, "model": "Falcon3-1B-Instruct (FC)", "model_name": "Falcon3-1B-Instruct (FC)", "overall_accuracy": 11.08, "cost": 1.72, "latency_mean_s": 5.23, "latency_std_s": 14.34, "latency_p95_s": 11.48, "non_live_ast_accuracy": 9.02, "non_live_simple_ast": 2.58, "non_live_multiple_ast": 6.0, "non_live_parallel_ast": 18.0, "non_live_parallel_multiple_ast": 9.5, "live_accuracy": 2.89, "live_simple_ast": 4.26, "live_multiple_ast": 2.37, "live_parallel_ast": 0.0, "live_parallel_multiple_ast": 12.5, "multi_turn_accuracy": 0.0, "multi_turn_base": 0.0, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 0.0, "multi_turn_long_context": 0.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 5.81, "memory_kv": 5.16, "memory_vector": 7.74, "memory_recursive_summarization": 4.52, "relevance_detection": 0.0, "irrelevance_detection": 87.3, "organization": "TII UAE", "license": "falcon-llm-license", "model_link": "https://huggingface.co/tiiuae/Falcon3-1B-Instruct" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 107, "model": "Llama-3.2-1B-Instruct (FC)", "model_name": "Llama-3.2-1B-Instruct (FC)", "overall_accuracy": 10.82, "cost": 1.64, "latency_mean_s": 3.21, "latency_std_s": 10.04, "latency_p95_s": 9.77, "non_live_ast_accuracy": 38.38, "non_live_simple_ast": 44.0, "non_live_multiple_ast": 50.5, "non_live_parallel_ast": 44.0, "non_live_parallel_multiple_ast": 15.0, "live_accuracy": 11.77, "live_simple_ast": 31.78, "live_multiple_ast": 7.31, "live_parallel_ast": 0.0, "live_parallel_multiple_ast": 0.0, "multi_turn_accuracy": 0.0, "multi_turn_base": 0.0, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 0.0, "multi_turn_long_context": 0.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 3.23, "memory_kv": 2.58, "memory_vector": 2.58, "memory_recursive_summarization": 4.52, "relevance_detection": 43.75, "irrelevance_detection": 51.57, "organization": "Meta", "license": "Meta Llama 3 Community", "model_link": "https://llama.meta.com/llama3" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 108, "model": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)", "model_name": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)", "overall_accuracy": 10.0, "cost": 0.72, "latency_mean_s": 1.42, "latency_std_s": 1.84, "latency_p95_s": 2.4, "non_live_ast_accuracy": 0.0, "non_live_simple_ast": 0.0, "non_live_multiple_ast": 0.0, "non_live_parallel_ast": 0.0, "non_live_parallel_multiple_ast": 0.0, "live_accuracy": 0.0, "live_simple_ast": 0.0, "live_multiple_ast": 0.0, "live_parallel_ast": 0.0, "live_parallel_multiple_ast": 0.0, "multi_turn_accuracy": 0.0, "multi_turn_base": 0.0, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 0.0, "multi_turn_long_context": 0.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 0.0, "memory_kv": 0.0, "memory_vector": 0.0, "memory_recursive_summarization": 0.0, "relevance_detection": 0.0, "irrelevance_detection": 100.0, "organization": "NVIDIA", "license": "nvidia-open-model-license", "model_link": "https://huggingface.co/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1" }, { "benchmark": "Berkeley Function Calling Leaderboard (BFCL)", "rank": 109, "model": "Gemma-3-1b-it (Prompt)", "model_name": "Gemma-3-1b-it (Prompt)", "overall_accuracy": 7.17, "cost": 3.4, "latency_mean_s": 3.98, "latency_std_s": 9.8, "latency_p95_s": 12.06, "non_live_ast_accuracy": 20.21, "non_live_simple_ast": 43.33, "non_live_multiple_ast": 36.0, "non_live_parallel_ast": 0.0, "non_live_parallel_multiple_ast": 1.5, "live_accuracy": 11.84, "live_simple_ast": 36.43, "live_multiple_ast": 6.27, "live_parallel_ast": 0.0, "live_parallel_multiple_ast": 0.0, "multi_turn_accuracy": 0.0, "multi_turn_base": 0.0, "multi_turn_miss_func": 0.0, "multi_turn_miss_param": 0.0, "multi_turn_long_context": 0.0, "web_search_accuracy": 0.0, "web_search_base": 0.0, "web_search_no_snippet": 0.0, "memory_accuracy": 3.23, "memory_kv": 3.87, "memory_vector": 3.87, "memory_recursive_summarization": 1.94, "relevance_detection": 37.5, "irrelevance_detection": 33.18, "organization": "Google", "license": "gemma-terms-of-use", "model_link": "https://blog.google/technology/developers/gemma-3/" }, { "benchmark": "LiveCodeBench", "model": "O4-Mini (High)", "model_name": "o4-mini__high", "overall_accuracy": 87.3, "pass_at_1": 87.3, "easy_accuracy": 98.45, "medium_accuracy": 92.69, "hard_accuracy": 71.14, "questions_evaluated": 1055, "model_link": "https://platform.openai.com/docs/api-reference/chat/create#chat-create-reasoning_effort", "organization": "OpenAIReason", "rank": 1 }, { "benchmark": "LiveCodeBench", "model": "O3 (High)", "model_name": "o3__high", "overall_accuracy": 84.74, "pass_at_1": 84.74, "easy_accuracy": 99.07, "medium_accuracy": 89.82, "hard_accuracy": 66.0, "questions_evaluated": 1055, "model_link": "https://platform.openai.com/docs/api-reference/chat/create#chat-create-reasoning_effort", "organization": "OpenAIReason", "rank": 2 }, { "benchmark": "LiveCodeBench", "model": "O4-Mini (Medium)", "model_name": "o4-mini__medium", "overall_accuracy": 84.45, "pass_at_1": 84.45, "easy_accuracy": 98.76, "medium_accuracy": 92.17, "hard_accuracy": 62.86, "questions_evaluated": 1055, "model_link": "https://platform.openai.com/docs/api-reference/chat/create#chat-create-reasoning_effort", "organization": "OpenAIReason", "rank": 3 }, { "benchmark": "LiveCodeBench", "model": "DeepSeek-R1-0528", "model_name": "deepseek-reasoner", "overall_accuracy": 84.36, "pass_at_1": 84.36, "easy_accuracy": 99.16, "medium_accuracy": 90.91, "hard_accuracy": 63.57, "questions_evaluated": 1055, "model_link": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528", "organization": "DeepSeekR1", "rank": 4 }, { "benchmark": "LiveCodeBench", "model": "Gemini-2.5-Pro-06-05", "model_name": "gemini-2.5-pro-preview-06-05", "overall_accuracy": 84.27, "pass_at_1": 84.27, "easy_accuracy": 99.07, "medium_accuracy": 92.17, "hard_accuracy": 62.0, "questions_evaluated": 1055, "model_link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/#advanced-coding", "organization": "GeminiThinking", "rank": 5 }, { "benchmark": "LiveCodeBench", "model": "Gemini-2.5-Pro-05-06", "model_name": "gemini-2.5-pro-preview-05-06", "overall_accuracy": 82.75, "pass_at_1": 82.75, "easy_accuracy": 98.76, "medium_accuracy": 90.6, "hard_accuracy": 59.43, "questions_evaluated": 1055, "model_link": "https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/#advanced-coding", "organization": "GeminiThinking", "rank": 6 }, { "benchmark": "LiveCodeBench", "model": "OpenReasoning-Nemotron-32B", "model_name": "nvidia/OpenReasoning-Nemotron-32B", "overall_accuracy": 80.96, "pass_at_1": 80.96, "easy_accuracy": 98.6, "medium_accuracy": 87.55, "hard_accuracy": 57.51, "questions_evaluated": 1055, "model_link": "https://huggingface.co/nvidia/OpenReasoning-Nemotron-32B", "organization": "DeepSeekR1", "rank": 7 }, { "benchmark": "LiveCodeBench", "model": "EXAONE-4.0-32B", "model_name": "EXAONE-4.0-32B", "overall_accuracy": 80.88, "pass_at_1": 80.88, "easy_accuracy": 98.84, "medium_accuracy": 88.25, "hard_accuracy": 56.29, "questions_evaluated": 1055, "model_link": "https://www.wenxiaobai.com/", "organization": "EXAONE", "rank": 8 }, { "benchmark": "LiveCodeBench", "model": "Qwen3-235B-A22B", "model_name": "Qwen/Qwen3-235B-A22B", "overall_accuracy": 80.38, "pass_at_1": 80.38, "easy_accuracy": 99.07, "medium_accuracy": 88.77, "hard_accuracy": 54.0, "questions_evaluated": 1055, "model_link": "https://huggingface.co/Qwen/Qwen3-235B-A22B", "organization": "CodeQwenInstruct", "rank": 9 }, { "benchmark": "LiveCodeBench", "model": "XBai-o4-medium", "model_name": "XBai o4-medium", "overall_accuracy": 80.09, "pass_at_1": 80.09, "easy_accuracy": 98.76, "medium_accuracy": 90.08, "hard_accuracy": 52.0, "questions_evaluated": 1055, "model_link": "https://www.wenxiaobai.com/", "organization": "XBai", "rank": 10 }, { "benchmark": "LiveCodeBench", "model": "OpenCodeReasoning-Nemotron-1.1-32B", "model_name": "nvidia/OpenCodeReasoning-Nemotron-1.1-32B", "overall_accuracy": 78.53, "pass_at_1": 78.53, "easy_accuracy": 98.01, "medium_accuracy": 85.69, "hard_accuracy": 52.77, "questions_evaluated": 1055, "model_link": "https://huggingface.co/nvidia/OpenCodeReasoning-Nemotron-1.1-32B", "organization": "DeepSeekR1", "rank": 11 }, { "benchmark": "LiveCodeBench", "model": "Grok-3-Mini (High)", "model_name": "grok-3-mini-beta_high", "overall_accuracy": 78.1, "pass_at_1": 78.1, "easy_accuracy": 98.14, "medium_accuracy": 85.9, "hard_accuracy": 51.14, "questions_evaluated": 1055, "model_link": "https://x.com/i/grok", "organization": "Grok", "rank": 12 }, { "benchmark": "LiveCodeBench", "model": "O3-Mini-2025-01-31 (High)", "model_name": "o3-mini-2025-01-31__high", "overall_accuracy": 77.73, "pass_at_1": 77.73, "easy_accuracy": 99.07, "medium_accuracy": 87.47, "hard_accuracy": 47.43, "questions_evaluated": 1055, "model_link": "https://platform.openai.com/docs/api-reference/chat/create#chat-create-reasoning_effort", "organization": "OpenAIReason", "rank": 13 }, { "benchmark": "LiveCodeBench", "model": "O4-Mini (Low)", "model_name": "o4-mini__low", "overall_accuracy": 77.44, "pass_at_1": 77.44, "easy_accuracy": 98.14, "medium_accuracy": 86.16, "hard_accuracy": 48.86, "questions_evaluated": 1055, "model_link": "https://platform.openai.com/docs/api-reference/chat/create#chat-create-reasoning_effort", "organization": "OpenAIReason", "rank": 14 }, { "benchmark": "LiveCodeBench", "model": "Gemini-2.5-Flash-05-20", "model_name": "gemini-2.5-flash-preview-05-20", "overall_accuracy": 76.21, "pass_at_1": 76.21, "easy_accuracy": 99.07, "medium_accuracy": 82.51, "hard_accuracy": 48.29, "questions_evaluated": 1055, "model_link": "https://developers.googleblog.com/en/start-building-with-gemini-25-flash/", "organization": "GeminiThinking", "rank": 15 }, { "benchmark": "LiveCodeBench", "model": "O3-Mini-2025-01-31 (Med)", "model_name": "o3-mini-2025-01-31__medium", "overall_accuracy": 75.36, "pass_at_1": 75.36, "easy_accuracy": 98.76, "medium_accuracy": 85.12, "hard_accuracy": 43.14, "questions_evaluated": 1055, "model_link": "https://platform.openai.com/docs/api-reference/chat/create#chat-create-reasoning_effort", "organization": "OpenAIReason", "rank": 16 }, { "benchmark": "LiveCodeBench", "model": "Gemini-2.5-Flash-04-17", "model_name": "gemini-2.5-flash-preview-04-17", "overall_accuracy": 75.07, "pass_at_1": 75.07, "easy_accuracy": 98.45, "medium_accuracy": 81.2, "hard_accuracy": 46.86, "questions_evaluated": 1055, "model_link": "https://developers.googleblog.com/en/start-building-with-gemini-25-flash/", "organization": "GeminiThinking", "rank": 17 }, { "benchmark": "LiveCodeBench", "model": "O3-Mini-2025-01-31 (Low)", "model_name": "o3-mini-2025-01-31__low", "overall_accuracy": 70.62, "pass_at_1": 70.62, "easy_accuracy": 97.83, "medium_accuracy": 77.55, "hard_accuracy": 38.0, "questions_evaluated": 1055, "model_link": "https://platform.openai.com/docs/api-reference/chat/create#chat-create-reasoning_effort", "organization": "OpenAIReason", "rank": 18 }, { "benchmark": "LiveCodeBench", "model": "Claude-Opus-4 (Thinking)", "model_name": "claude-opus-4-20250514", "overall_accuracy": 70.43, "pass_at_1": 70.43, "easy_accuracy": 99.07, "medium_accuracy": 80.42, "hard_accuracy": 33.14, "questions_evaluated": 1055, "model_link": "https://www.anthropic.com/claude/sonnet", "organization": "Claude3Thinking", "rank": 19 }, { "benchmark": "LiveCodeBench", "model": "Claude-Sonnet-4 (Thinking)", "model_name": "claude-sonnet-4-20250514", "overall_accuracy": 68.53, "pass_at_1": 68.53, "easy_accuracy": 98.76, "medium_accuracy": 75.98, "hard_accuracy": 32.57, "questions_evaluated": 1055, "model_link": "https://www.anthropic.com/claude/sonnet", "organization": "Claude3Thinking", "rank": 20 }, { "benchmark": "LiveCodeBench", "model": "Claude-Opus-4", "model_name": "claude-opus-4-20250514_nothink", "overall_accuracy": 62.37, "pass_at_1": 62.37, "easy_accuracy": 96.27, "medium_accuracy": 69.19, "hard_accuracy": 23.71, "questions_evaluated": 1055, "model_link": "https://www.anthropic.com/claude/sonnet", "organization": "Claude3Thinking", "rank": 21 }, { "benchmark": "LiveCodeBench", "model": "Claude-Sonnet-4", "model_name": "claude-sonnet-4-20250514_nothink", "overall_accuracy": 59.43, "pass_at_1": 59.43, "easy_accuracy": 96.27, "medium_accuracy": 63.97, "hard_accuracy": 20.57, "questions_evaluated": 1055, "model_link": "https://www.anthropic.com/claude/sonnet", "organization": "Claude3", "rank": 22 }, { "benchmark": "LiveCodeBench", "model": "DeepSeek-V3", "model_name": "deepseek-chat", "overall_accuracy": 49.55, "pass_at_1": 49.55, "easy_accuracy": 83.29, "medium_accuracy": 53.5, "hard_accuracy": 14.2, "questions_evaluated": 1055, "model_link": "https://huggingface.co/deepseek-ai/DeepSeek-V3", "organization": "DeepSeekAPI", "rank": 23 }, { "benchmark": "LiveCodeBench", "model": "Claude-3.5-Sonnet-20241022", "model_name": "claude-3-5-sonnet-20241022", "overall_accuracy": 48.74, "pass_at_1": 48.74, "easy_accuracy": 92.64, "medium_accuracy": 44.65, "hard_accuracy": 12.83, "questions_evaluated": 1055, "model_link": "https://www.anthropic.com/news/claude-3-5-sonnet", "organization": "Claude3", "rank": 24 }, { "benchmark": "LiveCodeBench", "model": "GPT-4O-2024-08-06", "model_name": "gpt-4o-2024-08-06", "overall_accuracy": 38.3, "pass_at_1": 38.3, "easy_accuracy": 82.52, "medium_accuracy": 32.06, "hard_accuracy": 4.46, "questions_evaluated": 1055, "model_link": "https://openai.com/index/spring-update", "organization": "OpenAIChat", "rank": 25 }, { "benchmark": "LiveCodeBench", "model": "GPT-4-Turbo-2024-04-09", "model_name": "gpt-4-turbo-2024-04-09", "overall_accuracy": 37.32, "pass_at_1": 37.32, "easy_accuracy": 81.4, "medium_accuracy": 30.7, "hard_accuracy": 4.0, "questions_evaluated": 1055, "model_link": "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4", "organization": "OpenAIChat", "rank": 26 }, { "benchmark": "LiveCodeBench", "model": "GPT-4O-mini-2024-07-18", "model_name": "gpt-4o-mini-2024-07-18", "overall_accuracy": 35.52, "pass_at_1": 35.52, "easy_accuracy": 81.71, "medium_accuracy": 25.17, "hard_accuracy": 4.34, "questions_evaluated": 1055, "model_link": "https://openai.com/index/spring-update", "organization": "OpenAIChat", "rank": 27 }, { "benchmark": "LiveCodeBench", "model": "Claude-3-Haiku", "model_name": "claude-3-haiku-20240307", "overall_accuracy": 22.53, "pass_at_1": 22.53, "easy_accuracy": 61.06, "medium_accuracy": 9.01, "hard_accuracy": 1.89, "questions_evaluated": 1055, "model_link": "https://www.anthropic.com/index/claude-3", "organization": "Claude3", "rank": 28 } ]