col_rank	col_model	col_org	col_score	col_elo	col_tokens
🥇	claude-3.7-sonnet (baseline)	Anthropic	50.0(-0.00, +0.00)	1114(-0.00, +0.00)	1094
🥈	deepseek-chat-v3-0324	Deepseek	49.73(-2.04, +2.04)	1112(-14.22, +14.19)	1085
🥉	gemini-2.0-pro-exp-02-05	Google	47.14(-2.65, +3.45)	1094(-18.52, +23.98)	2372
4	gemini-2.5-pro-exp-03-25	Google	47.06(-2.84, +2.83)	1094(-19.83, +19.74)	2577
5	chatgpt-4o-latest(2025-03-26)	OpenAI	46.37(-1.83, +2.19)	1089(-12.81, +15.21)	1111
6	deepseek-r1	Deepseek	44.26(-2.70, +2.14)	1074(-19.21, +14.95)	1211
7	o1-high	OpenAI	42.54(-2.26, +1.96)	1062(-16.20, +13.83)	1503
8	o3-mini-high	OpenAI	41.64(-2.20, +2.17)	1055(-15.84, +15.43)	1257
9	o1-medium	OpenAI	40.7(-2.08, +2.18)	1049(-15.10, +15.57)	1487
10	gpt-4.5-preview	OpenAI	40.64(-2.00, +2.43)	1048(-14.50, +17.39)	1040
11	claude-3.5-sonnet	Anthropic	37.06(-2.83, +2.60)	1022(-21.42, +19.11)	682
12	o1-low	OpenAI	36.9(-1.84, +1.92)	1021(-13.89, +14.19)	1513
13	o3-mini-medium	OpenAI	34.86(-2.30, +1.86)	1005(-17.94, +14.04)	1221
14	command-a	Cohere	34.53(-2.39, +1.73)	1003(-18.67, +13.14)	1083
15	gemini-2.0-flash-thinking-exp-01-21	Google	32.09(-2.33, +2.59)	984(-18.88, +20.26)	2488
16	o3-mini-low	OpenAI	31.98(-2.01, +1.68)	983(-16.32, +13.21)	1205
17	gpt-4o-2024-11-20	OpenAI	30.74(-1.51, +2.02)	973(-12.45, +16.25)	1142
18	claude-3.5-haiku	Anthropic	28.54(-2.68, +2.84)	955(-23.47, +23.54)	601
19	gemini-2.0-flash-001 (judge)	Google	27.4(-1.79, +2.23)	945(-15.97, +19.00)	1901
20	gemma-3-27b-it	Google	26.33(-2.03, +1.77)	935(-18.68, +15.51)	1654
21	gemini-2.0-flash-lite-001	Google	24.7(-2.11, +1.41)	920(-20.32, +12.92)	2196
22	deepseek-v3	Deepseek	23.75(-2.00, +1.75)	911(-19.80, +16.38)	1007
23	claude-3.5-sonnet-20240620	Anthropic	23.38(-1.95, +2.27)	908(-19.49, +21.37)	628
24	jamba-1.6-large	AI21	17.63(-1.54, +1.84)	846(-19.10, +21.16)	859
25	grok-2-1212	xAI	17.08(-1.78, +1.79)	840(-22.85, +21.15)	898
26	minimax-01	MiniMax	16.76(-2.08, +2.50)	836(-27.33, +29.42)	370
27	gpt-4o-mini (judge)	OpenAI	16.68(-1.44, +1.55)	835(-18.75, +18.65)	830
28	nova-pro-v1	Amazon	16.34(-1.26, +1.32)	830(-16.58, +16.32)	905
29	qwen-2.5-72b-instruct	Alibaba	16.13(-1.77, +1.40)	828(-23.80, +17.47)	1097
30	mistral-large-2411	Mistral	14.6(-1.34, +1.25)	807(-19.40, +16.85)	906
31	gemma-2-27b-it	Google	12.91(-1.20, +1.45)	782(-19.30, +21.38)	794
32	gpt-4-1106-preview	OpenAI	12.54(-1.06, +1.36)	777(-17.48, +20.62)	840
33	mistral-small-3.1-24b-instruct-2503	Mistral	10.74(-1.04, +1.31)	746(-19.64, +22.54)	961
34	command-r-plus-08-2024	Cohere	10.11(-0.96, +1.04)	735(-19.31, +19.00)	969
35	nova-lite-v1	Amazon	9.72(-1.08, +1.02)	727(-22.57, +19.22)	994
36	wizardlm-2-8x22b	Microsoft	9.49(-1.11, +0.97)	722(-23.82, +18.60)	1028
37	lfm-7b	Liquid AI	9.27(-1.07, +1.05)	718(-23.20, +20.70)	1011
38	qwen2.5-32b-instruct	Alibaba	8.85(-1.08, +1.50)	709(-24.68, +30.05)	795
39	mistral-small-24b-instruct-2501	Mistral	8.59(-1.16, +1.14)	703(-27.51, +23.73)	998
40	hermes-3-llama-3.1-405b	NousResearch	8.26(-1.25, +1.38)	696(-30.94, +29.40)	771
41	hermes-3-llama-3.1-70b	NousResearch	7.73(-1.08, +1.13)	683(-28.11, +25.90)	771
42	gemma-2-9b-it	Google	7.67(-1.16, +1.03)	682(-30.54, +23.86)	751
43	nova-micro-v1	Amazon	6.62(-0.96, +0.88)	654(-29.20, +23.30)	927
44	command-r-08-2024	Cohere	6.52(-0.90, +1.13)	651(-27.34, +30.02)	796
45	lfm-40b	Liquid AI	5.76(-0.87, +0.91)	628(-29.75, +27.41)	863
46	llama-3.3-70b-instruct	Meta	5.34(-1.02, +0.98)	615(-38.51, +31.04)	809
47	command-r7b-12-2024	Cohere	4.95(-0.76, +0.89)	601(-30.30, +30.26)	882
48	lfm-3b	Liquid AI	3.57(-0.55, +0.62)	542(-30.03, +28.64)	776
49	qwen-2.5-7b-instruct	Alibaba	3.57(-0.68, +0.86)	542(-38.11, +38.91)	844
50	olmo-2-0325-32b-instruct	Allen AI	3.31(-0.49, +0.62)	528(-28.93, +30.56)	890
51	jamba-1.6-mini	AI21	2.56(-0.51, +0.62)	482(-39.77, +38.96)	920
52	llama-3.1-nemotron-70b-instruct	Meta	2.29(-0.31, +0.42)	462(-25.83, +30.06)	1854
53	llama-3.1-405b-instruct	Meta	0.86(-0.19, +0.24)	289(-44.54, +42.37)	1735
54	llama-3.1-70b-instruct	Meta	0.52(-0.12, +0.13)	200(-46.44, +40.36)	1923
55	llama-3.1-8b-instruct	Meta	0.07(-0.02, +0.02)	-141(-54.75, +41.44)	5081

col_rank	col_model	col_org	col_score	col_elo	col_tokens
🥇	gemini-2.5-pro-exp-03-25	Google	79.56(-1.79, +2.07)	1350(-18.55, +22.99)	2577
🥈	gemini-2.0-pro-exp-02-05	Google	73.27(-1.88, +1.95)	1289(-16.27, +17.79)	2372
🥉	gemini-2.0-flash-thinking-exp-01-21	Google	65.62(-2.14, +1.97)	1226(-16.25, +15.43)	2488
4	o1-high	OpenAI	61.13(-2.32, +2.40)	1193(-16.76, +17.80)	1503
5	o1-medium	OpenAI	57.25(-1.95, +2.22)	1165(-13.78, +15.86)	1487
6	o1-low	OpenAI	54.78(-2.35, +2.32)	1147(-16.43, +16.38)	1513
7	o3-mini-high	OpenAI	54.21(-2.34, +2.67)	1143(-16.30, +18.79)	1257
8	gemini-2.0-flash-lite-001	Google	53.25(-2.10, +2.09)	1137(-14.61, +14.64)	2196
9	gemini-2.0-flash-001 (judge)	Google	53.23(-2.76, +1.90)	1136(-19.22, +13.30)	1901
10	chatgpt-4o-latest(2025-03-26)	OpenAI	50.78(-2.74, +2.35)	1119(-19.02, +16.37)	1111
11	claude-3.7-sonnet (baseline)	Anthropic	50.0(-0.00, +0.00)	1114(-0.00, +0.00)	1094
12	deepseek-chat-v3-0324	Deepseek	47.52(-1.97, +2.39)	1097(-13.76, +16.58)	1085
13	gemma-3-27b-it	Google	47.21(-2.73, +2.28)	1095(-19.13, +15.90)	1654
14	o3-mini-medium	OpenAI	46.77(-2.83, +2.11)	1092(-19.84, +14.69)	1221
15	o3-mini-low	OpenAI	43.59(-2.38, +2.15)	1069(-16.87, +15.13)	1205
16	deepseek-r1	Deepseek	42.23(-1.48, +2.25)	1060(-10.64, +15.87)	1211
17	gpt-4.5-preview	OpenAI	39.02(-2.11, +2.47)	1036(-15.58, +17.84)	1040
18	command-a	Cohere	35.36(-1.91, +1.89)	1009(-14.72, +14.18)	1083
19	gpt-4o-2024-11-20	OpenAI	32.67(-1.90, +2.06)	988(-15.24, +16.03)	1142
20	claude-3.5-sonnet	Anthropic	24.71(-1.38, +2.18)	920(-13.16, +19.75)	682
21	deepseek-v3	Deepseek	23.92(-1.56, +1.76)	913(-15.18, +16.45)	1007
22	qwen-2.5-72b-instruct	Alibaba	19.62(-1.53, +1.43)	869(-17.33, +15.41)	1097
23	wizardlm-2-8x22b	Microsoft	18.06(-1.32, +1.27)	851(-16.01, +14.55)	1028
24	nova-pro-v1	Amazon	18.04(-1.05, +1.77)	851(-12.73, +20.05)	905
25	grok-2-1212	xAI	17.66(-1.28, +1.49)	847(-15.80, +17.28)	898
26	gpt-4-1106-preview	OpenAI	16.71(-1.24, +1.87)	835(-16.01, +22.30)	840
27	mistral-large-2411	Mistral	15.99(-1.48, +1.06)	826(-19.95, +13.42)	906
28	claude-3.5-haiku	Anthropic	15.89(-1.75, +1.10)	824(-23.86, +13.98)	601
29	claude-3.5-sonnet-20240620	Anthropic	15.42(-1.47, +1.23)	818(-20.33, +15.93)	628
30	command-r-plus-08-2024	Cohere	14.28(-1.81, +1.34)	803(-27.13, +18.32)	969
31	gpt-4o-mini (judge)	OpenAI	14.24(-1.16, +1.39)	802(-17.17, +18.93)	830
32	mistral-small-3.1-24b-instruct-2503	Mistral	14.05(-1.41, +1.09)	799(-21.22, +15.25)	961
33	lfm-7b	Liquid AI	13.69(-1.09, +1.18)	794(-16.66, +16.73)	1011
34	jamba-1.6-large	AI21	13.45(-0.99, +1.10)	791(-15.27, +15.98)	859
35	nova-lite-v1	Amazon	13.3(-1.56, +1.28)	788(-24.86, +18.46)	994
36	mistral-small-24b-instruct-2501	Mistral	10.57(-1.27, +1.17)	743(-24.66, +20.44)	998
37	gemma-2-27b-it	Google	10.39(-1.14, +1.32)	740(-22.36, +23.46)	794
38	qwen2.5-32b-instruct	Alibaba	9.95(-1.03, +1.16)	731(-21.04, +21.29)	795
39	nova-micro-v1	Amazon	9.41(-1.02, +1.10)	721(-21.87, +21.41)	927
40	lfm-40b	Liquid AI	8.95(-1.24, +0.87)	711(-28.32, +17.73)	863

col_rank	col_model	col_org	col_score	col_elo	col_tokens
🥇	deepseek-chat-v3-0324	Deepseek	53.82(-2.32, +2.26)	1141(-16.15, +15.89)	1085
🥈	claude-3.7-sonnet (baseline)	Anthropic	50.0(-0.00, +0.00)	1114(-0.00, +0.00)	1094
🥉	deepseek-r1	Deepseek	47.13(-2.59, +2.28)	1094(-18.10, +15.87)	1211
4	gemini-2.5-pro-exp-03-25	Google	45.26(-2.63, +2.89)	1081(-18.54, +20.20)	2577
5	gemini-2.0-pro-exp-02-05	Google	45.26(-2.80, +3.63)	1081(-19.75, +25.34)	2372
6	gpt-4.5-preview	OpenAI	43.02(-2.26, +2.71)	1065(-16.08, +19.10)	1040
7	chatgpt-4o-latest(2025-03-26)	OpenAI	41.81(-2.27, +1.95)	1057(-16.37, +13.83)	1111
8	o1-high	OpenAI	41.5(-2.54, +2.37)	1054(-18.32, +16.84)	1503
9	o3-mini-high	OpenAI	40.26(-2.31, +2.70)	1045(-16.87, +19.30)	1257
10	o1-medium	OpenAI	39.24(-2.44, +2.42)	1038(-18.02, +17.44)	1487
11	command-a	Cohere	38.0(-2.02, +1.72)	1029(-15.02, +12.57)	1083
12	o1-low	OpenAI	35.04(-2.48, +1.95)	1007(-19.20, +14.77)	1513
13	gpt-4o-2024-11-20	OpenAI	34.7(-1.89, +1.88)	1004(-14.66, +14.31)	1142
14	o3-mini-medium	OpenAI	33.3(-1.73, +2.58)	993(-13.72, +19.77)	1221
15	claude-3.5-sonnet	Anthropic	32.22(-1.99, +2.45)	985(-16.04, +19.17)	682
16	gemini-2.0-flash-thinking-exp-01-21	Google	30.55(-2.11, +2.60)	971(-17.57, +20.81)	2488
17	o3-mini-low	OpenAI	30.36(-2.36, +2.24)	970(-19.85, +17.99)	1205
18	deepseek-v3	Deepseek	27.37(-1.96, +2.08)	944(-17.51, +17.80)	1007
19	gemini-2.0-flash-001 (judge)	Google	25.85(-1.47, +2.32)	931(-13.54, +20.46)	1901
20	gemma-3-27b-it	Google	25.16(-1.46, +2.41)	925(-13.69, +21.64)	1654
21	claude-3.5-haiku	Anthropic	24.91(-2.60, +1.80)	922(-25.03, +16.30)	601
22	gemini-2.0-flash-lite-001	Google	23.08(-1.67, +2.12)	905(-16.82, +20.08)	2196
23	jamba-1.6-large	AI21	20.52(-1.49, +2.03)	879(-16.36, +20.88)	859
24	claude-3.5-sonnet-20240620	Anthropic	20.39(-2.04, +1.76)	877(-22.71, +18.26)	628
25	minimax-01	MiniMax	19.27(-2.71, +2.65)	865(-32.07, +28.19)	370
26	nova-pro-v1	Amazon	18.1(-1.54, +1.97)	852(-18.74, +22.18)	905
27	grok-2-1212	xAI	17.94(-1.57, +1.66)	850(-19.10, +18.98)	898
28	gpt-4o-mini (judge)	OpenAI	17.87(-1.78, +1.85)	849(-22.00, +21.00)	830
29	qwen-2.5-72b-instruct	Alibaba	17.41(-1.50, +1.50)	844(-18.82, +17.53)	1097
30	mistral-large-2411	Mistral	15.04(-1.29, +1.48)	813(-18.26, +19.34)	906
31	gemma-2-27b-it	Google	12.7(-1.29, +1.45)	779(-21.17, +21.62)	794
32	gpt-4-1106-preview	OpenAI	11.87(-1.00, +1.43)	766(-17.25, +22.50)	840
33	mistral-small-3.1-24b-instruct-2503	Mistral	11.48(-1.40, +1.18)	759(-25.22, +19.29)	961
34	nova-lite-v1	Amazon	10.67(-1.11, +1.13)	745(-21.17, +19.67)	994
35	lfm-7b	Liquid AI	10.49(-1.23, +1.06)	742(-24.01, +18.78)	1011
36	command-r-plus-08-2024	Cohere	9.56(-0.99, +1.24)	724(-20.93, +23.43)	969
37	wizardlm-2-8x22b	Microsoft	9.27(-1.37, +1.14)	718(-30.35, +22.37)	1028
38	qwen2.5-32b-instruct	Alibaba	9.01(-1.34, +1.18)	712(-30.36, +23.64)	795
39	mistral-small-24b-instruct-2501	Mistral	8.93(-1.13, +1.12)	711(-25.66, +22.75)	998
40	hermes-3-llama-3.1-405b	NousResearch	8.12(-1.09, +1.46)	693(-27.31, +31.49)	771

col_rank	col_model	col_org	col_score	col_elo	col_tokens
🥇	gemini-2.5-pro-exp-03-25	Google	82.94(-0.81, +1.28)	1275(-19.18, +17.08)	2577
🥈	gemini-2.0-pro-exp-02-05	Google	76.12(-1.20, +1.13)	1201(-20.68, +11.84)	2372
🥉	gemini-2.0-flash-thinking-exp-01-21	Google	70.4(-1.89, +1.46)	1151(-24.86, +13.18)	2488
4	deepseek-chat-v3-0324	Deepseek	60.52(-0.98, +1.43)	1074(-16.44, +11.33)	1085
5	gemini-2.0-flash-lite-001	Google	58.22(-1.09, +1.54)	1058(-17.14, +11.90)	2196
6	gemini-2.0-flash-001 (judge)	Google	57.8(-0.87, +1.44)	1055(-15.53, +11.18)	1901
7	o1-high	OpenAI	54.86(-1.35, +1.05)	1034(-18.84, +8.24)	1503
8	gemma-3-27b-it	Google	52.54(-0.87, +1.47)	1018(-15.41, +11.13)	1654
9	deepseek-r1	Deepseek	52.44(-1.64, +1.27)	1017(-20.83, +9.66)	1211
10	o1-medium	OpenAI	51.13(-1.80, +1.31)	1008(-21.91, +9.97)	1487
11	claude-3.7-sonnet (baseline)	Anthropic	50.0(-0.00, +0.00)	1000(-9.37, +0.85)	1094
12	o3-mini-high	OpenAI	49.04(-1.34, +1.68)	993(-18.69, +12.52)	1257
13	o1-low	OpenAI	48.98(-1.56, +2.43)	993(-20.17, +17.79)	1513
14	chatgpt-4o-latest(2025-03-26)	OpenAI	46.62(-1.71, +1.47)	976(-21.36, +11.10)	1111
15	command-a	Cohere	45.82(-2.01, +1.24)	971(-23.48, +9.53)	1083
16	gpt-4.5-preview	OpenAI	45.27(-1.59, +1.48)	967(-20.55, +11.20)	1040
17	gpt-4o-2024-11-20	OpenAI	45.03(-1.38, +1.57)	965(-19.10, +11.84)	1142
18	o3-mini-medium	OpenAI	41.94(-1.22, +1.69)	943(-18.07, +12.83)	1221
19	o3-mini-low	OpenAI	39.2(-1.27, +1.25)	924(-18.65, +9.95)	1205
20	deepseek-v3	Deepseek	32.93(-1.48, +1.21)	876(-21.14, +10.34)	1007
21	qwen-2.5-72b-instruct	Alibaba	24.0(-1.74, +1.37)	800(-26.32, +13.66)	1097
22	nova-pro-v1	Amazon	21.08(-1.37, +1.75)	771(-24.01, +18.64)	905
23	claude-3.5-sonnet	Anthropic	20.6(-1.04, +1.50)	766(-20.55, +16.40)	682
24	jamba-1.6-large	AI21	20.35(-1.00, +1.42)	763(-20.24, +15.73)	859
25	grok-2-1212	xAI	18.05(-1.27, +1.19)	737(-24.78, +14.45)	898
26	mistral-large-2411	Mistral	16.83(-1.55, +1.46)	723(-29.41, +18.36)	906
27	lfm-7b	Liquid AI	16.44(-1.16, +0.70)	718(-24.42, +9.54)	1011
28	gpt-4o-mini (judge)	OpenAI	16.07(-1.28, +1.00)	713(-26.33, +13.39)	830
29	wizardlm-2-8x22b	Microsoft	15.74(-1.22, +1.12)	709(-25.98, +15.03)	1028
30	mistral-small-3.1-24b-instruct-2503	Mistral	15.24(-1.19, +1.52)	702(-25.96, +20.46)	961
31	nova-lite-v1	Amazon	15.04(-1.30, +1.08)	699(-27.64, +15.18)	994
32	gpt-4-1106-preview	OpenAI	13.32(-1.09, +0.90)	675(-26.35, +14.02)	840
33	llama-3.1-nemotron-70b-instruct	Meta	12.61(-0.90, +0.90)	664(-24.09, +14.54)	1854
34	claude-3.5-haiku	Anthropic	12.58(-1.04, +1.10)	663(-26.42, +17.71)	601
35	claude-3.5-sonnet-20240620	Anthropic	12.18(-0.85, +0.95)	657(-23.56, +15.81)	628
36	command-r-plus-08-2024	Cohere	11.93(-0.92, +1.30)	653(-25.16, +21.33)	969
37	gemma-2-27b-it	Google	11.65(-1.07, +0.77)	648(-28.25, +13.51)	794
38	mistral-small-24b-instruct-2501	Mistral	11.21(-1.13, +1.12)	640(-30.01, +19.68)	998
39	nova-micro-v1	Amazon	10.14(-0.94, +1.34)	621(-28.24, +25.01)	927
40	qwen2.5-32b-instruct	Alibaba	8.12(-0.82, +0.88)	578(-29.41, +20.43)	795