Fix bug in NE calculation for Neuroparl-ST

analysis/neuroparl_st/combine_csv.py

@@ -28,6 +28,7 @@
     "phi4multimodal",
     "voxtral-small-24b",
     "spirelm",
+    "gemini-2.5-flash"
 ]
 SYSTEM_ORDER = [system.replace("_", " ") for system in SYSTEM_ORDER]
 

analysis/neuroparl_st/combined.csv

@@ -1,23 +1,24 @@
 system,accuracy_ne,,,accuracy_term,,
 ,en-es,en-fr,en-it,en-es,en-fr,en-it
 whisper,0.0,0.0,0.0,0.0,0.0,0.0
-seamlessm4t,65.62881562881563,59.53905249679897,58.663028649386085,75.69496619083395,72.75195113674924,65.21344232515894
-canary-v2,70.94017094017094,67.41357234314981,64.12005457025921,80.954169797145,82.1174075330845,77.29336966394187
+seamlessm4t,65.56776556776556,59.53905249679897,58.663028649386085,75.6198347107438,72.71801832371904,65.12261580381471
+canary-v2,69.59706959706959,66.38924455825864,62.82401091405184,80.24042073628851,81.09942314217848,76.43051771117166
 owsm4.0-ctc,46.52014652014652,42.18950064020486,40.586630286493865,69.98497370398196,67.119104173736,56.902815622161675
-aya whisper,25.274725274725274,5.569782330345711,23.124147339699864,33.47107438016529,6.820495419070241,29.70027247956403
-gemma whisper,4.700854700854701,7.490396927016646,3.0013642564802185,5.447032306536439,9.161859518154055,2.7702089009990916
-tower whisper,69.59706959706959,68.75800256081946,63.915416098226466,82.1938392186326,82.7621309806583,78.24704813805631
-aya seamlessm4t,70.51282051282051,7.298335467349553,48.908594815825374,81.36739293764087,7.4652188666440455,62.44323342415985
-gemma seamlessm4t,14.041514041514041,23.367477592829704,12.346521145975444,16.641622839969948,29.250084832032574,15.349682107175294
-tower seamlessm4t,71.30647130647131,67.9257362355954,63.64256480218281,82.64462809917356,82.32100441126569,79.01907356948229
-aya canary-v2,57.87545787545788,66.19718309859155,53.883495145631066,68.63260706235913,81.13335595520869,68.65671641791045
-gemma canary-v2,16.483516483516482,15.108834827144687,3.819918144611187,17.731029301277236,18.08618934509671,3.451407811080836
-tower canary-v2,71.73382173382173,69.01408450704226,64.66575716234652,83.20811419984973,83.10145911096029,79.01907356948229
-aya owsm4.0-ctc,52.62515262515263,6.46606914212548,24.351978171896317,65.81517655897822,7.261621988462844,31.92552225249773
-gemma owsm4.0-ctc,4.151404151404151,4.673495518565941,10.845839017735335,4.808414725770098,5.395317271801833,14.441416893732969
-tower owsm4.0-ctc,54.21245421245421,51.08834827144686,47.0668485675307,66.190833959429,66.1689854088904,61.98910081743869
-desta2-8b,8.302808302808302,7.490396927016646,1.6371077762619373,10.668670172802404,9.399389209365456,1.0445049954586738
-qwen2audio-7b,5.372405372405373,10.371318822023047,8.935879945429742,6.799398948159278,12.487275195113675,8.810172570390554
-phi4multimodal,62.698412698412696,45.26248399487836,55.04774897680764,71.29977460555973,55.34441805225653,69.70935513169846
-voxtral-small-24b,69.78021978021978,66.90140845070422,64.12005457025921,79.60180315552216,82.4567356633865,76.93006357856494
-spirelm,68.92551892551893,65.55697823303457,65.00682128240109,77.98647633358378,76.34882931795046,73.56948228882834
+aya whisper,67.70451770451771,66.64532650448145,60.91405184174625,80.80390683696469,80.01357312521208,76.70299727520435
+gemma whisper,67.64346764346764,65.68501920614597,59.276944065484315,78.43726521412472,77.29894808279606,73.97820163487738
+tower whisper,68.98656898656898,68.24583866837388,63.43792633015006,81.59278737791135,82.04954190702409,77.52043596730245
+aya seamlessm4t,70.51282051282051,65.94110115236876,61.45975443383356,81.32982719759579,79.94570749915168,78.42870118074478
+gemma seamlessm4t,70.2075702075702,64.9807938540333,59.754433833560704,78.88805409466566,76.72209026128266,76.20345140781109
+tower seamlessm4t,71.18437118437119,67.9257362355954,63.5743519781719,82.60706235912848,82.2531387852053,78.88283378746594
+aya canary-v2,70.08547008547008,66.19718309859155,61.174551386623165,82.00601051840721,81.06549032914828,77.45415318230853
+gemma canary-v2,68.98656898656898,64.9167733674776,59.95907230559345,79.97746055597295,78.07940278249067,75.4314259763851
+tower canary-v2,70.94017094017094,68.24583866837388,63.915416098226466,82.71975957926371,82.32100441126569,78.33787465940054
+aya owsm4.0-ctc,66.42246642246641,63.70038412291933,58.2537517053206,81.17956423741548,80.04750593824228,77.29336966394187
+gemma owsm4.0-ctc,65.56776556776556,62.29193341869398,57.094133697135064,78.70022539444027,77.06141839158465,75.29518619436875
+tower owsm4.0-ctc,67.76556776556777,65.87708066581305,61.59618008185539,81.59278737791135,82.32100441126569,78.06539509536785
+desta2-8b,45.78754578754579,43.9820742637644,41.20054570259209,54.28249436513899,53.919239904988125,48.09264305177112
+qwen2audio-7b,65.44566544566544,62.355953905249685,60.16371077762619,74.07963936889557,77.12928401764506,68.7102633969119
+phi4multimodal,62.637362637362635,45.26248399487836,55.04774897680764,71.29977460555973,55.34441805225653,69.70935513169846
+voxtral-small-24b,69.71916971916971,66.90140845070422,64.12005457025921,79.60180315552216,82.4567356633865,76.83923705722071
+spirelm,68.92551892551893,65.55697823303457,65.00682128240109,77.94891059353868,76.34882931795046,73.433242506812
+gemini-2.5-flash,67.76556776556777,67.09346991037131,62.68758526603001,82.49436513899325,82.42280285035629,79.51861943687557

analysis/neuroparl_st/neuroparl-st.tex

@@ -0,0 +1,26 @@
+\begin{tabular}{lrrrrrrrr} \toprule
+ & \multicolumn{4}{c}{\bf \neacc} & \multicolumn{4}{c}{\bf \termacc} \\
+ & en-es & en-fr & en-it &  & en-es & en-fr & en-it &  \\
+\midrule
+\cellcolor{sfmcolor} \whisper & - & - & - &  & - & - & - &  \\
+\cellcolor{sfmcolor} \seamless & \cellcolor{Chartreuse3!77} 65.6 & \cellcolor{Chartreuse3!70} 59.5 & \cellcolor{Chartreuse3!69} 58.7 &  & \cellcolor{DarkSlateGray3!89} 75.6 & \cellcolor{DarkSlateGray3!86} 72.7 & \cellcolor{DarkSlateGray3!77} 65.1 &  \\
+\cellcolor{sfmcolor} \canary & \cellcolor{Chartreuse3!82} 69.6 & \cellcolor{Chartreuse3!78} 66.4 & \cellcolor{Chartreuse3!74} 62.8 &  & \cellcolor{DarkSlateGray3!94} 80.2 & \cellcolor{DarkSlateGray3!95} 81.1 & \cellcolor{DarkSlateGray3!90} 76.4 &  \\
+\cellcolor{sfmcolor} \owsm & \cellcolor{Chartreuse3!55} 46.5 & \cellcolor{Chartreuse3!50} 42.2 & \cellcolor{Chartreuse3!48} 40.6 &  & \cellcolor{DarkSlateGray3!82} 70.0 & \cellcolor{DarkSlateGray3!79} 67.1 & \cellcolor{DarkSlateGray3!67} 56.9 &  \\
+\cellcolor{cascadecolor} \whisperfixed \,+ \aya & \cellcolor{Chartreuse3!80} 67.7 & \cellcolor{Chartreuse3!78} 66.6 & \cellcolor{Chartreuse3!72} 60.9 &  & \cellcolor{DarkSlateGray3!95} 80.8 & \cellcolor{DarkSlateGray3!94} 80.0 & \cellcolor{DarkSlateGray3!90} 76.7 &  \\
+\cellcolor{cascadecolor} \nonefixed \,+ \gemma & \cellcolor{Chartreuse3!80} 67.6 & \cellcolor{Chartreuse3!77} 65.7 & \cellcolor{Chartreuse3!70} 59.3 &  & \cellcolor{DarkSlateGray3!92} 78.4 & \cellcolor{DarkSlateGray3!91} 77.3 & \cellcolor{DarkSlateGray3!87} 74.0 &  \\
+\cellcolor{cascadecolor} \nonefixed \,+ \tower & \cellcolor{Chartreuse3!81} 69.0 & \cellcolor{Chartreuse3!80} 68.2 & \cellcolor{Chartreuse3!75} 63.4 &  & \cellcolor{DarkSlateGray3!96} 81.6 & \cellcolor{DarkSlateGray3!97} 82.0 & \cellcolor{DarkSlateGray3!91} 77.5 &  \\
+\cellcolor{cascadecolor} \seamlessfixed \,+ \aya & \cellcolor{Chartreuse3!83} 70.5 & \cellcolor{Chartreuse3!78} 65.9 & \cellcolor{Chartreuse3!72} 61.5 &  & \cellcolor{DarkSlateGray3!96} 81.3 & \cellcolor{DarkSlateGray3!94} 79.9 & \cellcolor{DarkSlateGray3!92} 78.4 &  \\
+\cellcolor{cascadecolor} \nonefixed \,+ \gemma & \cellcolor{Chartreuse3!83} 70.2 & \cellcolor{Chartreuse3!76} 65.0 & \cellcolor{Chartreuse3!70} 59.8 &  & \cellcolor{DarkSlateGray3!93} 78.9 & \cellcolor{DarkSlateGray3!90} 76.7 & \cellcolor{DarkSlateGray3!90} 76.2 &  \\
+\cellcolor{cascadecolor} \nonefixed \,+ \tower & \cellcolor{Chartreuse3!84} 71.2 & \cellcolor{Chartreuse3!80} 67.9 & \cellcolor{Chartreuse3!75} 63.6 &  & \cellcolor{DarkSlateGray3!97} 82.6 & \cellcolor{DarkSlateGray3!97} 82.3 & \cellcolor{DarkSlateGray3!93} 78.9 &  \\
+\cellcolor{cascadecolor} \canaryfixed \,+ \aya & \cellcolor{Chartreuse3!82} 70.1 & \cellcolor{Chartreuse3!78} 66.2 & \cellcolor{Chartreuse3!72} 61.2 &  & \cellcolor{DarkSlateGray3!96} 82.0 & \cellcolor{DarkSlateGray3!95} 81.1 & \cellcolor{DarkSlateGray3!91} 77.5 &  \\
+\cellcolor{cascadecolor} \nonefixed \,+ \gemma & \cellcolor{Chartreuse3!81} 69.0 & \cellcolor{Chartreuse3!76} 64.9 & \cellcolor{Chartreuse3!71} 60.0 &  & \cellcolor{DarkSlateGray3!94} 80.0 & \cellcolor{DarkSlateGray3!92} 78.1 & \cellcolor{DarkSlateGray3!89} 75.4 &  \\
+\cellcolor{cascadecolor} \nonefixed \,+ \tower & \cellcolor{Chartreuse3!83} 70.9 & \cellcolor{Chartreuse3!80} 68.2 & \cellcolor{Chartreuse3!75} 63.9 &  & \cellcolor{DarkSlateGray3!97} 82.7 & \cellcolor{DarkSlateGray3!97} 82.3 & \cellcolor{DarkSlateGray3!92} 78.3 &  \\
+\cellcolor{cascadecolor} \owsmfixed \,+ \aya & \cellcolor{Chartreuse3!78} 66.4 & \cellcolor{Chartreuse3!75} 63.7 & \cellcolor{Chartreuse3!69} 58.3 &  & \cellcolor{DarkSlateGray3!96} 81.2 & \cellcolor{DarkSlateGray3!94} 80.0 & \cellcolor{DarkSlateGray3!91} 77.3 &  \\
+\cellcolor{cascadecolor} \nonefixed \,+ \gemma & \cellcolor{Chartreuse3!77} 65.6 & \cellcolor{Chartreuse3!73} 62.3 & \cellcolor{Chartreuse3!67} 57.1 &  & \cellcolor{DarkSlateGray3!93} 78.7 & \cellcolor{DarkSlateGray3!91} 77.1 & \cellcolor{DarkSlateGray3!89} 75.3 &  \\
+\cellcolor{cascadecolor} \nonefixed \,+ \tower & \cellcolor{Chartreuse3!80} 67.8 & \cellcolor{Chartreuse3!78} 65.9 & \cellcolor{Chartreuse3!72} 61.6 &  & \cellcolor{DarkSlateGray3!96} 81.6 & \cellcolor{DarkSlateGray3!97} 82.3 & \cellcolor{DarkSlateGray3!92} 78.1 &  \\
+\cellcolor{speechllmcolor}{\desta} & \cellcolor{Chartreuse3!54} 45.8 & \cellcolor{Chartreuse3!52} 44.0 & \cellcolor{Chartreuse3!48} 41.2 &  & \cellcolor{DarkSlateGray3!64} 54.3 & \cellcolor{DarkSlateGray3!63} 53.9 & \cellcolor{DarkSlateGray3!57} 48.1 &  \\
+\cellcolor{speechllmcolor}{\qwenaudio} & \cellcolor{Chartreuse3!77} 65.4 & \cellcolor{Chartreuse3!73} 62.4 & \cellcolor{Chartreuse3!71} 60.2 &  & \cellcolor{DarkSlateGray3!87} 74.1 & \cellcolor{DarkSlateGray3!91} 77.1 & \cellcolor{DarkSlateGray3!81} 68.7 &  \\
+\cellcolor{speechllmcolor}{\phimultimodal} & \cellcolor{Chartreuse3!74} 62.6 & \cellcolor{Chartreuse3!53} 45.3 & \cellcolor{Chartreuse3!65} 55.0 &  & \cellcolor{DarkSlateGray3!84} 71.3 & \cellcolor{DarkSlateGray3!65} 55.3 & \cellcolor{DarkSlateGray3!82} 69.7 &  \\
+\cellcolor{speechllmcolor}{\voxtral} & \cellcolor{Chartreuse3!82} 69.7 & \cellcolor{Chartreuse3!79} 66.9 & \cellcolor{Chartreuse3!75} 64.1 &  & \cellcolor{DarkSlateGray3!94} 79.6 & \cellcolor{DarkSlateGray3!97} 82.5 & \cellcolor{DarkSlateGray3!90} 76.8 &  \\
+\cellcolor{speechllmcolor}{\spire} & \cellcolor{Chartreuse3!81} 68.9 & \cellcolor{Chartreuse3!77} 65.6 & \cellcolor{Chartreuse3!76} 65.0 &  & \cellcolor{DarkSlateGray3!92} 77.9 & \cellcolor{DarkSlateGray3!90} 76.3 & \cellcolor{DarkSlateGray3!86} 73.4 &  \\
+\bottomrule \end{tabular}

analysis/neuroparl_st/neuroparl-st_en_es.csv

@@ -1,22 +1,23 @@
 system,accuracy_ne,accuracy_term
-aya_canary-v2,57.87545787545788,68.63260706235913
-aya_owsm4.0-ctc,52.62515262515263,65.81517655897822
-aya_seamlessm4t,70.51282051282051,81.36739293764087
-aya_whisper,25.274725274725274,33.47107438016529
-canary-v2,70.94017094017094,80.954169797145
-desta2-8b,8.302808302808302,10.668670172802404
-gemma_canary-v2,16.483516483516482,17.731029301277236
-gemma_owsm4.0-ctc,4.151404151404151,4.808414725770098
-gemma_seamlessm4t,14.041514041514041,16.641622839969948
-gemma_whisper,4.700854700854701,5.447032306536439
+aya_canary-v2,70.08547008547008,82.00601051840721
+aya_owsm4.0-ctc,66.42246642246641,81.17956423741548
+aya_seamlessm4t,70.51282051282051,81.32982719759579
+aya_whisper,67.70451770451771,80.80390683696469
+canary-v2,69.59706959706959,80.24042073628851
+desta2-8b,45.78754578754579,54.28249436513899
+gemini-2.5-flash,67.76556776556777,82.49436513899325
+gemma_canary-v2,68.98656898656898,79.97746055597295
+gemma_owsm4.0-ctc,65.56776556776556,78.70022539444027
+gemma_seamlessm4t,70.2075702075702,78.88805409466566
+gemma_whisper,67.64346764346764,78.43726521412472
 owsm4.0-ctc,46.52014652014652,69.98497370398196
-phi4multimodal,62.698412698412696,71.29977460555973
-qwen2audio-7b,5.372405372405373,6.799398948159278
-seamlessm4t,65.62881562881563,75.69496619083395
-spirelm,68.92551892551893,77.98647633358378
-tower_canary-v2,71.73382173382173,83.20811419984973
-tower_owsm4.0-ctc,54.21245421245421,66.190833959429
-tower_seamlessm4t,71.30647130647131,82.64462809917356
-tower_whisper,69.59706959706959,82.1938392186326
-voxtral-small-24b,69.78021978021978,79.60180315552216
+phi4multimodal,62.637362637362635,71.29977460555973
+qwen2audio-7b,65.44566544566544,74.07963936889557
+seamlessm4t,65.56776556776556,75.6198347107438
+spirelm,68.92551892551893,77.94891059353868
+tower_canary-v2,70.94017094017094,82.71975957926371
+tower_owsm4.0-ctc,67.76556776556777,81.59278737791135
+tower_seamlessm4t,71.18437118437119,82.60706235912848
+tower_whisper,68.98656898656898,81.59278737791135
+voxtral-small-24b,69.71916971916971,79.60180315552216
 whisper,,

analysis/neuroparl_st/neuroparl-st_en_fr.csv

@@ -1,22 +1,23 @@
 system,accuracy_ne,accuracy_term
-aya_canary-v2,66.19718309859155,81.13335595520869
-aya_owsm4.0-ctc,6.46606914212548,7.261621988462844
-aya_seamlessm4t,7.298335467349553,7.4652188666440455
-aya_whisper,5.569782330345711,6.820495419070241
-canary-v2,67.41357234314981,82.1174075330845
-desta2-8b,7.490396927016646,9.399389209365456
-gemma_canary-v2,15.108834827144687,18.08618934509671
-gemma_owsm4.0-ctc,4.673495518565941,5.395317271801833
-gemma_seamlessm4t,23.367477592829704,29.250084832032574
-gemma_whisper,7.490396927016646,9.161859518154055
+aya_canary-v2,66.19718309859155,81.06549032914828
+aya_owsm4.0-ctc,63.70038412291933,80.04750593824228
+aya_seamlessm4t,65.94110115236876,79.94570749915168
+aya_whisper,66.64532650448145,80.01357312521208
+canary-v2,66.38924455825864,81.09942314217848
+desta2-8b,43.9820742637644,53.919239904988125
+gemini-2.5-flash,67.09346991037131,82.42280285035629
+gemma_canary-v2,64.9167733674776,78.07940278249067
+gemma_owsm4.0-ctc,62.29193341869398,77.06141839158465
+gemma_seamlessm4t,64.9807938540333,76.72209026128266
+gemma_whisper,65.68501920614597,77.29894808279606
 owsm4.0-ctc,42.18950064020486,67.119104173736
 phi4multimodal,45.26248399487836,55.34441805225653
-qwen2audio-7b,10.371318822023047,12.487275195113675
-seamlessm4t,59.53905249679897,72.75195113674924
+qwen2audio-7b,62.355953905249685,77.12928401764506
+seamlessm4t,59.53905249679897,72.71801832371904
 spirelm,65.55697823303457,76.34882931795046
-tower_canary-v2,69.01408450704226,83.10145911096029
-tower_owsm4.0-ctc,51.08834827144686,66.1689854088904
-tower_seamlessm4t,67.9257362355954,82.32100441126569
-tower_whisper,68.75800256081946,82.7621309806583
+tower_canary-v2,68.24583866837388,82.32100441126569
+tower_owsm4.0-ctc,65.87708066581305,82.32100441126569
+tower_seamlessm4t,67.9257362355954,82.2531387852053
+tower_whisper,68.24583866837388,82.04954190702409
 voxtral-small-24b,66.90140845070422,82.4567356633865
 whisper,,

analysis/neuroparl_st/neuroparl-st_en_it.csv

@@ -1,22 +1,23 @@
 system,accuracy_ne,accuracy_term
-aya_canary-v2,53.883495145631066,68.65671641791045
-aya_owsm4.0-ctc,24.351978171896317,31.92552225249773
-aya_seamlessm4t,48.908594815825374,62.44323342415985
-aya_whisper,23.124147339699864,29.70027247956403
-canary-v2,64.12005457025921,77.29336966394187
-desta2-8b,1.6371077762619373,1.0445049954586738
-gemma_canary-v2,3.819918144611187,3.451407811080836
-gemma_owsm4.0-ctc,10.845839017735335,14.441416893732969
-gemma_seamlessm4t,12.346521145975444,15.349682107175294
-gemma_whisper,3.0013642564802185,2.7702089009990916
+aya_canary-v2,61.174551386623165,77.45415318230853
+aya_owsm4.0-ctc,58.2537517053206,77.29336966394187
+aya_seamlessm4t,61.45975443383356,78.42870118074478
+aya_whisper,60.91405184174625,76.70299727520435
+canary-v2,62.82401091405184,76.43051771117166
+desta2-8b,41.20054570259209,48.09264305177112
+gemini-2.5-flash,62.68758526603001,79.51861943687557
+gemma_canary-v2,59.95907230559345,75.4314259763851
+gemma_owsm4.0-ctc,57.094133697135064,75.29518619436875
+gemma_seamlessm4t,59.754433833560704,76.20345140781109
+gemma_whisper,59.276944065484315,73.97820163487738
 owsm4.0-ctc,40.586630286493865,56.902815622161675
 phi4multimodal,55.04774897680764,69.70935513169846
-qwen2audio-7b,8.935879945429742,8.810172570390554
-seamlessm4t,58.663028649386085,65.21344232515894
-spirelm,65.00682128240109,73.56948228882834
-tower_canary-v2,64.66575716234652,79.01907356948229
-tower_owsm4.0-ctc,47.0668485675307,61.98910081743869
-tower_seamlessm4t,63.64256480218281,79.01907356948229
-tower_whisper,63.915416098226466,78.24704813805631
-voxtral-small-24b,64.12005457025921,76.93006357856494
+qwen2audio-7b,60.16371077762619,68.7102633969119
+seamlessm4t,58.663028649386085,65.12261580381471
+spirelm,65.00682128240109,73.433242506812
+tower_canary-v2,63.915416098226466,78.33787465940054
+tower_owsm4.0-ctc,61.59618008185539,78.06539509536785
+tower_seamlessm4t,63.5743519781719,78.88283378746594
+tower_whisper,63.43792633015006,77.52043596730245
+voxtral-small-24b,64.12005457025921,76.83923705722071
 whisper,,