ziyjiang JUNJIE99 commited on
Commit
24a1afa
·
1 Parent(s): 9c67d9e

Fix score aggregation: exclude MCMR from Overall-V2, use nDCG@5 for text (#180)

Browse files

- Fix score aggregation: exclude MCMR from Overall-V2, use nDCG@5 for text (9a24d5003df26f6cd29fd591e2a4dfb4edbfabdd)


Co-authored-by: JUNJIE ZHOU <JUNJIE99@users.noreply.huggingface.co>

Files changed (1) hide show
  1. utils_v3.py +6 -1
utils_v3.py CHANGED
@@ -122,6 +122,9 @@ def load_scores(raw_scores={}):
122
  if isinstance(score, dict):
123
  if 'visdoc' in modality:
124
  metric = "ndcg_linear@5" if "ndcg_linear@5" in score else "ndcg@5"
 
 
 
125
  score = score.get(metric, 0.0)
126
  all_scores[dataset] = round(score * 100.0, 2)
127
  return all_scores
@@ -139,7 +142,9 @@ def calculate_score(raw_scores=None):
139
 
140
  # Calculate overall score for all datasets
141
  avg_scores['Overall'] = get_avg(sum(all_scores.values()), len(ALL_DATASETS))
142
- v2_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['image'] or k in ALL_DATASETS_SPLITS['video'] or k in ALL_DATASETS_SPLITS['visdoc']}
 
 
143
  avg_scores['Overall-V2'] = get_avg(sum(v2_scores.values()), len(v2_scores))
144
  v3_newonly_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['text'] or k in ALL_DATASETS_SPLITS['audio'] or k in ALL_DATASETS_SPLITS['agent'] or k == "MCMR"}
145
  avg_scores['Overall-V3🆕'] = get_avg(sum(v3_newonly_scores.values()), len(v3_newonly_scores))
 
122
  if isinstance(score, dict):
123
  if 'visdoc' in modality:
124
  metric = "ndcg_linear@5" if "ndcg_linear@5" in score else "ndcg@5"
125
+ elif modality == 'text':
126
+ # The MMEB-V3 paper reports nDCG@5 for the text retrieval tasks.
127
+ metric = "ndcg_linear@5"
128
  score = score.get(metric, 0.0)
129
  all_scores[dataset] = round(score * 100.0, 2)
130
  return all_scores
 
142
 
143
  # Calculate overall score for all datasets
144
  avg_scores['Overall'] = get_avg(sum(all_scores.values()), len(ALL_DATASETS))
145
+ # MCMR is a V3-new dataset that lives in the image split, so it is excluded here to keep
146
+ # Overall-V2 consistent with Image-Overall(V2) below. It is still counted in Overall-V3.
147
+ v2_scores = {k:v for k,v in all_scores.items() if (k in ALL_DATASETS_SPLITS['image'] or k in ALL_DATASETS_SPLITS['video'] or k in ALL_DATASETS_SPLITS['visdoc']) and k != "MCMR"}
148
  avg_scores['Overall-V2'] = get_avg(sum(v2_scores.values()), len(v2_scores))
149
  v3_newonly_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['text'] or k in ALL_DATASETS_SPLITS['audio'] or k in ALL_DATASETS_SPLITS['agent'] or k == "MCMR"}
150
  avg_scores['Overall-V3🆕'] = get_avg(sum(v3_newonly_scores.values()), len(v3_newonly_scores))