Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Fix score aggregation: exclude MCMR from Overall-V2, use nDCG@5 for text (#180)
Browse files- Fix score aggregation: exclude MCMR from Overall-V2, use nDCG@5 for text (9a24d5003df26f6cd29fd591e2a4dfb4edbfabdd)
Co-authored-by: JUNJIE ZHOU <JUNJIE99@users.noreply.huggingface.co>
- utils_v3.py +6 -1
utils_v3.py
CHANGED
|
@@ -122,6 +122,9 @@ def load_scores(raw_scores={}):
|
|
| 122 |
if isinstance(score, dict):
|
| 123 |
if 'visdoc' in modality:
|
| 124 |
metric = "ndcg_linear@5" if "ndcg_linear@5" in score else "ndcg@5"
|
|
|
|
|
|
|
|
|
|
| 125 |
score = score.get(metric, 0.0)
|
| 126 |
all_scores[dataset] = round(score * 100.0, 2)
|
| 127 |
return all_scores
|
|
@@ -139,7 +142,9 @@ def calculate_score(raw_scores=None):
|
|
| 139 |
|
| 140 |
# Calculate overall score for all datasets
|
| 141 |
avg_scores['Overall'] = get_avg(sum(all_scores.values()), len(ALL_DATASETS))
|
| 142 |
-
|
|
|
|
|
|
|
| 143 |
avg_scores['Overall-V2'] = get_avg(sum(v2_scores.values()), len(v2_scores))
|
| 144 |
v3_newonly_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['text'] or k in ALL_DATASETS_SPLITS['audio'] or k in ALL_DATASETS_SPLITS['agent'] or k == "MCMR"}
|
| 145 |
avg_scores['Overall-V3🆕'] = get_avg(sum(v3_newonly_scores.values()), len(v3_newonly_scores))
|
|
|
|
| 122 |
if isinstance(score, dict):
|
| 123 |
if 'visdoc' in modality:
|
| 124 |
metric = "ndcg_linear@5" if "ndcg_linear@5" in score else "ndcg@5"
|
| 125 |
+
elif modality == 'text':
|
| 126 |
+
# The MMEB-V3 paper reports nDCG@5 for the text retrieval tasks.
|
| 127 |
+
metric = "ndcg_linear@5"
|
| 128 |
score = score.get(metric, 0.0)
|
| 129 |
all_scores[dataset] = round(score * 100.0, 2)
|
| 130 |
return all_scores
|
|
|
|
| 142 |
|
| 143 |
# Calculate overall score for all datasets
|
| 144 |
avg_scores['Overall'] = get_avg(sum(all_scores.values()), len(ALL_DATASETS))
|
| 145 |
+
# MCMR is a V3-new dataset that lives in the image split, so it is excluded here to keep
|
| 146 |
+
# Overall-V2 consistent with Image-Overall(V2) below. It is still counted in Overall-V3.
|
| 147 |
+
v2_scores = {k:v for k,v in all_scores.items() if (k in ALL_DATASETS_SPLITS['image'] or k in ALL_DATASETS_SPLITS['video'] or k in ALL_DATASETS_SPLITS['visdoc']) and k != "MCMR"}
|
| 148 |
avg_scores['Overall-V2'] = get_avg(sum(v2_scores.values()), len(v2_scores))
|
| 149 |
v3_newonly_scores = {k:v for k,v in all_scores.items() if k in ALL_DATASETS_SPLITS['text'] or k in ALL_DATASETS_SPLITS['audio'] or k in ALL_DATASETS_SPLITS['agent'] or k == "MCMR"}
|
| 150 |
avg_scores['Overall-V3🆕'] = get_avg(sum(v3_newonly_scores.values()), len(v3_newonly_scores))
|