Fix eval_mteb.py of undefined variables
Browse files- scripts/eval_mteb.py +3 -3
scripts/eval_mteb.py
CHANGED
|
@@ -544,9 +544,9 @@ class Wrapper:
|
|
| 544 |
|
| 545 |
def _tokenize(self, sentences: List[str], is_query: bool):
|
| 546 |
|
| 547 |
-
batch_dict = tokenizer(sentences, max_length=
|
| 548 |
-
batch_dict['input_ids'] = [input_ids + [tokenizer.eos_token_id] for input_ids in batch_dict['input_ids']]
|
| 549 |
-
batch_dict = tokenizer.pad(batch_dict, padding=True, return_attention_mask=True, return_tensors='pt')
|
| 550 |
batch_dict['is_causal'] = False
|
| 551 |
return batch_dict
|
| 552 |
|
|
|
|
| 544 |
|
| 545 |
def _tokenize(self, sentences: List[str], is_query: bool):
|
| 546 |
|
| 547 |
+
batch_dict = self.tokenizer(sentences, max_length=self.max_seq_length - 1, return_attention_mask=False, padding=False, truncation=True)
|
| 548 |
+
batch_dict['input_ids'] = [input_ids + [self.tokenizer.eos_token_id] for input_ids in batch_dict['input_ids']]
|
| 549 |
+
batch_dict = self.tokenizer.pad(batch_dict, padding=True, return_attention_mask=True, return_tensors='pt')
|
| 550 |
batch_dict['is_causal'] = False
|
| 551 |
return batch_dict
|
| 552 |
|