Upload 2 files

modeling_internvl_chat.py

@@ -26,8 +26,6 @@ weight_tensor = torch.Tensor([5.,4.,3.,2.,1.])
 
 def get_special_token(tokenizer):
     preferential_ids_ = [id_[-1] for id_ in tokenizer(special_words)["input_ids"]]
-    print(preferential_ids_)
-    print(tokenizer.batch_decode(preferential_ids_))
     return preferential_ids_
 
 

modeling_qwen.py

@@ -2,6 +2,7 @@ from transformers.models.qwen2.modeling_qwen2 import *
 from transformers.modeling_outputs import dataclass, ModelOutput
 import torch.nn as nn
 import torch.nn.init as init
+import torch.nn.functional as F
 
 @dataclass
 class CausalLMOutputWithPastAndScore(ModelOutput):
@@ -113,8 +114,7 @@ class Expert_Head(nn.Module):
 
             pooled_expert_scores = torch.cat([pooled_scores2[:,:5], pooled_scores3, pooled_scores2[:,5:], pooled_scores4, expert_scores], dim=1)
 
-        return pooled_expert_scores
-
+        return F.sigmoid(pooled_expert_scores)
 
 class Qwen2ForCausalLM_score(Qwen2ForCausalLM):
     _tied_weights_keys = ["lm_head.weight", "regression_head.weight"]
@@ -200,6 +200,7 @@ class Qwen2ForCausalLM_score(Qwen2ForCausalLM):
         logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
 
         scores = self.lm_regression_head(hidden_states)
+        scores = F.sigmoid(scores)
 
         if input_ids is not None:
             batch_size = input_ids.shape[0]