feat: quantization per channel

README.md

@@ -38,16 +38,16 @@ This repository provides a ONNX converted and quantized version of meta-llama/Ll
 ## 📥 Evaluation Dataset
 
 We use [`jackhhao/jailbreak-classification`](https://huggingface.co/datasets/jackhhao/jailbreak-classification)
-for the evaluation
+for the evaluation (train+test)
 
 ## 🧪 Evaluation Results
 
-| Model                      | Accuracy | Precision | Recall | F1 Score | AUC-ROC | Inference Time |
-|----------------------------|----------|-----------|--------|----------|---------|----------------|
-| Llama-Prompt-Guard-2-22M   | 0.9569   | 0.9879    | 0.9260 | 0.9559   | 0.9259  | 33s            |
-| Llama-Prompt-Guard-2-22M-q | 0.9473   | 1.0000    | 0.8956 | 0.9449   | 0.9032  | 29s            |
-| Llama-Prompt-Guard-2-86M   | 0.9770   | 0.9980    | 0.9564 | 0.9767   | 0.9523  | 1m29s          |
-| Llama-Prompt-Guard-2-86M-q | 0.8937   | 1.0000    | 0.7894 | 0.8823   | 0.7263  | 1m15s          |
+| Model                      | Accuracy | Precision | Recall | F1 Score | AUC-ROC |
+|----------------------------|----------|-----------|--------|----------|---------|
+| Llama-Prompt-Guard-2-22M   | 0.9564   | 0.9888    | 0.9249 | 0.9558   | 0.9234  |
+| Llama-Prompt-Guard-2-22M-q | 0.9579   | 0.9967    | 0.9204 | 0.9449   | 0.9180  |
+| Llama-Prompt-Guard-2-86M   | 0.9801   | 0.9984    | 0.9625 | 0.9801   | 0.9519  |
+| Llama-Prompt-Guard-2-86M-q | 0.8989   | 1.0000    | 0.8018 | 0.89     | 0.7452  |
 
 ## 🤗 Usage
 

config.json

@@ -1,4 +1,5 @@
 {
+  "_name_or_path": "meta-llama/Llama-Prompt-Guard-2-22M",
   "architectures": [
     "DebertaV2ForSequenceClassification"
   ],
@@ -9,7 +10,6 @@
   "initializer_range": 0.02,
   "intermediate_size": 1536,
   "layer_norm_eps": 1e-07,
-  "legacy": true,
   "max_position_embeddings": 512,
   "max_relative_positions": -1,
   "model_type": "deberta-v2",
@@ -29,7 +29,7 @@
   "relative_attention": true,
   "share_att_key": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.51.3",
+  "transformers_version": "4.40.2",
   "type_vocab_size": 0,
   "vocab_size": 128100,
   "id2label": {

model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:61fe892154f76fffdb583ea9671a89ea5733a5ba5776a714780241559e820c0a
-size 284233055
+oid sha256:c6f4d2ebf59be36f3557ad484bf11a51f1b356f570958d1ec1cbbe322a8dfddf
+size 284217797

model.quant.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab5903d2c34dfc728a36074bbf272840d75df95cec6472efbb311ad06ddc99ee
-size 72563602
+oid sha256:ceefc0eedded030caf427faf828729b25a97f3ce70a043987aa68b912fdf836a
+size 72752037

tokenizer_config.json

@@ -46,7 +46,6 @@
   "cls_token": "[CLS]",
   "do_lower_case": false,
   "eos_token": "[SEP]",
-  "extra_special_tokens": {},
   "mask_token": "[MASK]",
   "max_length": 512,
   "model_max_length": 1000000000000000019884624838656,
@@ -58,7 +57,7 @@
   "sp_model_kwargs": {},
   "split_by_punct": false,
   "stride": 0,
-  "tokenizer_class": "DebertaV2TokenizerFast",
+  "tokenizer_class": "DebertaV2Tokenizer",
   "truncation_side": "right",
   "truncation_strategy": "longest_first",
   "unk_token": "[UNK]",