perf: use de-quantized mistral-7b-ielts-evaluator

README.md

@@ -11,7 +11,7 @@ license: mit
 preload_from_hub:
     - JacobLinCool/IELTS_essay_scoring_safetensors
     - KevSun/Engessay_grading_ML
-    - chillies/mistral-7b-ielts-evaluator-q4
+    - JacobLinCool/mistral-7b-ielts-evaluator-safetensors
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

model/common.py

@@ -0,0 +1,4 @@
+from accelerate import Accelerator
+
+accelerator = Accelerator()
+device = accelerator.device

model/mistral_7b_ielts_evaluator.py

@@ -1,13 +1,11 @@
 from typing import *
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from transformers import pipeline
 import spaces
+from .common import device
 
-model_name = "chillies/mistral-7b-ielts-evaluator-q4"
-file_name = "GUFF-ielts-fighter-unsloth.Q4_K_M.gguf"
-model = AutoModelForCausalLM.from_pretrained(model_name, gguf_file=file_name)
-tokenizer = AutoTokenizer.from_pretrained(model_name, gguf_file=file_name)
-pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+model_name = "JacobLinCool/mistral-7b-ielts-evaluator-safetensors"
+pipe = pipeline("text-generation", model_name, device=device)
 
 system = """In this task, you are required to evaluate an IELTS Writing Task 2 essay. Consider the following four criteria and provide a detailed assessment for each, along with a suggested band score:
 
@@ -60,7 +58,7 @@ def grade_mistral_7b_ielts_evaluator(question: str, answer: str) -> Tuple[float,
 ## Evaluation:
 """
 
-    outputs = pipe(text)
+    outputs = pipe(text, max_length=1024)
     overall_score = float(0.0)
 
     return overall_score, outputs