pytorch
/

Phi-4-mini-instruct-AWQ-INT4

Text Generation

text-generation-inference

Model card Files Files and versions

jerryzh168 commited on Sep 19

Commit

134b774

·

verified ·

1 Parent(s): 7d361ed

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -275,7 +275,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TorchAoConfig
 # use "microsoft/Phi-4-mini-instruct" or "pytorch/Phi-4-mini-instruct-AWQ-INT4"
 model_id = "pytorch/Phi-4-mini-instruct-AWQ-INT4"
-quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 torch.cuda.reset_peak_memory_stats()

 # use "microsoft/Phi-4-mini-instruct" or "pytorch/Phi-4-mini-instruct-AWQ-INT4"
 model_id = "pytorch/Phi-4-mini-instruct-AWQ-INT4"
+quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map="cuda:0", torch_dtype=torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 torch.cuda.reset_peak_memory_stats()