mosaicml
/

mpt-30b-instruct

Text Generation

text-generation-inference

Model card Files Files and versions

abhi-mosaic commited on Jun 22, 2023

Commit

8a9262e

·

1 Parent(s): 716e2c1

Update README.md

Files changed (1) hide show

README.md +2 -2

README.md CHANGED Viewed

@@ -80,8 +80,8 @@ import transformers
 name = 'mosaicml/mpt-30b-instruct'
 config = transformers.AutoConfig.from_pretrained(name, trust_remote_code=True)
-config.attn_config['attn_impl'] = 'torch' # change this to use triton
-config.init_device = 'cpu' # For fast initialization directly on GPU! (if you have enough memory)
 model = transformers.AutoModelForCausalLM.from_pretrained(
   name,

 name = 'mosaicml/mpt-30b-instruct'
 config = transformers.AutoConfig.from_pretrained(name, trust_remote_code=True)
+config.attn_config['attn_impl'] = 'triton'  # change this to use triton-based FlashAttention
+config.init_device = 'cuda:0' # For fast initialization directly on GPU!
 model = transformers.AutoModelForCausalLM.from_pretrained(
   name,