initial commit

modeling_phi.py

@@ -19,16 +19,16 @@ from transformers.modeling_outputs import CausalLMOutputWithPast
 
 from .configuration_phi import PhiConfig
 
-try:
-    from flash_attn.bert_padding import pad_input, unpad_input
-    from flash_attn.layers.rotary import RotaryEmbedding as FlashRotaryEmbedding
-    from flash_attn.modules.mha import FlashCrossAttention, FlashSelfAttention
-    from flash_attn.ops.fused_dense import FusedDense
-except:
-    pad_input, unpad_input = None, None
-    FlashRotaryEmbedding = None
-    FlashSelfAttention, FlashCrossAttention = None, None
-    FusedDense = None
+# try:
+#     from flash_attn.bert_padding import pad_input, unpad_input
+#     from flash_attn.layers.rotary import RotaryEmbedding as FlashRotaryEmbedding
+#     from flash_attn.modules.mha import FlashCrossAttention, FlashSelfAttention
+#     from flash_attn.ops.fused_dense import FusedDense
+# except:
+pad_input, unpad_input = None, None
+FlashRotaryEmbedding = None
+FlashSelfAttention, FlashCrossAttention = None, None
+FusedDense = None
 
 
 @dataclass