Add fp32 fallback for torch<2.0

Author: skotapati

src/diffusers/models/activations.py

@@ -79,6 +79,9 @@ def __init__(self, dim_in: int, dim_out: int, approximate: str = "none", bias: b
         self.approximate = approximate
 
     def gelu(self, gate: torch.Tensor) -> torch.Tensor:
+        if gate.device.type == "mps" and torch.__version__ <'2.0.0':
+            # fp16 gelu not supported on mps before torch 2.0
+            return F.gelu(gate.to(dtype=torch.float32), approximate=self.approximate).to(dtype=gate.dtype)
         return F.gelu(gate, approximate=self.approximate)
 
     def forward(self, hidden_states):
@@ -102,6 +105,9 @@ def __init__(self, dim_in: int, dim_out: int, bias: bool = True):
         self.proj = nn.Linear(dim_in, dim_out * 2, bias=bias)
 
     def gelu(self, gate: torch.Tensor) -> torch.Tensor:
+        if gate.device.type == "mps" and torch.__version__ <'2.0.0':
+            # fp16 gelu not supported on mps before torch 2.0
+            return F.gelu(gate.to(dtype=torch.float32)).to(dtype=gate.dtype)
         return F.gelu(gate)
 
     def forward(self, hidden_states, *args, **kwargs):