Fix performance issue introduced by torch cuda cache clear during generation

Author: Kyle0654

ldm/modules/attention.py

@@ -282,7 +282,6 @@ def einsum_op_cuda(self, q, k, v):
 
     def get_attention_mem_efficient(self, q, k, v):
         if q.device.type == 'cuda':
-            torch.cuda.empty_cache()
             #print("in get_attention_mem_efficient with q shape", q.shape, ", k shape", k.shape, ", free memory is", get_mem_free_total(q.device))
             return self.einsum_op_cuda(q, k, v)