diffusers: let the scheduler do its scaling of the initial latents

Remove IPNDM scheduler; it is not behaving.

Author: keturn

backend/modules/parameters.py

@@ -11,7 +11,6 @@
     "k_lms",
     "plms",
     # diffusers:
-    "ipndm",
     "pndm",
 ]
 

ldm/invoke/args.py

@@ -105,7 +105,6 @@
     'k_lms',
     'plms',
     # diffusers:
-    "ipndm",
     "pndm",
 ]
 

ldm/invoke/generator/diffusers_pipeline.py

@@ -231,6 +231,8 @@ def generate_from_embeddings(self, latents: torch.Tensor, text_embeddings: torch
                                  run_id: str = None, **extra_step_kwargs):
         if run_id is None:
             run_id = secrets.token_urlsafe(self.ID_LENGTH)
+        # scale the initial noise by the standard deviation required by the scheduler
+        latents *= self.scheduler.init_noise_sigma
         yield PipelineIntermediateState(run_id=run_id, step=-1, timestep=self.scheduler.num_train_timesteps,
                                         latents=latents)
         # NOTE: Depends on scheduler being already initialized!