Merge branch 'development' into dev/diffusers

Author: keturn

docs/features/OUTPAINTING.md

@@ -92,6 +92,21 @@ The new image is larger than the original (576x704) because 64 pixels were added
 to the top and right sides. You will need enough VRAM to process an image of
 this size.
 
+#### Outcropping non-InvokeAI images
+
+You can outcrop an arbitrary image that was not generated by InvokeAI,
+but your results will vary. The `inpainting-1.5` model is highly
+recommended, but if not feasible, then you may be able to improve the
+output by conditioning the outcropping with a text prompt that
+describes the scene using the `--new_prompt` argument:
+
+```bash
+invoke> !fix images/vacation.png --outcrop top 128 --new_prompt "family vacation"
+```
+
+You may also provide a different seed for outcropping to use by passing
+`-S<seed>`. A seed of "0" will generate a new random seed.
+
 A number of caveats:
 
 1. Although you can specify any pixel values, they will be rounded up to the

ldm/generate.py

@@ -566,17 +566,22 @@ def apply_postprocessor(
             ):
         # retrieve the seed from the image;
         seed   = None
-        image_metadata = None
         prompt = None
 
         args   = metadata_from_png(image_path)
-        seed   = args.seed
-        prompt = args.prompt
-        print(f'>> retrieved seed {seed} and prompt "{prompt}" from {image_path}')
+        if opt.seed is not None:
+            seed = opt.seed
+        elif args.seed >= 0:
+            seed = args.seed
+        else:
+            seed   = random.randrange(0, np.iinfo(np.uint32).max)
+
+        if opt.prompt is not None:
+            prompt = opt.prompt
+        else:
+            prompt = args.prompt
 
-        if not seed:
-            print('* Could not recover seed for image. Replacing with 42. This will not affect image quality')
-            seed = 42
+        print(f'>> using seed {seed} and prompt "{prompt}" for {image_path}')
 
         # try to reuse the same filename prefix as the original file.
         # we take everything up to the first period
@@ -623,6 +628,10 @@ def apply_postprocessor(
                     extend_instructions[direction]=int(pixels)
                 except ValueError:
                     print(f'** invalid extension instruction. Use <directions> <pixels>..., as in "top 64 left 128 right 64 bottom 64"')
+
+            opt.seed = seed
+            opt.prompt = prompt
+            
             if len(extend_instructions)>0:
                 restorer = Outcrop(image,self,)
                 return restorer.process (
@@ -1085,7 +1094,9 @@ def _has_transparency(self, image):
                 return True
         return False
 
-    def _check_for_erasure(self, image):
+    def _check_for_erasure(self, image:Image.Image)->bool:
+        if image.mode not in ('RGBA','RGB'):
+            return False
         width, height = image.size
         pixdata = image.load()
         colored = 0

ldm/invoke/args.py

@@ -866,6 +866,11 @@ def _create_dream_cmd_parser(self):
             default=32,
             help='When outpainting, the tile size to use for filling outpaint areas',
         )
+        postprocessing_group.add_argument(
+            '--new_prompt',
+            type=str,
+            help='Change the text prompt applied during postprocessing (default, use original generation prompt)',
+        )
         postprocessing_group.add_argument(
             '-ft',
             '--facetool',

ldm/invoke/generator/base.py

@@ -72,7 +72,7 @@ def generate(self,prompt,init_image,width,height,sampler, iterations=1,seed=None
             **kwargs
         )
         results             = []
-        seed                = seed if seed is not None else self.new_seed()
+        seed                = seed if seed is not None and seed > 0 else self.new_seed()
         first_seed          = seed
         seed, initial_noise = self.generate_initial_noise(seed, width, height)
 

ldm/invoke/model_cache.py

@@ -112,10 +112,13 @@ def set_default_model(self,model_name:str):
         Set the default model. The change will not take
         effect until you call model_cache.commit()
         '''
+        print(f'DEBUG: before set_default_model()\n{OmegaConf.to_yaml(self.config)}')
         assert model_name in self.models,f"unknown model '{model_name}'"
-        for model in self.models:
-            self.models[model].pop('default',None)
-        self.models[model_name]['default'] = True
+        config = self.config
+        for model in config:
+            config[model].pop('default',None)
+        config[model_name]['default'] = True
+        print(f'DEBUG: after set_default_model():\n{OmegaConf.to_yaml(self.config)}')
 
     def list_models(self) -> dict:
         '''

ldm/invoke/prompt_parser.py

@@ -636,7 +636,7 @@ def split_weighted_subprompts(text, skip_normalize=False)->list:
     weight_sum = sum(map(lambda x: x[1], parsed_prompts))
     if weight_sum == 0:
         print(
-            "Warning: Subprompt weights add up to zero. Discarding and using even weights instead.")
+            "* Warning: Subprompt weights add up to zero. Discarding and using even weights instead.")
         equal_weight = 1 / max(len(parsed_prompts), 1)
         return [(x[0], equal_weight) for x in parsed_prompts]
     return [(x[0], x[1] / weight_sum) for x in parsed_prompts]

ldm/invoke/readline.py

@@ -284,6 +284,7 @@ def _path_completions(self, text, state, extensions, shortcut_ok=True):
             switch,partial_path  = match.groups()
         partial_path = partial_path.lstrip()
 
+
         matches = list()
         path = os.path.expanduser(partial_path)
 
@@ -321,6 +322,7 @@ def _path_completions(self, text, state, extensions, shortcut_ok=True):
                 matches.append(
                     switch+os.path.join(os.path.dirname(full_path), node)
                 )
+
         return matches
 
 class DummyCompleter(Completer):

ldm/invoke/restoration/outcrop.py

@@ -28,11 +28,12 @@ def process (
         self.generate._set_sampler()
 
         def wrapped_callback(img,seed,**kwargs):
-            image_callback(img,orig_opt.seed,use_prefix=prefix,**kwargs)
+            preferred_seed = orig_opt.seed if orig_opt.seed> 0 else seed
+            image_callback(img,preferred_seed,use_prefix=prefix,**kwargs)
 
         result= self.generate.prompt2image(
-            orig_opt.prompt,
-            seed        = orig_opt.seed,    # uncomment to make it deterministic
+            opt.prompt,
+            seed        = opt.seed if opt.seed else orig_opt.seed,
             sampler     = self.generate.sampler,
             steps       = opt.steps,
             cfg_scale   = opt.cfg_scale,

ldm/modules/attention.py

@@ -282,7 +282,6 @@ def einsum_op_cuda(self, q, k, v):
 
     def get_attention_mem_efficient(self, q, k, v):
         if q.device.type == 'cuda':
-            torch.cuda.empty_cache()
             #print("in get_attention_mem_efficient with q shape", q.shape, ", k shape", k.shape, ", free memory is", get_mem_free_total(q.device))
             return self.einsum_op_cuda(q, k, v)
 

requirements.txt

@@ -38,4 +38,4 @@ git+https://github.com/openai/CLIP.git@main#egg=clip
 git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
 git+https://github.com/invoke-ai/Real-ESRGAN.git#egg=realesrgan
 git+https://github.com/invoke-ai/GFPGAN.git#egg=gfpgan
--e git+https://github.com/invoke-ai/clipseg.git@models-rename#egg=clipseg
+git+https://github.com/invoke-ai/clipseg.git@models-rename#egg=clipseg

scripts/invoke.py

@@ -28,6 +28,7 @@
 def main():
     """Initialize command-line parsers and the diffusion model"""
     global infile
+    print('* Initializing, be patient...')
 
     opt  = Args()
     args = opt.parse_args()
@@ -45,7 +46,6 @@ def main():
             print('--max_loaded_models must be >= 1; using 1')
             args.max_loaded_models = 1
 
-    print('* Initializing, be patient...')
     from ldm.generate import Generate
 
     # these two lines prevent a horrible warning message from appearing
@@ -89,7 +89,12 @@ def main():
             safety_checker=opt.safety_checker,
             max_loaded_models=opt.max_loaded_models,
             )
-    except (FileNotFoundError, IOError, KeyError) as e:
+    except FileNotFoundError:
+        print('** You appear to be missing configs/models.yaml')
+        print('** You can either exit this script and run scripts/preload_models.py, or fix the problem now.')
+        emergency_model_create(opt)
+        sys.exit(-1)
+    except (IOError, KeyError) as e:
         print(f'{e}. Aborting.')
         sys.exit(-1)
 
@@ -276,7 +281,7 @@ def image_writer(image, seed, upscaled=False, first_seed=None, use_prefix=None):
                     filename = f'{prefix}.{use_prefix}.{seed}.png'
                     tm = opt.text_mask[0]
                     th = opt.text_mask[1] if len(opt.text_mask)>1 else 0.5
-                    formatted_dream_prompt = f'!mask {opt.prompt} -tm {tm} {th}'
+                    formatted_dream_prompt = f'!mask {opt.input_file_path} -tm {tm} {th}'
                     path = file_writer.save_image_and_prompt_to_png(
                         image           = image,
                         dream_prompt    = formatted_dream_prompt,
@@ -316,7 +321,7 @@ def image_writer(image, seed, upscaled=False, first_seed=None, use_prefix=None):
                         tool = re.match('postprocess:(\w+)',opt.last_operation).groups()[0]
                         add_postprocessing_to_metadata(
                             opt,
-                            opt.prompt,
+                            opt.input_file_path,
                             filename,
                             tool,
                             formatted_dream_prompt,
@@ -481,6 +486,7 @@ def do_command(command:str, gen, opt:Args, completer) -> tuple:
         command = '-h'
     return command, operation
 
+
 def add_weights_to_config(model_path:str, gen, opt, completer):
     print(f'>> Model import in process. Please enter the values needed to configure this model:')
     print()
@@ -577,7 +583,7 @@ def write_config_file(conf_path, gen, model_name, new_config, clobber=False, mak
 
     try:
         print('>> Verifying that new model loads...')
-        yaml_str = gen.model_cache.add_model(model_name, new_config, clobber)
+        gen.model_cache.add_model(model_name, new_config, clobber)
         assert gen.set_model(model_name) is not None, 'model failed to load'
     except AssertionError as e:
         print(f'** aborting **')
@@ -613,10 +619,16 @@ def do_textmask(gen, opt, callback):
     )
 
 def do_postprocess (gen, opt, callback):
-    file_path = opt.prompt     # treat the prompt as the file pathname
+    file_path = opt.prompt      # treat the prompt as the file pathname
+    if opt.new_prompt is not None:
+        opt.prompt = opt.new_prompt
+    else:
+        opt.prompt = None
+        
     if os.path.dirname(file_path) == '': #basename given
         file_path = os.path.join(opt.outdir,file_path)
 
+    opt.input_file_path = file_path
     tool=None
     if opt.facetool_strength > 0:
         tool = opt.facetool
@@ -655,7 +667,10 @@ def do_postprocess (gen, opt, callback):
 def add_postprocessing_to_metadata(opt,original_file,new_file,tool,command):
     original_file = original_file if os.path.exists(original_file) else os.path.join(opt.outdir,original_file)
     new_file       = new_file     if os.path.exists(new_file)      else os.path.join(opt.outdir,new_file)
-    meta = retrieve_metadata(original_file)['sd-metadata']
+    try:
+        meta = retrieve_metadata(original_file)['sd-metadata']
+    except AttributeError:
+        meta = retrieve_metadata(new_file)['sd-metadata']
     if 'image' not in meta:
         meta = metadata_dumps(opt,seeds=[opt.seed])['image']
         meta['image'] = {}
@@ -703,7 +718,7 @@ def prepare_image_metadata(
     elif len(prior_variations) > 0:
         formatted_dream_prompt = opt.dream_prompt_str(seed=first_seed)
     elif operation == 'postprocess':
-        formatted_dream_prompt = '!fix '+opt.dream_prompt_str(seed=seed)
+        formatted_dream_prompt = '!fix '+opt.dream_prompt_str(seed=seed,prompt=opt.input_file_path)
     else:
         formatted_dream_prompt = opt.dream_prompt_str(seed=seed)
     return filename,formatted_dream_prompt
@@ -877,6 +892,36 @@ def write_commands(opt, file_path:str, outfilepath:str):
             f.write('\n'.join(commands))
         print(f'>> File {outfilepath} with commands created')
 
+def emergency_model_create(opt:Args):
+    completer   = get_completer(opt)
+    completer.complete_extensions(('.yaml','.yml','.ckpt','.vae.pt'))
+    completer.set_default_dir('.')
+    valid_path = False
+    while not valid_path:
+        weights_file = input('Enter the path to a downloaded models file, or ^C to exit: ')
+        valid_path = os.path.exists(weights_file)
+    dir,basename = os.path.split(weights_file)
+
+    valid_name = False
+    while not valid_name:
+        name = input('Enter a short name for this model (no spaces): ')
+        name = 'unnamed model' if len(name)==0 else name
+        valid_name = ' ' not in name
+
+    description = input('Enter a description for this model: ')
+    description = 'no description' if len(description)==0 else description
+
+    with open(opt.conf, 'w', encoding='utf-8') as f:
+        f.write(f'{name}:\n')
+        f.write(f'  description: {description}\n')
+        f.write(f'  weights: {weights_file}\n')
+        f.write(f'  config: ./configs/stable-diffusion/v1-inference.yaml\n')
+        f.write(f'  width: 512\n')
+        f.write(f'  height: 512\n')
+        f.write(f'  default: true\n')
+    print(f'Config file {opt.conf} is created. This script will now exit.')
+    print(f'After restarting you may examine the entry with !models and edit it with !edit.')
+    
 ######################################
 
 if __name__ == '__main__':