From 50d0f0eab52fda64b002092dd7525855c2b59d2d Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Mon, 5 Sep 2022 15:03:32 -0700 Subject: [PATCH 1/9] fix #111: embed generation metadata in picture files. Problems: 1. The pip package python-xmp-toolkit isn't supported on Windows, and doesn't automatically install the required C++-based dependency libexempi3. See https://python-xmp-toolkit.readthedocs.io/en/latest/installation.html for manual installation steps. 2. The metadata is just a dump of argparse's parsed options, which could leak private information in arguments such as outdir. A todo is to build a dictionary from the argparse namespace, but exclude fields that could contain sensitive information. this might work --- environment.yaml | 1 + optimizedSD/optimized_txt2img.py | 29 ++++++++++++++++++++++++++--- scripts/txt2img.py | 3 +-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/environment.yaml b/environment.yaml index 7f25da800..ddf23effc 100644 --- a/environment.yaml +++ b/environment.yaml @@ -24,6 +24,7 @@ dependencies: - transformers==4.19.2 - torchmetrics==0.6.0 - kornia==0.6 + - python-xmp-toolkit==2.0.1 - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers - -e git+https://github.com/openai/CLIP.git@main#egg=clip - -e . diff --git a/optimizedSD/optimized_txt2img.py b/optimizedSD/optimized_txt2img.py index 6ead8618f..ec726ee1d 100644 --- a/optimizedSD/optimized_txt2img.py +++ b/optimizedSD/optimized_txt2img.py @@ -15,6 +15,8 @@ from ldm.util import instantiate_from_config from optimUtils import split_weighted_subprompts, logger from transformers import logging +from libxmp import XMPFiles, consts, XMPMeta # apt install exempi; pip install python-xmp-toolkit + # from samplers import CompVisDenoiser logging.set_verbosity_error() @@ -33,6 +35,22 @@ def load_model_from_config(ckpt, verbose=False): return sd +def add_metadata(filename, opt): + if opt.skip_metadata: + return + + metadata = str(opt) + + xmpfile = XMPFiles(file_path=filename, open_forupdate=True) + xmp = xmpfile.get_xmp() + if xmp is None: + xmp = XMPMeta() + xmp.append_array_item(consts.XMP_NS_DC, 'description', metadata, + {'prop_array_is_ordered':True, 'prop_value_is_array':True}) + xmpfile.put_xmp(xmp) + xmpfile.close_file() + + config = "optimizedSD/v1-inference.yaml" ckpt = "models/ldm/stable-diffusion-v1/model.ckpt" @@ -167,6 +185,11 @@ def load_model_from_config(ckpt, verbose=False): choices=["ddim", "plms"], default="plms", ) +parser.add_argument( + "--skip_metadata", + action='store_true', + help="do not add generation metadata to image file.", +) opt = parser.parse_args() tic = time.time() @@ -309,9 +332,9 @@ def load_model_from_config(ckpt, verbose=False): x_samples_ddim = modelFS.decode_first_stage(samples_ddim[i].unsqueeze(0)) x_sample = torch.clamp((x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0) x_sample = 255.0 * rearrange(x_sample[0].cpu().numpy(), "c h w -> h w c") - Image.fromarray(x_sample.astype(np.uint8)).save( - os.path.join(sample_path, "seed_" + str(opt.seed) + "_" + f"{base_count:05}.{opt.format}") - ) + filename = os.path.join(sample_path, "seed_" + str(opt.seed) + "_" + f"{base_count:05}.{opt.format}") + Image.fromarray(x_sample.astype(np.uint8)).save(filename) + add_metadata(filename, opt) seeds += str(opt.seed) + "," opt.seed += 1 base_count += 1 diff --git a/scripts/txt2img.py b/scripts/txt2img.py index da77e1a03..5468dc280 100644 --- a/scripts/txt2img.py +++ b/scripts/txt2img.py @@ -251,8 +251,7 @@ def main(): if not opt.skip_save: for x_sample in x_samples_ddim: x_sample = 255. * rearrange(x_sample.cpu().numpy(), 'c h w -> h w c') - Image.fromarray(x_sample.astype(np.uint8)).save( - os.path.join(sample_path, f"{base_count:05}.png")) + Image.fromarray(x_sample.astype(np.uint8)).save(os.path.join(sample_path, f"{base_count:05}.png")) base_count += 1 if not opt.skip_grid: From 7f28699856df83bde8394ca019a6febe64b52403 Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Mon, 5 Sep 2022 17:18:56 -0700 Subject: [PATCH 2/9] Switch to machine-readable JSON; suppress outdir --- optimizedSD/optimized_txt2img.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/optimizedSD/optimized_txt2img.py b/optimizedSD/optimized_txt2img.py index ec726ee1d..cfbe363c2 100644 --- a/optimizedSD/optimized_txt2img.py +++ b/optimizedSD/optimized_txt2img.py @@ -16,6 +16,7 @@ from optimUtils import split_weighted_subprompts, logger from transformers import logging from libxmp import XMPFiles, consts, XMPMeta # apt install exempi; pip install python-xmp-toolkit +import json # from samplers import CompVisDenoiser logging.set_verbosity_error() @@ -39,7 +40,13 @@ def add_metadata(filename, opt): if opt.skip_metadata: return - metadata = str(opt) + SKIP_OPT_KEYS = ['outdir'] + safe_opts = {} + for k, v in vars(opt).items(): + if k in SKIP_OPT_KEYS: + continue + safe_opts[k] = v + metadata = json.dumps(safe_opts) xmpfile = XMPFiles(file_path=filename, open_forupdate=True) xmp = xmpfile.get_xmp() From 7182afe3e317a0611bc0ecc33450cec623c5537a Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Mon, 5 Sep 2022 18:16:50 -0700 Subject: [PATCH 3/9] Switch from non-portable python-xmp-toolkit to pyexiv2 --- environment.yaml | 2 +- optimizedSD/optimized_txt2img.py | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/environment.yaml b/environment.yaml index ddf23effc..a2c453ac8 100644 --- a/environment.yaml +++ b/environment.yaml @@ -24,7 +24,7 @@ dependencies: - transformers==4.19.2 - torchmetrics==0.6.0 - kornia==0.6 - - python-xmp-toolkit==2.0.1 + - pyexiv2==2.8.0 - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers - -e git+https://github.com/openai/CLIP.git@main#egg=clip - -e . diff --git a/optimizedSD/optimized_txt2img.py b/optimizedSD/optimized_txt2img.py index cfbe363c2..ee88a272a 100644 --- a/optimizedSD/optimized_txt2img.py +++ b/optimizedSD/optimized_txt2img.py @@ -15,7 +15,7 @@ from ldm.util import instantiate_from_config from optimUtils import split_weighted_subprompts, logger from transformers import logging -from libxmp import XMPFiles, consts, XMPMeta # apt install exempi; pip install python-xmp-toolkit +import pyexiv2 import json # from samplers import CompVisDenoiser @@ -48,14 +48,8 @@ def add_metadata(filename, opt): safe_opts[k] = v metadata = json.dumps(safe_opts) - xmpfile = XMPFiles(file_path=filename, open_forupdate=True) - xmp = xmpfile.get_xmp() - if xmp is None: - xmp = XMPMeta() - xmp.append_array_item(consts.XMP_NS_DC, 'description', metadata, - {'prop_array_is_ordered':True, 'prop_value_is_array':True}) - xmpfile.put_xmp(xmp) - xmpfile.close_file() + img = pyexiv2.Image(filename) + img.modify_xmp({'Xmp.dc.description': metadata}) config = "optimizedSD/v1-inference.yaml" From 932e71c80afc1a83adb0a9e70e7ad18dae77645f Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Mon, 5 Sep 2022 19:50:55 -0700 Subject: [PATCH 4/9] trying py3exiv2 --- environment.yaml | 2 +- optimizedSD/optimized_txt2img.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/environment.yaml b/environment.yaml index a2c453ac8..86934d9a8 100644 --- a/environment.yaml +++ b/environment.yaml @@ -24,7 +24,7 @@ dependencies: - transformers==4.19.2 - torchmetrics==0.6.0 - kornia==0.6 - - pyexiv2==2.8.0 + - py3exiv2==0.11.0 - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers - -e git+https://github.com/openai/CLIP.git@main#egg=clip - -e . diff --git a/optimizedSD/optimized_txt2img.py b/optimizedSD/optimized_txt2img.py index ee88a272a..d1ce4ab40 100644 --- a/optimizedSD/optimized_txt2img.py +++ b/optimizedSD/optimized_txt2img.py @@ -48,8 +48,11 @@ def add_metadata(filename, opt): safe_opts[k] = v metadata = json.dumps(safe_opts) - img = pyexiv2.Image(filename) - img.modify_xmp({'Xmp.dc.description': metadata}) + img_md = pyexiv2.ImageMetadata(filename) + img_md.read() + key = 'Xmp.dc.description' + img_md[key] = pyexiv2.XmpTag(key, metadata) + img_md.write() config = "optimizedSD/v1-inference.yaml" From 263911f9adc6bc32b93808a1aba6a254e6f07532 Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Tue, 6 Sep 2022 10:29:14 -0700 Subject: [PATCH 5/9] Fourth library: tinyxmp --- environment.yaml | 2 +- optimizedSD/optimized_txt2img.py | 26 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/environment.yaml b/environment.yaml index 86934d9a8..2dd477183 100644 --- a/environment.yaml +++ b/environment.yaml @@ -24,7 +24,7 @@ dependencies: - transformers==4.19.2 - torchmetrics==0.6.0 - kornia==0.6 - - py3exiv2==0.11.0 + - -e https://github.com/sowbug/tinyxmp/tarball/master - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers - -e git+https://github.com/openai/CLIP.git@main#egg=clip - -e . diff --git a/optimizedSD/optimized_txt2img.py b/optimizedSD/optimized_txt2img.py index d1ce4ab40..5c9fa3bd7 100644 --- a/optimizedSD/optimized_txt2img.py +++ b/optimizedSD/optimized_txt2img.py @@ -15,7 +15,8 @@ from ldm.util import instantiate_from_config from optimUtils import split_weighted_subprompts, logger from transformers import logging -import pyexiv2 +import tinyxmp +import xml.dom.minidom import json # from samplers import CompVisDenoiser @@ -48,11 +49,24 @@ def add_metadata(filename, opt): safe_opts[k] = v metadata = json.dumps(safe_opts) - img_md = pyexiv2.ImageMetadata(filename) - img_md.read() - key = 'Xmp.dc.description' - img_md[key] = pyexiv2.XmpTag(key, metadata) - img_md.write() + xmp_file = tinyxmp.Metadata.load(filename) + # Since we just generated this file, we know there's no meaningful XMP data in it. + # So we create an empty template. + xmp = ''' + + + + + + + +''' + + doc = xml.dom.minidom.parseString(xmp) + e = doc.getElementsByTagName("rdf:li")[0] + textnode = doc.createTextNode(metadata) + e.appendChild(textnode) + xmp_file.write_xmp(doc.childNodes[0].toxml().encode("utf-8")) config = "optimizedSD/v1-inference.yaml" From 7eb0f597ae5d7d4b8e5f986fe0e296f2414bdd5b Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Tue, 6 Sep 2022 10:49:01 -0700 Subject: [PATCH 6/9] fix conda syntax --- environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index 2dd477183..026cb3c0d 100644 --- a/environment.yaml +++ b/environment.yaml @@ -24,7 +24,7 @@ dependencies: - transformers==4.19.2 - torchmetrics==0.6.0 - kornia==0.6 - - -e https://github.com/sowbug/tinyxmp/tarball/master + - -e git+https://github.com/sowbug/tinyxmp.git@master#egg=tinyxmp - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers - -e git+https://github.com/openai/CLIP.git@main#egg=clip - -e . From 92a6a10088ec407d8675f4752c913ca805274a46 Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Tue, 6 Sep 2022 11:04:21 -0700 Subject: [PATCH 7/9] documentation --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index ea33dcfdc..a3a43e8b0 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,12 @@ This will launch gradio on port 7860 with txt2img. You can also use `docker comp - Should generally be a multiple of 2x(n_samples) +## `--skip_metadata` + +**Whether to embed generation metadata (prompt, seed, size, etc.) in the generated file.** + +- Uses the [XMP](https://en.wikipedia.org/wiki/Extensible_Metadata_Platform) Description field to embed most of the command-line parameters that were used to generate the image. Excludes potentially privacy-sensitive parameters such as `outdir`. Currently works only with txt2img, not img2img. Enabled by default. +

Weighted Prompts

- Prompts can also be weighted to put relative emphasis on certain words. From f7249cd102d4d18ef6ef7d3b9bf7a65e828a8875 Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Tue, 6 Sep 2022 11:06:56 -0700 Subject: [PATCH 8/9] undo accidental whitespace change --- scripts/txt2img.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/txt2img.py b/scripts/txt2img.py index 5468dc280..da77e1a03 100644 --- a/scripts/txt2img.py +++ b/scripts/txt2img.py @@ -251,7 +251,8 @@ def main(): if not opt.skip_save: for x_sample in x_samples_ddim: x_sample = 255. * rearrange(x_sample.cpu().numpy(), 'c h w -> h w c') - Image.fromarray(x_sample.astype(np.uint8)).save(os.path.join(sample_path, f"{base_count:05}.png")) + Image.fromarray(x_sample.astype(np.uint8)).save( + os.path.join(sample_path, f"{base_count:05}.png")) base_count += 1 if not opt.skip_grid: From 70f71af3f9c512c704fa4c3f9e54aa4af1210cda Mon Sep 17 00:00:00 2001 From: Mike Tsao Date: Tue, 6 Sep 2022 12:55:23 -0700 Subject: [PATCH 9/9] implement for img2img --- README.md | 2 +- optimizedSD/optimized_img2img.py | 46 +++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a3a43e8b0..24023ae60 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ This will launch gradio on port 7860 with txt2img. You can also use `docker comp **Whether to embed generation metadata (prompt, seed, size, etc.) in the generated file.** -- Uses the [XMP](https://en.wikipedia.org/wiki/Extensible_Metadata_Platform) Description field to embed most of the command-line parameters that were used to generate the image. Excludes potentially privacy-sensitive parameters such as `outdir`. Currently works only with txt2img, not img2img. Enabled by default. +- Uses the [XMP](https://en.wikipedia.org/wiki/Extensible_Metadata_Platform) Description field to embed most of the command-line parameters that were used to generate the image. Excludes potentially privacy-sensitive parameters such as `outdir`. Enabled by default.

Weighted Prompts

diff --git a/optimizedSD/optimized_img2img.py b/optimizedSD/optimized_img2img.py index 9ca304c2e..9238a8cad 100644 --- a/optimizedSD/optimized_img2img.py +++ b/optimizedSD/optimized_img2img.py @@ -17,6 +17,9 @@ from optimUtils import split_weighted_subprompts, logger from transformers import logging import pandas as pd +import tinyxmp +import xml.dom.minidom +import json logging.set_verbosity_error() @@ -53,6 +56,38 @@ def load_img(path, h0, w0): return 2.0 * image - 1.0 +def add_metadata(filename, opt): + if opt.skip_metadata: + return + + SKIP_OPT_KEYS = ['outdir', 'init_img', 'from_file'] + safe_opts = {} + for k, v in vars(opt).items(): + if k in SKIP_OPT_KEYS: + continue + safe_opts[k] = v + metadata = json.dumps(safe_opts) + + xmp_file = tinyxmp.Metadata.load(filename) + # Since we just generated this file, we know there's no meaningful XMP data in it. + # So we create an empty template. + xmp = ''' + + + + + + + +''' + + doc = xml.dom.minidom.parseString(xmp) + e = doc.getElementsByTagName("rdf:li")[0] + textnode = doc.createTextNode(metadata) + e.appendChild(textnode) + xmp_file.write_xmp(doc.childNodes[0].toxml().encode("utf-8")) + + config = "optimizedSD/v1-inference.yaml" ckpt = "models/ldm/stable-diffusion-v1/model.ckpt" @@ -174,6 +209,11 @@ def load_img(path, h0, w0): choices=["ddim"], default="ddim", ) +parser.add_argument( + "--skip_metadata", + action='store_true', + help="do not add generation metadata to image file.", +) opt = parser.parse_args() tic = time.time() @@ -332,9 +372,9 @@ def load_img(path, h0, w0): x_samples_ddim = modelFS.decode_first_stage(samples_ddim[i].unsqueeze(0)) x_sample = torch.clamp((x_samples_ddim + 1.0) / 2.0, min=0.0, max=1.0) x_sample = 255.0 * rearrange(x_sample[0].cpu().numpy(), "c h w -> h w c") - Image.fromarray(x_sample.astype(np.uint8)).save( - os.path.join(sample_path, "seed_" + str(opt.seed) + "_" + f"{base_count:05}.{opt.format}") - ) + filename = os.path.join(sample_path, "seed_" + str(opt.seed) + "_" + f"{base_count:05}.{opt.format}") + Image.fromarray(x_sample.astype(np.uint8)).save(filename) + add_metadata(filename, opt) seeds += str(opt.seed) + "," opt.seed += 1 base_count += 1