Skip to content

Commit e96a038

Browse files
authored
Add VP test in Stable diffusion pipeline (#19300)
### Description 1. Add visual parity test based on openai clip model 2. Add trigger rules ### Motivation and Context 1. check generated image is expected 2. reduce unnecessary triggers
1 parent 82c1cb4 commit e96a038

File tree

8 files changed

+131
-6
lines changed

8 files changed

+131
-6
lines changed

onnxruntime/python/tools/transformers/models/stable_diffusion/demo_txt2img.py

+1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ def run_inference(warmup=False):
6161
controlnet_scales=controlnet_scale,
6262
show_latency=not warmup,
6363
output_type="pil",
64+
deterministic=args.deterministic,
6465
)
6566

6667
if not args.disable_cuda_graph:

onnxruntime/python/tools/transformers/models/stable_diffusion/demo_utils.py

+1
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ def parse_arguments(is_xl: bool, parser):
239239
)
240240
parser.add_argument("--nvtx-profile", action="store_true", help="Enable NVTX markers for performance profiling.")
241241
parser.add_argument("--seed", type=int, default=None, help="Seed for random generator to get consistent results.")
242+
parser.add_argument("--deterministic", action="store_true", help="use deterministic algorithms.")
242243
parser.add_argument("-dc", "--disable-cuda-graph", action="store_true", help="Disable cuda graph.")
243244

244245
group = parser.add_argument_group("Options for ORT_CUDA engine only")

onnxruntime/python/tools/transformers/models/stable_diffusion/pipeline_stable_diffusion.py

+4
Original file line numberDiff line numberDiff line change
@@ -754,6 +754,7 @@ def run(
754754
controlnet_scales: Optional[torch.Tensor] = None,
755755
show_latency: bool = False,
756756
output_type: str = "pil",
757+
deterministic: bool = False,
757758
):
758759
"""
759760
Run the diffusion pipeline.
@@ -783,6 +784,9 @@ def run(
783784
output_type (str):
784785
It can be "latent", "pt" or "pil".
785786
"""
787+
if deterministic:
788+
torch.use_deterministic_algorithms(True)
789+
786790
if self.is_backend_tensorrt():
787791
import tensorrt as trt
788792
from trt_utilities import TRT_LOGGER
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import argparse
2+
import os
3+
4+
import cv2
5+
import open_clip
6+
import torch
7+
from PIL import Image
8+
from sentence_transformers import util
9+
10+
11+
def arg_parser():
12+
parser = argparse.ArgumentParser(description="Options for Compare 2 image")
13+
parser.add_argument("--image1", type=str, help="Path to image 1")
14+
parser.add_argument("--image2", type=str, help="Path to image 2")
15+
args = parser.parse_args()
16+
return args
17+
18+
19+
def image_encoder(img: Image.Image): # -> torch.Tensor:
20+
device = "cuda" if torch.cuda.is_available() else "cpu"
21+
model, _, preprocess = open_clip.create_model_and_transforms("ViT-B-16-plus-240", pretrained="laion400m_e32")
22+
model.to(device)
23+
24+
img1 = Image.fromarray(img).convert("RGB")
25+
img1 = preprocess(img1).unsqueeze(0).to(device)
26+
img1 = model.encode_image(img1)
27+
return img1
28+
29+
30+
def load_image(image_path: str): # -> Image.Image:
31+
# cv2.imread() can silently fail when the path is too long
32+
# https://stackoverflow.com/questions/68716321/how-to-use-absolute-path-in-cv2-imread
33+
if os.path.isabs(image_path):
34+
directory = os.path.dirname(image_path)
35+
current_directory = os.getcwd()
36+
os.chdir(directory)
37+
img = cv2.imread(os.path.basename(image_path), cv2.IMREAD_UNCHANGED)
38+
os.chdir(current_directory)
39+
else:
40+
img = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
41+
return img
42+
43+
44+
def generate_score(image1: str, image2: str): # -> float:
45+
test_img = load_image(image1)
46+
data_img = load_image(image2)
47+
img1 = image_encoder(test_img)
48+
img2 = image_encoder(data_img)
49+
cos_scores = util.pytorch_cos_sim(img1, img2)
50+
score = round(float(cos_scores[0][0]) * 100, 2)
51+
return score
52+
53+
54+
def main():
55+
args = arg_parser()
56+
image1 = args.image1
57+
image2 = args.image2
58+
score = round(generate_score(image1, image2), 2)
59+
print("similarity Score: ", {score})
60+
if score < 99:
61+
print(f"{image1} and {image2} are different")
62+
raise SystemExit(1)
63+
else:
64+
print(f"{image1} and {image2} are same")
65+
66+
67+
if __name__ == "__main__":
68+
main()
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
git+https://github.com/openai/CLIP.git
2+
open_clip_torch
3+
sentence_transformers
4+
pillow

tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml

+52-5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,32 @@
1+
##### start trigger Don't edit it manually, Please do edit set-trigger-rules.py ####
2+
trigger:
3+
branches:
4+
include:
5+
- main
6+
- rel-*
7+
paths:
8+
exclude:
9+
- docs/**
10+
- README.md
11+
- CONTRIBUTING.md
12+
- BUILD.md
13+
- 'js/web'
14+
- 'onnxruntime/core/providers/js'
15+
pr:
16+
branches:
17+
include:
18+
- main
19+
- rel-*
20+
paths:
21+
exclude:
22+
- docs/**
23+
- README.md
24+
- CONTRIBUTING.md
25+
- BUILD.md
26+
- 'js/web'
27+
- 'onnxruntime/core/providers/js'
28+
#### end trigger ####parameters:
29+
130
# reference: https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/python/tools/transformers/models/stable_diffusion/README.md
231
parameters:
332
- name: specificArtifact
@@ -143,7 +172,6 @@ stages:
143172
- job: Stable_Diffusion
144173
variables:
145174
skipComponentGovernanceDetection: true
146-
CCACHE_DIR: $(Pipeline.Workspace)/ccache
147175
workspace:
148176
clean: all
149177
pool: onnxruntime-Linux-GPU-A10-12G
@@ -162,7 +190,7 @@ stages:
162190

163191
- script: |
164192
docker run --rm --gpus all -v $PWD:/workspace -v $(Build.BinariesDirectory)/Release:/Release nvcr.io/nvidia/pytorch:22.11-py3 \
165-
bash -c "
193+
bash -c '
166194
set -ex; \
167195
python3 --version; \
168196
python3 -m pip install --upgrade pip; \
@@ -171,12 +199,31 @@ stages:
171199
python3 -m pip install -r requirements-cuda11.txt; \
172200
python3 -m pip install --upgrade polygraphy onnx-graphsurgeon --extra-index-url https://pypi.ngc.nvidia.com; \
173201
echo Generate an image guided by a text prompt; \
174-
python3 demo_txt2img.py 'astronaut riding a horse on mars'; \
175-
popd; \
176-
"
202+
python3 demo_txt2img.py --seed 1 --deterministic "astronaut riding a horse on mars" ; \
203+
find $(pwd) -name "*.png" ; \
204+
popd ; \
205+
'
177206
displayName: 'Run stable diffusion demo'
178207
workingDirectory: $(Build.SourcesDirectory)
179208
209+
- script: |
210+
docker run --rm --gpus all -v $PWD:/workspace nvcr.io/nvidia/pytorch:22.11-py3 \
211+
bash -c '
212+
set -ex; \
213+
python3 --version; \
214+
python3 -m pip install --upgrade pip; \
215+
pushd /workspace/onnxruntime/python/tools/transformers/models/stable_diffusion/; \
216+
image2=$(find $(pwd) -name "astronaut_riding_a_h*.png") ; \
217+
pushd test; \
218+
python3 -m pip install -r requirements.txt; \
219+
echo check demo_txt2image.py generate image; \
220+
python3 -u check_image.py --image1 astronaut_riding_txt2image-DDIM-50.png --image2 $image2; \
221+
popd ; \
222+
popd ; \
223+
'
224+
displayName: 'Check the generated image'
225+
workingDirectory: $(Build.SourcesDirectory)
226+
180227
- stage: Llama2_ONNX_FP16
181228
dependsOn:
182229
- Build_Onnxruntime_Cuda

tools/ci_build/set-trigger-rules.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
skip_js_changes = [
1515
"android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml",
1616
"android-x86_64-crosscompile-ci-pipeline.yml",
17+
"bigmodels-ci-pipeline.yml",
1718
"linux-ci-pipeline.yml",
1819
"linux-cpu-aten-pipeline.yml",
1920
"linux-cpu-eager-pipeline.yml",
@@ -31,7 +32,6 @@
3132
"orttraining-linux-ci-pipeline.yml",
3233
"orttraining-linux-gpu-ci-pipeline.yml",
3334
"orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml",
34-
"orttraining-linux-gpu-training-apis.yml",
3535
"orttraining-mac-ci-pipeline.yml",
3636
"win-ci-pipeline.yml",
3737
"win-gpu-ci-pipeline.yml",

0 commit comments

Comments
 (0)