use 🧨diffusers model

.github/workflows/test-invoke-conda.yml

@@ -1,21 +1,20 @@
 name: Test invoke.py
-on:
-  push:
-    branches:
-      - 'main'
-      - 'development'
-      - 'fix-gh-actions-fork'
-  pull_request:
-    branches:
-      - 'main'
-      - 'development'
+on: [push, pull_request]
 
 jobs:
   matrix:
+    # Run on:
+    # - pull requests
+    # - pushes to forks (will run in the forked project with that fork's secrets)
+    # - pushes to branches that are *not* pull requests
+    if: |
+      github.event_name == 'pull_request'
+      || github.repository != 'invoke-ai/InvokeAI'
+      || github.ref_protected
     strategy:
       matrix:
         stable-diffusion-model:
-          - 'stable-diffusion-1.5'
+          - diffusers-1.4
         environment-yaml:
           - environment-lin-amd.yml
           - environment-lin-cuda.yml
@@ -30,15 +29,13 @@ jobs:
           - environment-yaml: environment-mac.yml
             os: macos-12
             default-shell: bash -l {0}
-          - stable-diffusion-model: stable-diffusion-1.5
-            stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
-            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
-            stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
     name: ${{ matrix.environment-yaml }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     env:
       CONDA_ENV_NAME: invokeai
       INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
+      PYTHONUNBUFFERED: 1
+      HAVE_SECRETS: ${{ secrets.HUGGINGFACE_TOKEN != '' }}
     defaults:
       run:
         shell: ${{ matrix.default-shell }}
@@ -55,6 +52,15 @@ jobs:
       - name: create environment.yml
         run: cp "environments-and-requirements/${{ matrix.environment-yaml }}" environment.yml
 
+      - name: Use Cached Stable Diffusion Model
+        id: cache-sd-model
+        uses: actions/cache@v3
+        env:
+          cache-name: huggingface-${{ matrix.stable-diffusion-model }}
+        with:
+          path: ~/.cache/huggingface
+          key: ${{ env.cache-name }}
+
       - name: Use cached conda packages
         id: use-cached-conda-packages
         uses: actions/cache@v3
@@ -82,32 +88,24 @@ jobs:
         if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/development' }}
         run: echo "TEST_PROMPTS=tests/validate_pr_prompt.txt" >> $GITHUB_ENV
 
-      - name: Use Cached Stable Diffusion Model
-        id: cache-sd-model
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-${{ matrix.stable-diffusion-model }}
-        with:
-          path: ${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}
-          key: ${{ env.cache-name }}
-
-      - name: Download ${{ matrix.stable-diffusion-model }}
-        id: download-stable-diffusion-model
-        if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
-        run: |
-          mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
-          curl \
-            -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \
-            -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \
-            -L ${{ matrix.stable-diffusion-model-url }}
-
       - name: run configure_invokeai.py
         id: run-preload-models
         run: |
-          python scripts/configure_invokeai.py --no-interactive --yes
+          if [ "${HAVE_SECRETS}" == true ] ; then
+            mkdir -p ~/.huggingface
+            echo -n '${{ secrets.HUGGINGFACE_TOKEN }}' > ~/.huggingface/token
+          fi
+          python scripts/configure_invokeai.py \
+            --no-interactive --yes \
+            --full-precision  # can't use fp16 weights without a GPU
 
       - name: Run the tests
         id: run-tests
+        env:
+          # Set offline mode to make sure configure preloaded successfully.
+          HF_HUB_OFFLINE: 1
+          HF_DATASETS_OFFLINE: 1
+          TRANSFORMERS_OFFLINE: 1
         run: |
           time python scripts/invoke.py \
             --model ${{ matrix.stable-diffusion-model }} \

.github/workflows/test-invoke-pip.yml

@@ -1,20 +1,20 @@
 name: Test invoke.py pip
-on:
-  push:
-    branches:
-      - 'main'
-      - 'development'
-  pull_request:
-    branches:
-      - 'main'
-      - 'development'
+on: [push, pull_request]
 
 jobs:
   matrix:
+    # Run on:
+    # - pull requests
+    # - pushes to forks (will run in the forked project with that fork's secrets)
+    # - pushes to branches that are *not* pull requests
+    if: |
+      github.event_name == 'pull_request'
+      || github.repository != 'invoke-ai/InvokeAI'
+      || github.ref_protected
     strategy:
       matrix:
         stable-diffusion-model:
-          - stable-diffusion-1.5
+          - diffusers-1.4
         requirements-file:
           - requirements-lin-cuda.txt
           - requirements-lin-amd.txt
@@ -32,17 +32,15 @@ jobs:
           - requirements-file: requirements-mac-mps-cpu.txt
             os: macOS-12
             default-shell: bash -l {0}
-          - stable-diffusion-model: stable-diffusion-1.5
-            stable-diffusion-model-url: https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
-            stable-diffusion-model-dl-path: models/ldm/stable-diffusion-v1
-            stable-diffusion-model-dl-name: v1-5-pruned-emaonly.ckpt
     name: ${{ matrix.requirements-file }} on ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: ${{ matrix.default-shell }}
     env:
       INVOKEAI_ROOT: '${{ github.workspace }}/invokeai'
+      PYTHONUNBUFFERED: 1
+      HAVE_SECRETS: ${{ secrets.HUGGINGFACE_TOKEN != '' }}
     steps:
       - name: Checkout sources
         id: checkout-sources
@@ -53,6 +51,15 @@ jobs:
           mkdir -p ${{ env.INVOKEAI_ROOT }}/configs
           cp configs/models.yaml.example ${{ env.INVOKEAI_ROOT }}/configs/models.yaml
 
+      - name: Use Cached Stable Diffusion Model
+        id: cache-sd-model
+        uses: actions/cache@v3
+        env:
+          cache-name: huggingface-${{ matrix.stable-diffusion-model }}
+        with:
+          path: ~/.cache/huggingface
+          key: ${{ env.cache-name }}
+
       - name: set test prompt to main branch validation
         if: ${{ github.ref == 'refs/heads/main' }}
         run: echo "TEST_PROMPTS=tests/preflight_prompts.txt" >> $GITHUB_ENV
@@ -81,32 +88,24 @@ jobs:
       - name: install requirements
         run: ${{ env.pythonLocation }}/bin/pip install -r '${{ matrix.requirements-file }}'
 
-      - name: Use Cached Stable Diffusion Model
-        id: cache-sd-model
-        uses: actions/cache@v3
-        env:
-          cache-name: cache-${{ matrix.stable-diffusion-model }}
-        with:
-          path: ${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}
-          key: ${{ env.cache-name }}
-
-      - name: Download ${{ matrix.stable-diffusion-model }}
-        id: download-stable-diffusion-model
-        if: ${{ steps.cache-sd-model.outputs.cache-hit != 'true' }}
-        run: |
-          mkdir -p "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}"
-          curl \
-            -H "Authorization: Bearer ${{ secrets.HUGGINGFACE_TOKEN }}" \
-            -o "${{ env.INVOKEAI_ROOT }}/${{ matrix.stable-diffusion-model-dl-path }}/${{ matrix.stable-diffusion-model-dl-name }}" \
-            -L ${{ matrix.stable-diffusion-model-url }}
-
       - name: run configure_invokeai.py
         id: run-preload-models
         run: |
-          ${{ env.pythonLocation }}/bin/python scripts/configure_invokeai.py --no-interactive --yes
+          if [ "${HAVE_SECRETS}" == true ] ; then
+            mkdir -p ~/.huggingface
+            echo -n '${{ secrets.HUGGINGFACE_TOKEN }}' > ~/.huggingface/token
+          fi
+          ${{ env.pythonLocation }}/bin/python scripts/configure_invokeai.py \
+              --no-interactive --yes \
+              --full-precision  # can't use fp16 weights without a GPU
 
       - name: Run the tests
         id: run-tests
+        env:
+          # Set offline mode to make sure configure preloaded successfully.
+          HF_HUB_OFFLINE: 1
+          HF_DATASETS_OFFLINE: 1
+          TRANSFORMERS_OFFLINE: 1
         run: |
           time ${{ env.pythonLocation }}/bin/python scripts/invoke.py \
             --model ${{ matrix.stable-diffusion-model }} \

backend/invoke_ai_web_server.py

@@ -19,6 +19,7 @@
 from threading import Event
 
 from ldm.invoke.args import Args, APP_ID, APP_VERSION, calculate_init_img_hash
+from ldm.invoke.generator.diffusers_pipeline import PipelineIntermediateState
 from ldm.invoke.pngwriter import PngWriter, retrieve_metadata
 from ldm.invoke.prompt_parser import split_weighted_subprompts
 from ldm.invoke.generator.inpaint import infill_methods
@@ -847,7 +848,9 @@ def generate_images(
                 init_img_path = self.get_image_path_from_url(init_img_url)
                 generation_parameters["init_img"] = Image.open(init_img_path).convert('RGB')
 
-            def image_progress(sample, step):
+            def image_progress(progress_state: PipelineIntermediateState):
+                step = progress_state.step
+                sample = progress_state.latents
                 if self.canceled.is_set():
                     raise CanceledException
 

backend/modules/parameters.py

@@ -10,6 +10,8 @@
     "k_heun",
     "k_lms",
     "plms",
+    # diffusers:
+    "pndm",
 ]
 
 

configs/models.yaml.example

@@ -5,6 +5,15 @@
 # model requires a model config file, a weights file,
 # and the width and height of the images it
 # was trained on.
+diffusers-1.4:
+  description: Diffusers version of Stable Diffusion version 1.4
+  format: diffusers
+  repo_name: CompVis/stable-diffusion-v1-4
+  default: true
+diffusers-1.5:
+  description: Diffusers version of Stable Diffusion version 1.5
+  format: diffusers
+  repo_name: runwayml/stable-diffusion-v1-5
 stable-diffusion-1.5:
   description: The newest Stable Diffusion version 1.5 weight file (4.27 GB)
   weights: models/ldm/stable-diffusion-v1/v1-5-pruned-emaonly.ckpt

environments-and-requirements/environment-lin-amd.yml

@@ -11,7 +11,7 @@ dependencies:
     - --extra-index-url  https://download.pytorch.org/whl/rocm5.2/
     - albumentations==0.4.3
     - dependency_injector==4.40.0
-    - diffusers==0.6.0
+    - diffusers~=0.9
     - einops==0.3.0
     - eventlet
     - flask==2.1.3

environments-and-requirements/environment-lin-cuda.yml

@@ -12,9 +12,10 @@ dependencies:
   - pytorch=1.12.1
   - cudatoolkit=11.6
   - pip:
+    - accelerate~=0.13
     - albumentations==0.4.3
     - dependency_injector==4.40.0
-    - diffusers==0.6.0
+    - diffusers~=0.9
     - einops==0.3.0
     - eventlet
     - flask==2.1.3

environments-and-requirements/environment-mac.yml

@@ -22,7 +22,7 @@ dependencies:
 
   - albumentations=1.2
   - coloredlogs=15.0
-  - diffusers=0.6
+  - diffusers~=0.9
   - einops=0.3
   - eventlet
   - grpcio=1.46

environments-and-requirements/environment-win-cuda.yml

@@ -15,7 +15,7 @@ dependencies:
     - albumentations==0.4.3
     - basicsr==1.4.1
     - dependency_injector==4.40.0
-    - diffusers==0.6.0
+    - diffusers~=0.9
     - einops==0.3.0
     - eventlet
     - flask==2.1.3

environments-and-requirements/requirements-base.txt

@@ -1,7 +1,7 @@
 # pip will resolve the version which matches torch
 albumentations
 dependency_injector==4.40.0
-diffusers
+diffusers[torch]~=0.9
 einops
 eventlet
 facexlib
@@ -30,9 +30,10 @@ taming-transformers-rom1504
 test-tube>=0.7.5
 torch-fidelity
 torchmetrics
-transformers==4.21.*
+transformers~=4.24
 picklescan
 git+https://github.com/openai/CLIP.git@main#egg=clip
 git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
 git+https://github.com/invoke-ai/clipseg.git@relaxed-python-requirement#egg=clipseg
 git+https://github.com/invoke-ai/[email protected]#egg=pypatchmatch
+

installer/requirements.in

@@ -3,7 +3,7 @@
 --trusted-host https://download.pytorch.org
 accelerate~=0.14
 albumentations
-diffusers
+diffusers[torch]~=0.9
 einops
 eventlet
 facexlib