[Smoothquant] Phi3 Vision Mappings (#1089)

## Purpose ##
* Provide phi3 vision smoothquant mappings so that users who use the
phi3 vision model from the multimodal blog post don't run into an issue

## Changes ##
* Add  `PHI3_VISION_SMOOTHQUANT_MAPPINGS`
* Add smoothquant to phi3 vision example

## Testing ##
* Ran `examples/multimodal_vision/phi3_vision_example.py` to completion

Signed-off-by: Kyle Sayers <[email protected]>

Author: kylesayrs

examples/multimodal_vision/phi3_vision_example.py

@@ -2,6 +2,7 @@
 from transformers import AutoModelForCausalLM, AutoProcessor
 
 from llmcompressor.modifiers.quantization import GPTQModifier
+from llmcompressor.modifiers.smoothquant import SmoothQuantModifier
 from llmcompressor.transformers import oneshot
 from llmcompressor.transformers.utils.data_collator import phi3_vision_data_collator
 
@@ -61,6 +62,7 @@ def tokenize(sample):
 
 # Recipe
 recipe = [
+    SmoothQuantModifier(smoothing_strength=0.8),
     GPTQModifier(
         targets="Linear",
         scheme="W4A16",

src/llmcompressor/modifiers/smoothquant/utils.py

@@ -43,6 +43,16 @@
         smooth_layers="re:.*post_attention_layernorm",
     ),
 ]
+PHI3_VISION_SMOOTHQUANT_MAPPINGS: List[LayerMap] = [
+    LayerMap(
+        balance_layers=["re:.*qkv_proj"],
+        smooth_layers="re:.*input_layernorm",
+    ),
+    LayerMap(
+        balance_layers=["re:.*gate_up_proj"],
+        smooth_layers="re:.*post_attention_layernorm",
+    ),
+]
 
 
 # Registry of layer mappings for different architectures
@@ -54,6 +64,7 @@
     "Qwen2ForCausalLM": DEFAULT_SMOOTHQUANT_MAPPINGS,
     "BloomForCausalLM": BLOOM_SMOOTHQUANT_MAPPINGS,
     "ChatGLMForConditionalGeneration": BLOOM_SMOOTHQUANT_MAPPINGS,
+    "Phi3VForCausalLM": PHI3_VISION_SMOOTHQUANT_MAPPINGS,
 }