[Core] Default to using per_token quantization for fp8 when cutlass is supported. (vllm-project#8651)

Signed-off-by: mgoin <[email protected]>
Co-authored-by: Michael Goin <[email protected]>
Co-authored-by: mgoin <[email protected]>
Signed-off-by: Isotr0py <[email protected]>

Author: 2 people

vllm/model_executor/layers/quantization/fp8.py

@@ -355,7 +355,8 @@ def apply(self,
             input_scale=layer.input_scale,
             bias=bias,
             cutlass_fp8_supported=self.cutlass_fp8_supported,
-            use_per_token_if_dynamic=False)
+            # Default to using per_token quantization if cutlass is supported
+            use_per_token_if_dynamic=self.cutlass_fp8_supported)
 
 
 class Fp8MoEMethod(FusedMoEMethodBase):