Fix rocm compat (#1373)

* fix rocm compat

Signed-off-by: Qubitium <[email protected]>

* format

Signed-off-by: Qubitium <[email protected]>

* fix rocm compat

Signed-off-by: Qubitium <[email protected]>

* fix call

* fix rocm compat

Signed-off-by: Qubitium <[email protected]>

* use svd for rocm

Signed-off-by: Qubitium <[email protected]>

* try ck backend

Signed-off-by: Qubitium <[email protected]>

* try magma backend

Signed-off-by: Qubitium <[email protected]>

* cleanup linalg backend

Signed-off-by: Qubitium <[email protected]>

* disable torch.compile

Signed-off-by: Qubitium <[email protected]>

* fix eora_save

* model_save_dir is optional

Signed-off-by: Qubitium <[email protected]>

* fix eora save path cannot be a .safetensors

* add linalg test

Signed-off-by: Qubitium <[email protected]>

---------

Signed-off-by: Qubitium <[email protected]>
Co-authored-by: CSY-ModelCloud <[email protected]>

Author: Qubitium

examples/benchmark/generation_speed.py

@@ -23,13 +23,11 @@
 
 import torch
 from datasets import Dataset, load_dataset
+from gptqmodel import BACKEND, GPTQModel, QuantizeConfig
 from logbar import LogBar
 from transformers import AutoTokenizer, GenerationConfig
 from transformers.generation.logits_process import LogitsProcessor
 
-from gptqmodel import BACKEND, GPTQModel, QuantizeConfig
-
-
 logger = LogBar.shared()
 
 random.seed(0)

examples/benchmark/ipex.py

@@ -20,7 +20,6 @@
 import torch
 from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
 
-
 try:
     from optimum.intel.utils.modeling_utils import bind_cores_for_best_perf
     bind_cores_for_best_perf()
@@ -30,7 +29,6 @@
 
 import argparse
 
-
 parser = argparse.ArgumentParser(description="Benchmark IPEX vs HF on a pre-trained model.")
 parser.add_argument("--model", type=str, required=True, help="Path or name of the pre-trained model.")
 parser.add_argument("--cores", type=int, default=8, help="Number of CPU cores to use.")

examples/benchmark/perplexity.py

@@ -17,10 +17,8 @@
 import argparse
 import os
 
-from transformers import AutoTokenizer
-
 from gptqmodel.utils import Perplexity
-
+from transformers import AutoTokenizer
 
 os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
 

examples/evaluation/run_language_modeling_task.py

@@ -18,12 +18,10 @@
 
 import datasets
 import torch
-from transformers import AutoTokenizer
-
 from gptqmodel import BACKEND, GPTQModel, QuantizeConfig
 from gptqmodel.eval_tasks import LanguageModelingTask
 from gptqmodel.utils.torch import torch_empty_cache
-
+from transformers import AutoTokenizer
 
 DATASET = "tatsu-lab/alpaca"
 WITH_INPUT_TEMPLATE = "Instruction:\n{instruction}\n\nInput:\n{input}\n\nOutput:\n"

examples/evaluation/run_sequence_classification_task.py

@@ -19,12 +19,10 @@
 
 import datasets
 import torch
-from transformers import AutoTokenizer
-
 from gptqmodel import BACKEND, GPTQModel, QuantizeConfig
 from gptqmodel.eval_tasks import SequenceClassificationTask
 from gptqmodel.utils.torch import torch_empty_cache
-
+from transformers import AutoTokenizer
 
 DATASET = "cardiffnlp/tweet_sentiment_multilingual"
 TEMPLATE = "Question:What's the sentiment of the given text? Choices are {labels}.\nText: {text}\nAnswer:"

examples/evaluation/run_text_summarization_task.py

@@ -19,12 +19,10 @@
 
 import datasets
 import torch
-from transformers import AutoTokenizer, GenerationConfig
-
 from gptqmodel import BACKEND, GPTQModel, QuantizeConfig
 from gptqmodel.eval_tasks import TextSummarizationTask
 from gptqmodel.utils.torch import torch_empty_cache
-
+from transformers import AutoTokenizer, GenerationConfig
 
 os.system("pip install py7zr")
 

examples/inference/run_transformers.py

@@ -16,7 +16,6 @@
 
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
-
 tokenizer = AutoTokenizer.from_pretrained("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ")
 quantized_model = AutoModelForCausalLM.from_pretrained("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ")
 print(tokenizer.decode(quantized_model.generate(**tokenizer("gptqmodel is", return_tensors="pt").to(quantized_model.device))[0]))

examples/inference/run_with_different_backends.py

@@ -19,10 +19,8 @@
 import sys
 from argparse import ArgumentParser
 
-from transformers import AutoTokenizer
-
 from gptqmodel import BACKEND, GPTQModel, QuantizeConfig, get_best_device
-
+from transformers import AutoTokenizer
 
 os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
 pretrained_model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"

examples/quantization/basic_usage.py

@@ -16,10 +16,8 @@
 
 import os
 
-from transformers import AutoTokenizer
-
 from gptqmodel import GPTQModel, QuantizeConfig, get_best_device
-
+from transformers import AutoTokenizer
 
 os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
 

examples/quantization/basic_usage_autoround.py

@@ -15,11 +15,9 @@
 # limitations under the License.
 
 import torch
-from transformers import AutoTokenizer
-
 from gptqmodel import GPTQModel
 from gptqmodel.quantization.config import AutoRoundQuantizeConfig  # noqa: E402
-
+from transformers import AutoTokenizer
 
 pretrained_model_id = "/monster/data/model/TinyLlama-1.1B-Chat-v1.0" # "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 quantized_model_id = "./autoround/TinyLlama-1.1B-Chat-v1.0-4bit-128g"

examples/quantization/basic_usage_wikitext2.py

@@ -16,10 +16,8 @@
 
 import torch
 from datasets import load_dataset
-from transformers import AutoTokenizer
-
 from gptqmodel import GPTQModel, QuantizeConfig
-
+from transformers import AutoTokenizer
 
 pretrained_model_id = "/monster/data/model/TinyLlama-1.1B-Chat-v1.0" # "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 quantized_model_id = "TinyLlama-1.1B-Chat-v1.0-4bit-128g"

examples/quantization/transformers_usage.py

@@ -16,7 +16,6 @@
 
 from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig
 
-
 model_id = "facebook/opt-125m"
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 dataset = ["gptqmodel is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm."]

gptqmodel/__init__.py

@@ -22,7 +22,6 @@
 from .utils.exllama import exllama_set_max_input_length
 from .version import __version__
 
-
 if os.getenv('GPTQMODEL_USE_MODELSCOPE', 'False').lower() in ['true', '1']:
     try:
         from modelscope.utils.hf_util.patcher import patch_hub

gptqmodel/eora/eora.py

@@ -21,6 +21,7 @@
 
 from ..looper.named_module import NamedModule
 from ..utils.logger import setup_logger
+from ..utils.rocm import IS_ROCM
 
 log = setup_logger()
 
@@ -51,7 +52,13 @@ def eora_compute_lora(
     # save this later for SVD
     raw_scaling_diag_matrix = eigen_scaling_diag_matrix.to(dtype=torch.float64, device=device)
 
+    if IS_ROCM:
+        # hip cannot resolve linalg ops
+        original_backend = torch.backends.cuda.preferred_linalg_library()
+        torch.backends.cuda.preferred_linalg_library(backend="magma")
+
     L, Q = torch.linalg.eigh(raw_scaling_diag_matrix)
+
     if (L < 0).any():
         ## When expanding the calibration data size for EoRA, I suggest maintaining the balance by allocating 50% to general input (C4) and the remaining 50% to downstream task data.
         log.warn(f"Found negative eigenvalues in `{module.name}`. Please increase your calibration data set for EoRA.")
@@ -88,5 +95,9 @@ def eora_compute_lora(
     del L, Q, U, S, V,
     del w_wq_delta, raw_scaling_diag_matrix, sqrtEigenvalues, scaling_diag_matrix, scaling_matrix_inv, delta_scale
     del truc_s, truc_u, truc_v, truc_sigma, sqrtS
-    
+
+    # revert linalg backend
+    if IS_ROCM:
+        torch.backends.cuda.preferred_linalg_library(original_backend)
+
     return A, B

gptqmodel/looper/eora_processor.py

@@ -58,11 +58,11 @@ def __init__(self, tokenizer, qcfg: QuantizeConfig, calibration_dataset, prepare
         # needed by eora
         # torch._dynamo.config.capture_scalar_outputs = True
 
-        self.eora_compute_lora = torch_compile(eora_compute_lora)
-        self.eora_process_input = torch_compile(eora_process_input)
+        #self.eora_compute_lora = torch_compile(eora_compute_lora)
+        #self.eora_process_input = torch_compile(eora_process_input)
 
-        # self.eora_compute_lora = eora_compute_lora
-        # self.eora_process_input = eora_process_input
+        self.eora_compute_lora = eora_compute_lora
+        self.eora_process_input = eora_process_input
 
     def log_plotly(self):
         task = self.logger_task

gptqmodel/models/_const.py

@@ -25,7 +25,6 @@
 from ..utils.rocm import IS_ROCM
 from ..utils.torch import HAS_CUDA, HAS_MPS, HAS_XPU
 
-
 CPU = device("cpu")
 CUDA = device("cuda")
 CUDA_0 = device("cuda:0")

gptqmodel/models/auto.py

@@ -20,7 +20,6 @@
 
 from ..utils.logger import setup_logger
 
-
 log = setup_logger()
 
 if not os.environ.get("PYTORCH_CUDA_ALLOC_CONF", None):
@@ -33,7 +32,6 @@
 
 import sys  # noqa: E402
 
-
 # TODO: waiting for pytorch implementgation of aten ops for MPS
 if sys.platform == "darwin":
     os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
@@ -108,7 +106,6 @@
 from .definitions.xverse import XverseGPTQ  # noqa: E402
 from .definitions.yi import YiGPTQ  # noqa: E402
 
-
 # make quants and inference more determinisitc
 torch.manual_seed(787)
 random.seed(787)

gptqmodel/models/base.py

@@ -28,14 +28,8 @@
 from packaging import version
 from packaging.version import Version
 from tokenicer import Tokenicer
-from transformers import (
-    AutoModelForCausalLM,
-    AutoProcessor,
-    PreTrainedModel,
-    PreTrainedTokenizerBase,
-    ProcessorMixin,
-    modeling_utils,
-)
+from transformers import (AutoModelForCausalLM, AutoProcessor, PreTrainedModel,
+                          PreTrainedTokenizerBase, ProcessorMixin, modeling_utils)
 
 from ..adapter.adapter import Adapter
 from ..nn_modules.hooked_linear import replace_linear_with_hooked_linear
@@ -49,31 +43,13 @@
 from ..utils.hf import autofix_hf_model_config
 from ..utils.importer import select_quant_linear
 from ..utils.logger import setup_logger
-from ..utils.model import (
-    MODALITY,
-    check_to_quantized,
-    find_modules,
-    get_device,
-    get_module,
-    get_module_by_name_prefix,
-    get_moe_layer_modules,
-    move_to,
-    nested_move_to,
-    pack_model,
-)
+from ..utils.model import (MODALITY, check_to_quantized, find_modules, get_device, get_module,
+                           get_module_by_name_prefix, get_moe_layer_modules, move_to, nested_move_to, pack_model)
 from ..utils.torch import torch_compile, torch_empty_cache
 from ._const import CALIBRATION_DATASET_CONCAT_CHAR, CPU, DEFAULT_MAX_SHARD_SIZE, DEVICE, SUPPORTS_MODULE_TYPES
 from .loader import ModelLoader
-from .writer import (
-    PROCESS_LOG_FWD_TIME,
-    PROCESS_LOG_LAYER,
-    PROCESS_LOG_MODULE,
-    PROCESS_LOG_TIME,
-    QUANT_LOG_DAMP,
-    QUANT_LOG_LOSS,
-    ModelWriter,
-)
-
+from .writer import (PROCESS_LOG_FWD_TIME, PROCESS_LOG_LAYER, PROCESS_LOG_MODULE,
+                     PROCESS_LOG_TIME, QUANT_LOG_DAMP, QUANT_LOG_LOSS, ModelWriter)
 
 # pytorch 2.6.0 fixes many compilation errors
 TORCH_MIN_VERSION_STR = '2.6.0'
@@ -511,7 +487,7 @@ def _eora_generate(
             auto_gc=auto_gc,
         )
 
-        self.eora_save(eora_path=adapter.path)
+        self.eora_save(save_dir=adapter.path, model_save_dir=self.model_local_path)
         return
 
     @torch.no_grad()

gptqmodel/models/definitions/gemma2.py

@@ -18,7 +18,6 @@
 from ...utils.logger import setup_logger
 from ..base import BaseGPTQModel
 
-
 log = setup_logger()
 
 SUPPORT_ERR = "Currently, only vLLM/SGLang with flashinfer enabled can correctly inference a quantized Gemma2-27B model. Pre-quantized model with sample vLLM code: https://huggingface.co/ModelCloud/gemma-2-27b-it-gptq-4bit ."

gptqmodel/models/loader.py

@@ -24,7 +24,6 @@
 import torch
 import transformers
 
-
 if os.getenv('GPTQMODEL_USE_MODELSCOPE', 'False').lower() in ['true', '1']:
     try:
         from modelscope import snapshot_download
@@ -49,22 +48,11 @@
 from ..utils.importer import auto_select_device, normalize_device_device_map, select_quant_linear
 from ..utils.logger import setup_logger
 from ..utils.marlin import _validate_marlin_compatibility, _validate_marlin_device_support
-from ..utils.model import (
-    auto_dtype,
-    convert_gptq_v1_to_v2_format,
-    find_modules,
-    get_checkpoints,
-    get_moe_layer_modules,
-    gptqmodel_post_init,
-    load_checkpoint_in_model_then_tie_weights,
-    make_quant,
-    simple_dispatch_model,
-    verify_model_hash,
-    verify_sharded_model_hashes,
-)
+from ..utils.model import (auto_dtype, convert_gptq_v1_to_v2_format, find_modules, get_checkpoints,
+                           get_moe_layer_modules, gptqmodel_post_init, load_checkpoint_in_model_then_tie_weights,
+                           make_quant, simple_dispatch_model, verify_model_hash, verify_sharded_model_hashes)
 from ._const import DEVICE, SUPPORTED_MODELS, normalize_device
 
-
 log = setup_logger()
 
 ATTN_IMPLEMENTATION = "attn_implementation"

gptqmodel/models/writer.py

@@ -37,36 +37,18 @@
 
 from ..adapter.adapter import HF_ADAPTER_FILE_NAME, HF_ADAPTER_WEIGHT_KEY_PREFIX, Lora
 from ..adapter.peft import LoraConfig
-from ..quantization.config import (
-    FORMAT,
-    META_FIELD_DAMP_AUTO_INCREMENT,
-    META_FIELD_DAMP_PERCENT,
-    META_FIELD_MSE,
-    META_FIELD_QUANTIZER,
-    META_FIELD_STATIC_GROUPS,
-    META_FIELD_TRUE_SEQUENTIAL,
-    META_FIELD_URI,
-    META_QUANTIZER_GPTQMODEL,
-    META_VALUE_URI,
-    MIN_VERSION_WITH_V2,
-)
+from ..quantization.config import (FORMAT, META_FIELD_DAMP_AUTO_INCREMENT, META_FIELD_DAMP_PERCENT, META_FIELD_MSE,
+                                   META_FIELD_QUANTIZER, META_FIELD_STATIC_GROUPS, META_FIELD_TRUE_SEQUENTIAL,
+                                   META_FIELD_URI, META_QUANTIZER_GPTQMODEL, META_VALUE_URI, MIN_VERSION_WITH_V2)
 from ..utils.backend import BACKEND
 from ..utils.logger import setup_logger
-from ..utils.model import (
-    convert_gptq_v2_to_v1_format,
-    copy_py_files,
-    find_modules,
-    get_model_files_size,
-    get_moe_layer_modules,
-    get_state_dict_for_save,
-    load_checkpoint_in_model_then_tie_weights,
-    make_quant,
-)
+from ..utils.model import (convert_gptq_v2_to_v1_format, copy_py_files, find_modules,
+                           get_model_files_size, get_moe_layer_modules, get_state_dict_for_save,
+                           load_checkpoint_in_model_then_tie_weights, make_quant)
 from ..utils.torch import torch_empty_cache
 from ..version import __version__
 from ._const import CPU, DEFAULT_MAX_SHARD_SIZE
 
-
 log = setup_logger()
 
 PROCESS_LOG_NAME = "process"
@@ -90,7 +72,7 @@ def save_pretrained(
 
     cls.save_pretrained = save_pretrained
 
-    def _eora_save(self, save_dir: str, model_save_dir: str):
+    def _eora_save(self, save_dir: str, model_save_dir: str = None):
         assert isinstance(self.quantize_config.adapter, Lora)
 
         assert hasattr(self, 'lora_results')