update code

Author: 温进

.DS_Store

@@ -0,0 +1,18 @@
+# model_path: 要测试的模型路径
+# model_name: 模型配置文件对应的模型命名
+# model_conf_path: 模型配置文件的地址，一般就为 conf 路径下的 devopseval_dataset_fp.json
+# eval_dataset_list: 要测试的数据集名称，默认 all，全部测试，如果需要测试单个或者多个，用 # 符号链接，示例：dataset1#dataset2
+# eval_dataset_fp_conf_path: 数据集配置地址
+# eval_dataset_type: 测试哪种类型，只支持默认 test 类型的测试集
+# data_path: 评测数据集地址，填写下载数据集后的地址就可以
+# k_shot: 支持 0-5，代表 few-shot 会给模型前缀加的示例数量
+
+python src/run_eval.py \
+    --model_path path_to_model \
+    --model_name model_name_in_conf \
+    --model_conf_path path_to_model_conf \
+    --eval_dataset_list all \
+    --eval_dataset_fp_conf_path path_to_dataset_conf \
+    --eval_dataset_type test \
+    --data_path path_to_downloaded_devops_eval_data \
+    --k_shot 0

scripts/run_eval.sh

@@ -15,7 +15,7 @@ def evaluate(model, tokenizer, context_builder, all_dataset):
         for question in dataset:
             if do_verbose:
                 question['pred'] = get_pred(model, tokenizer, context_builder, question, do_verbose)
-                do_verbose = True
+                do_verbose = False
             else:
                 question['pred'] = get_pred(model, tokenizer, context_builder, question, do_verbose)
     return all_dataset_pred
@@ -35,8 +35,6 @@ def get_pred(model, tokenizer, context_builder, question: dict, verbose: bool =
             option_dict[option] = encoded
         else:
             option_dict[option] = tokenizer._convert_token_to_id(option)
-            
-    logger.debug('option_dict={}'.format(option_dict))
 
     # build context
     raw_text, context_tokens = context_builder.make_context(model, tokenizer, query)

scripts/run_eval_example.sh

@@ -38,7 +38,7 @@ class EvaluateArguments:
     )
     k_shot: int = field(
         default = 0,
-        metadata={"help": "k-shot test, k should be in (0, 1,2,3,4,5)]"}
+        metadata={"help": "k-shot test, k should be in (0, 1,2,3,4,5)"}
     )
     seed: int = field(
         default = 100,

src/evaluate/evaluate.py

@@ -1,6 +1,7 @@
 import torch
 import transformers
 from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
+from loguru import logger
 
 
 class ModelAndTokenizerLoader:
@@ -14,6 +15,8 @@ def load_model_and_tokenizer(self, model_path: str):
 
     def load_model(self, model_path: str):
         model =  AutoModelForCausalLM.from_pretrained(model_path, device_map="auto", trust_remote_code=True).eval()
+        # for name, param in model.named_parameters():
+        #     logger.debug('param_name={}, param.device={}'.format(name, param.device))
         return model
 
     def load_tokenizer(self, model_path: str):

src/hparams/evaluate_args.py

@@ -19,7 +19,7 @@ def run_eval(args=None):
 
     # Get all dataset
     eval_datasets = load_all_dataset(eval_args)
-    logger.info('Load all dataset success')
+    logger.info('Load all dataset success, total question number={}'.format(sum(len(v) for v in eval_datasets.values())))
 
     # Load model and tokenizer
     model, tokenizer = load_model_and_tokenizer(eval_args)
@@ -35,7 +35,7 @@ def run_eval(args=None):
 
     # get metric
     score_dict = get_acc_score(all_pred)
-    logger.info('model_path={}\nk_shot={}\nEvaluation result={}'.format(eval_args.model_path, eval_args.k_shot, score_dict))
+    logger.info('model_path={} k_shot={} Evaluation result={}'.format(eval_args.model_path, eval_args.k_shot, score_dict))
 
     # save metric