add funccall evalution features

Author: lightislost

scripts/run_fc_example.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+
+python src/qwen_eval_main.py

src/datasets/__init__.py

@@ -0,0 +1,9 @@
+from .base_dataset import ToolDataset
+from .toolfill_dataset import ToolFillDataset
+from .toolparser_dataset import ToolParserDataset
+from .toolsummary_dataset import ToolSummaryDataset
+from .funccall_dataset import FuncCallDataset
+
+__all__ = [
+    "ToolFillDataset", "ToolDataset", "ToolParserDataset", "ToolSummaryDataset", "FuncCallDataset"
+]

src/datasets/base_dataset.py

@@ -0,0 +1,40 @@
+from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
+from src.utils.json_utils import read_json_file, save_to_json_file
+
+
+
+class ToolDataset:
+    def __init__(self, dataset_name, tool_task, filepath):
+        self.dataset_name = dataset_name
+        self.tool_task = tool_task
+        self.filepath = filepath
+        self.datas = self.load_data()
+
+    def load_data(self, ) -> list:
+        if self.filepath:
+            return self.load_data_from_local(self.filepath)
+        elif self.dataset_name and self.tool_task:
+            return self.load_data_from_hf(self.tool_task)
+        return []
+
+    def load_data_from_local(self, filepath):
+        ''''''
+        pass
+        
+    def load_data_from_hf(self, tool_task):
+        pass
+        
+    def __iter__(self):
+        self.current_index = 0
+        return self
+
+    def __next__(self):
+        if self.current_index < len(self.datas):
+            current_item = self.datas[self.current_index]
+            self.current_index += 1
+            return current_item
+        else:
+            raise StopIteration
+
+    def __len__(self):
+        return len(self.datas)

src/datasets/funccall_dataset.py

@@ -0,0 +1,41 @@
+from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
+from src.utils.json_utils import read_json_file, save_to_json_file
+from .base_dataset import ToolDataset
+
+import os
+
+
+
+
+class FuncCallDataset(ToolDataset):
+    def __init__(self, dataset_name, tool_task, filepath):
+        self.dataset_name = dataset_name
+        self.tool_task = tool_task
+        self.filepath = filepath
+        self.datas = self.load_data()
+
+    def load_data(self, ) -> list:
+        if self.filepath:
+            return self.load_data_from_local(self.filepath)
+        elif self.dataset_name and self.tool_task:
+            return self.load_data_from_hf(self.tool_task)
+        return []
+
+    def load_data_from_local(self, filepath):
+        def _load_from_file(filename):
+            if "jsonl" in filename:
+                return read_jsonl_file(filename)
+            elif "json" in filename:
+                return read_json_file(filename)
+
+        datas = []
+        if os.path.isdir(filepath):
+            for filename in os.listdir(filepath):
+                datas.extend(_load_from_file(os.path.join(filepath, filename)))
+        else:
+            datas = _load_from_file(filepath)
+
+        return datas
+        
+    def load_data_from_hf(self, tool_task):
+        pass

src/datasets/toolfill_dataset.py

@@ -0,0 +1,30 @@
+from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
+from src.utils.json_utils import read_json_file, save_to_json_file
+from .base_dataset import ToolDataset
+
+
+
+
+class ToolFillDataset(ToolDataset):
+    def __init__(self, dataset_name, tool_task, filepath):
+        self.dataset_name = dataset_name
+        self.tool_task = tool_task
+        self.filepath = filepath
+        self.datas = self.load_data()
+
+    def load_data(self, ) -> list:
+        if self.filepath:
+            return self.load_data_from_local(self.filepath)
+        elif self.dataset_name and self.tool_task:
+            return self.load_data_from_hf(self.tool_task)
+        return []
+
+    def load_data_from_local(self, filepath):
+        if "jsonl" in filepath:
+            return read_jsonl_file(filepath)
+        elif "json" in filepath:
+            return read_json_file(filepath)
+        return []
+        
+    def load_data_from_hf(self, tool_task):
+        pass

src/datasets/toolparser_dataset.py

@@ -0,0 +1,30 @@
+from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
+from src.utils.json_utils import read_json_file, save_to_json_file
+from .base_dataset import ToolDataset
+
+
+
+
+class ToolParserDataset(ToolDataset):
+    def __init__(self, dataset_name, tool_task, filepath):
+        self.dataset_name = dataset_name
+        self.tool_task = tool_task
+        self.filepath = filepath
+        self.datas = self.load_data()
+
+    def load_data(self, ) -> list:
+        if self.filepath:
+            return self.load_data_from_local(self.filepath)
+        elif self.dataset_name and self.tool_task:
+            return self.load_data_from_hf(self.tool_task)
+        return []
+
+    def load_data_from_local(self, filepath):
+        if "jsonl" in filepath:
+            return read_jsonl_file(filepath)
+        elif "json" in filepath:
+            return read_json_file(filepath)
+        return []
+        
+    def load_data_from_hf(self, tool_task):
+        pass

src/datasets/toolsummary_dataset.py

@@ -0,0 +1,28 @@
+from src.utils.jsonl_utils import read_jsonl_file, save_to_jsonl_file
+from src.utils.json_utils import read_json_file, save_to_json_file
+from .base_dataset import ToolDataset
+
+
+class ToolSummaryDataset(ToolDataset):
+    def __init__(self, dataset_name, tool_task, filepath):
+        self.dataset_name = dataset_name
+        self.tool_task = tool_task
+        self.filepath = filepath
+        self.datas = self.load_data()
+
+    def load_data(self, ) -> list:
+        if self.filepath:
+            return self.load_data_from_local(self.filepath)
+        elif self.dataset_name and self.tool_task:
+            return self.load_data_from_hf(self.tool_task)
+        return []
+
+    def load_data_from_local(self, filepath):
+        if "jsonl" in filepath:
+            return read_jsonl_file(filepath)
+        elif "json" in filepath:
+            return read_json_file(filepath)
+        return []
+        
+    def load_data_from_hf(self, tool_task):
+        pass

src/evals/__init__.py

@@ -0,0 +1,10 @@
+from .base_evalution import ToolEvalution
+from .toolfill_evalution import ToolFillEvalution
+from .toolparser_evalution import ToolParserEvalution
+from .toolsummary_evalution import ToolSummaryEvalution
+from .func_call_evalution import FuncCallEvalution
+
+
+__all__ = [
+    "ToolEvalution", "ToolFillEvalution", "ToolParserEvalution", "ToolSummaryEvalution", "FuncCallEvalution"
+]

src/evals/base_evalution.py

@@ -0,0 +1,43 @@
+from src.models.base_model import ToolModel
+from src.models.generate_configs import GenerateConfigs
+from src.datasets import ToolFillDataset
+
+
+
+class ToolEvalution:
+    def __init__(
+        self, 
+        model: ToolModel,
+        dataset: ToolFillDataset,
+        base_prompt: str = '',
+        generate_configs: GenerateConfigs = None,
+    ):
+        self.model = model
+        self.dataset = dataset
+        self.base_prompt = base_prompt
+        self.generate_configs = generate_configs
+        
+        if not isinstance(model, ToolModel):
+            raise BaseException(f"must be ToolModel Class! not {model}")
+            
+    def calc(self):
+        '''开始计算结果'''
+        self.predicts = []
+        for idx, data in enumerate(self.dataset):
+            # if idx >= 5: break
+            prompt = self.base_prompt.format(**data)
+            answer = data["api_param"]
+            predict = self.generate(prompt, self.generate_configs)
+            self.predicts.append({"prompt": prompt, "predict": predict, "answer": answer})
+        
+        metric = self.eval_metric(self.predicts)
+        return metric
+        
+    def generate(self, prompt, generate_configs):
+        '''返回结果'''
+        return self.model.generate(prompt, generate_configs)
+    
+    def eval_metric(self, datas):
+        '''calc custom metric'''
+        pass
+