[SIG] add LogiQA dataset

configs/datasets/LogiQA/LogiQA_en_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+ from .LogiQA_en_ppl_20dfb3 import LogiQA_en_datasets # noqa: F401, F403

configs/datasets/LogiQA/LogiQA_en_ppl_20dfb3.py

@@ -0,0 +1,48 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import FixKRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import LogiQADataset
+
+
+_hint = "The following are logical reasoning questions from the Chinese National " \
+ "Civil Service Examination. Please choose the correct answer.\n"
+LogiQA_en_infer_cfg = dict(
+ ice_template=dict(
+ type=PromptTemplate,
+ template="Context: {context}\nQuery: {query}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: {correct_option}",
+ ),
+ prompt_template=dict(
+ type=PromptTemplate,
+ template={
+ answer:
+ f"{_hint}</E>Context: {{context}}\nQuery: {{query}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nAnswer: {answer}"
+ for answer in ['A', 'B', 'C', 'D']
+ },
+ ice_token='</E>',
+ ),
+ retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+ inferencer=dict(type=PPLInferencer))
+
+LogiQA_en_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+
+LogiQA_en_datasets = []
+for _split in ["validation", "test"]:
+
+ LogiQA_en_reader_cfg = dict(
+ input_columns=['context', 'query', 'A', 'B', 'C', 'D'],
+ output_column='correct_option',
+ test_split=_split
+ )
+
+ LogiQA_en_datasets.append(
+ dict(
+ abbr=f'LogiQA_en-{_split}',
+ type=LogiQADataset,
+ path='lucasmccabe/logiqa',
+ reader_cfg=LogiQA_en_reader_cfg,
+ infer_cfg=LogiQA_en_infer_cfg,
+ eval_cfg=LogiQA_en_eval_cfg
+ )
+ )

configs/datasets/LogiQA/LogiQA_zh_ppl.py

@@ -0,0 +1,4 @@
+from mmengine.config import read_base
+
+with read_base():
+ from .LogiQA_zh_ppl_19fd62 import LogiQA_zh_datasets # noqa: F401, F403

configs/datasets/LogiQA/LogiQA_zh_ppl_19fd62.py

@@ -0,0 +1,47 @@
+from opencompass.openicl.icl_prompt_template import PromptTemplate
+from opencompass.openicl.icl_retriever import FixKRetriever
+from opencompass.openicl.icl_inferencer import PPLInferencer
+from opencompass.openicl.icl_evaluator import AccEvaluator
+from opencompass.datasets import LogiQADataset
+
+
+_hint = "以下是中国国家公务员考试的逻辑推理题，请选出其中的正确答案。\n"
+LogiQA_zh_infer_cfg = dict(
+ ice_template=dict(
+ type=PromptTemplate,
+ template="上下文信息: {context}\n提问: {query}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\n答案: {correct_option}",
+ ),
+ prompt_template=dict(
+ type=PromptTemplate,
+ template={
+ answer:
+ f"{_hint}</E>上下文信息: {{context}}\n提问: {{query}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\n答案: {answer}"
+ for answer in ['A', 'B', 'C', 'D']
+ },
+ ice_token='</E>',
+ ),
+ retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+ inferencer=dict(type=PPLInferencer))
+
+LogiQA_zh_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
+
+
+LogiQA_zh_datasets = []
+for _split in ["validation", "test"]:
+
+ LogiQA_zh_reader_cfg = dict(
+ input_columns=['context', 'query', 'A', 'B', 'C', 'D'],
+ output_column='correct_option',
+ test_split=_split
+ )
+
+ LogiQA_zh_datasets.append(
+ dict(
+ abbr=f'LogiQA_zh-{_split}',
+ type=LogiQADataset,
+ path='jiacheng-ye/logiqa-zh',
+ reader_cfg=LogiQA_zh_reader_cfg,
+ infer_cfg=LogiQA_zh_infer_cfg,
+ eval_cfg=LogiQA_zh_eval_cfg
+ )
+ )

opencompass/datasets/__init__.py

@@ -43,6 +43,7 @@
 from .lawbench import * # noqa: F401, F403
 from .lcsts import * # noqa: F401, F403
 from .leval import * # noqa: F401, F403
+from .logiqa import * # noqa: F401, F403
 from .longbench import * # noqa: F401, F403
 from .math import * # noqa: F401, F403
 from .mathbench import * # noqa: F401, F403

opencompass/datasets/logiqa.py

@@ -0,0 +1,25 @@
+from datasets import load_dataset
+
+from opencompass.registry import LOAD_DATASET
+
+from .base import BaseDataset
+
+
+@LOAD_DATASET.register_module()
+class LogiQADataset(BaseDataset):
+
+ @staticmethod
+ def load(path: str):
+ dataset = load_dataset(path=path)
+
+ def preprocess(example):
+ options = example.pop('options')
+ example['A'] = options[0]
+ example['B'] = options[1]
+ example['C'] = options[2]
+ example['D'] = options[3]
+ example['correct_option'] = chr(65 + example['correct_option'])
+ return example
+
+ dataset = dataset.map(preprocess)
+ return dataset