[Feature] Add lmdeploy tis python backend model (#1014)

* add lmdeploy tis python backend model

* fix pr check

* update

configs/eval_internlm_chat_lmdeploy_tis.py

@@ -0,0 +1,41 @@
+from mmengine.config import read_base
+from opencompass.models.lmdeploy_tis import LmdeployTisModel
+
+with read_base():
+ # choose a list of datasets
+ from .datasets.mmlu.mmlu_gen_a484b3 import mmlu_datasets
+ from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
+ from .datasets.SuperGLUE_WiC.SuperGLUE_WiC_gen_d06864 import WiC_datasets
+ from .datasets.SuperGLUE_WSC.SuperGLUE_WSC_gen_7902a7 import WSC_datasets
+ from .datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
+ from .datasets.gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
+ from .datasets.humaneval.humaneval_gen_8e312c import humaneval_datasets
+ from .datasets.race.race_gen_69ee4f import race_datasets
+ from .datasets.crowspairs.crowspairs_gen_381af0 import crowspairs_datasets
+ # and output the results in a choosen format
+ from .summarizers.medium import summarizer
+
+datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
+
+meta_template = dict(
+ round=[
+ dict(role='HUMAN', begin='<|im_start|>user\n', end='<|im_end|>\n'),
+ dict(role='BOT', begin='<|im_start|>assistant\n', end='<|im_end|>\n', generate=True),
+ ],
+ eos_token_id=92542
+)
+
+models = [
+ dict(
+ type=LmdeployTisModel,
+ abbr='internlm-chat-20b-lmdeploy-tis',
+ path="internlm/internlm-chat-20b",
+ tis_addr='0.0.0.0:33337',
+ max_out_len=100,
+ max_seq_len=2048,
+ batch_size=8,
+ meta_template=meta_template,
+ run_cfg=dict(num_gpus=1, num_procs=1),
+ end_str='<|im_end|>',
+ )
+]

opencompass/models/__init__.py

@@ -18,6 +18,7 @@
 from .lightllm_api import LightllmAPI # noqa: F401
 from .llama2 import Llama2, Llama2Chat # noqa: F401, F403
 from .lmdeploy_pytorch import LmdeployPytorchModel # noqa: F401
+from .lmdeploy_tis import LmdeployTisModel # noqa: F401
 from .minimax_api import MiniMax # noqa: F401
 from .mistral_api import Mistral # noqa: F401
 from .mixtral import Mixtral # noqa: F401

opencompass/models/lmdeploy_tis.py

@@ -0,0 +1,200 @@
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from queue import Queue
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+
+from opencompass.models.base import BaseModel, LMTemplateParser
+from opencompass.utils.logging import get_logger
+from opencompass.utils.prompt import PromptList
+
+PromptType = Union[PromptList, str]
+
+
+def valid_str(string, coding='utf-8'):
+ """decode text according to its encoding type."""
+ invalid_chars = [b'\xef\xbf\xbd']
+ bstr = bytes(string, coding)
+ for invalid_char in invalid_chars:
+ bstr = bstr.replace(invalid_char, b'')
+ ret = bstr.decode(encoding=coding, errors='ignore')
+ return ret
+
+
+def prepare_tensor(name, input_tensor):
+ """Create grpcclient's InferInput instance according to a given tensor."""
+ import tritonclient.grpc as grpcclient
+ from tritonclient.utils import np_to_triton_dtype
+ t = grpcclient.InferInput(name, list(input_tensor.shape),
+ np_to_triton_dtype(input_tensor.dtype))
+ t.set_data_from_numpy(input_tensor)
+ return t
+
+
+def stream_callback(que, result, error):
+ """callback function invoked by triton client."""
+ que.put((result, error))
+
+
+class LmdeployTisModel(BaseModel):
+ """Model wrapper for LMDeploy Python Backend Triton Inference Server gRPC
+ API.
+
+ Args:
+ path (str): The name of OpenAI's model.
+ tis_addr (str): The address (ip:port format) of turbomind's
+ triton inference server
+ max_seq_len (int): The maximum allowed sequence length of a model.
+ Note that the length of prompt + generated tokens shall not exceed
+ this value. Defaults to 2048.
+ meta_template (Dict, optional): The model's meta prompt
+ template if needed, in case the requirement of injecting or
+ wrapping of any meta instructions.
+ """
+
+ is_api: bool = True
+
+ def __init__(self,
+ path: str,
+ tis_addr: str = '0.0.0.0:33337',
+ max_seq_len: int = 2048,
+ meta_template: Optional[Dict] = None,
+ end_str: Optional[str] = None):
+ super().__init__(path=path,
+ max_seq_len=max_seq_len,
+ meta_template=meta_template)
+ from lmdeploy.tokenizer import Tokenizer
+
+ self.logger = get_logger()
+ self.template_parser = LMTemplateParser(meta_template)
+ self.eos_token_id = None
+ if meta_template and 'eos_token_id' in meta_template:
+ self.eos_token_id = meta_template['eos_token_id']
+ self.tis_addr = tis_addr
+ self.tokenizer = Tokenizer(path)
+ self.end_str = end_str
+
+ def generate(
+ self,
+ inputs: List[str or PromptList],
+ max_out_len: int = 512,
+ temperature: float = 1.0,
+ ) -> List[str]:
+ """Generate results given a list of inputs.
+
+ Args:
+ inputs (List[str or PromptList]): A list of strings or PromptDicts.
+ The PromptDict should be organized in OpenCompass'
+ API format.
+ max_out_len (int): The maximum length of the output.
+ temperature (float): What sampling temperature to use,
+ between 0 and 2. Higher values like 0.8 will make the output
+ more random, while lower values like 0.2 will make it more
+ focused and deterministic. Defaults to 0.7.
+
+ Returns:
+ List[str]: A list of generated strings.
+ """
+
+ with ThreadPoolExecutor() as executor:
+ results = list(
+ executor.map(self._generate, inputs,
+ [max_out_len] * len(inputs),
+ [temperature] * len(inputs),
+ [self.end_str] * len(inputs)))
+ return results
+
+ def wait(self):
+ """Wait till the next query can be sent.
+
+ Applicable in both single-thread and multi-thread environments.
+ """
+ return self.token_bucket.get_token()
+
+ def get_token_len(self, prompt: str) -> int:
+ input_ids = self.tokenizer.encode(prompt)
+ return len(input_ids)
+
+ def _call_triton_server(self, prompt, tis_addr, session_id,
+ request_output_len, temperature, res_que):
+ import tritonclient.grpc as grpcclient
+
+ with grpcclient.InferenceServerClient(tis_addr) as client:
+ inputs = [
+ prepare_tensor('prompt',
+ np.array([prompt.encode()], dtype=np.object_)),
+ prepare_tensor('max_tokens',
+ np.array([request_output_len], dtype=np.int32)),
+ prepare_tensor('temperature',
+ np.array([temperature], dtype=np.float_)),
+ prepare_tensor('top_p', np.array([1.0], dtype=np.float_)),
+ prepare_tensor('top_k', np.array([1], dtype=np.int32)),
+ prepare_tensor('ignore_eos', np.array([False],
+ dtype=np.bool_)),
+ prepare_tensor('stream', np.array([True], dtype=np.bool_)),
+ ]
+
+ # async_stream
+ client.start_stream(partial(stream_callback, res_que))
+ client.async_stream_infer('lmdeploy_model',
+ inputs,
+ sequence_id=session_id,
+ sequence_start=True,
+ sequence_end=True)
+
+ res_que.put(None)
+ return
+
+ def _process_result(self, que):
+ text = ''
+ while True:
+ res = que.get()
+ if res is not None:
+ result, err = res
+ if err is not None:
+ print(err)
+ else:
+ res = result.as_numpy('response').item().decode()
+ text += res
+ else:
+ return text
+
+ def _generate(self,
+ prompt: str or PromptList,
+ max_out_len: int,
+ temperature: float,
+ end_str: Optional[str] = None) -> str:
+ """Generate results given a list of inputs.
+
+ Args:
+ prompt (str or PromptList): A string or PromptDict.
+ The PromptDict should be organized in OpenCompass'
+ API format.
+ max_out_len (int): The maximum length of the output.
+ temperature (float): What sampling temperature to use,
+ between 0 and 2. Higher values like 0.8 will make the output
+ more random, while lower values like 0.2 will make it more
+ focused and deterministic.
+
+ Returns:
+ str: The generated string.
+ """
+ assert type(
+ prompt
+ ) is str, 'We only support string for LMDeploy Python Backend TIS API'
+
+ res_que = Queue()
+
+ self._call_triton_server(prompt=prompt,
+ tis_addr=self.tis_addr,
+ session_id=threading.currentThread().ident,
+ request_output_len=max_out_len,
+ temperature=temperature,
+ res_que=res_que)
+ text = self._process_result(res_que)
+ response = valid_str(text)
+ if end_str:
+ response = response.split(end_str)[0]
+ return response