Merge pull request #48 from GoGoJoestar/main

fix bug in gradio_demo.py when launch vLLM server
ymcui · Aug 2, 2023 · 2e0aca6 · 2e0aca6
2 parents cce5350 + 4e733da
commit 2e0aca6
Showing 1 changed file with 10 additions and 10 deletions.
diff --git a/scripts/inference/gradio_demo.py b/scripts/inference/gradio_demo.py
@@ -80,7 +80,7 @@
 parser.add_argument(
  "--post_host",
  type=str,
- default="localhost",
+ default="0.0.0.0",
  help="Host of vLLM service.")
 parser.add_argument(
  "--post_port",
@@ -128,15 +128,15 @@ def setup():
  tokenizer = LlamaTokenizer.from_pretrained(args.tokenizer_path, legacy=True)
 
  print("Start launch vllm server.")
- cmd = [
- "python -m vllm.entrypoints.api_server",
- f"--model={args.base_model}",
- f"--tokenizer={args.tokenizer_path}",
- "--tokenizer-mode=slow",
- f"--tensor-parallel-size={len(args.gpus.split(','))}",
- "&",
- ]
- subprocess.check_call(cmd)
+ cmd = f"python -m vllm.entrypoints.api_server \
+ --model={args.base_model} \
+ --tokenizer={args.tokenizer_path} \
+ --tokenizer-mode=slow \
+ --tensor-parallel-size={len(args.gpus.split(','))} \
+ --host {args.post_host} \
+ --port {args.post_port} \
+  &"
+ subprocess.check_call(cmd, shell=True)
  else:
  max_memory = args.max_memory
  port = args.port