Filter out special tokens and placeholder tokens for Phi-3 (#983)

pytorch · May 20, 2024 · 27f6872 · 27f6872
1 parent 4203311
commit 27f6872
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/torchtune/models/phi3/_sentencepiece.py b/torchtune/models/phi3/_sentencepiece.py
@@ -112,7 +112,9 @@ def decode(self, ids: List[int]) -> str:
  """
  ids_for_decode = []
  for token_id in ids:
- if token_id in self.special_tokens.values():
+ # Filter out special tokens and the placeholder tokens added
+ # by the Phi3 team
+ if token_id >= 32_000 and token_id <= 32_064:
  continue
  else:
  ids_for_decode.append(token_id)