Calculate IsLower and IsAlpha sets in advance

sacremoses/tokenize.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import re
+from itertools import chain
 
 from sacremoses.corpus import Perluniprops
 from sacremoses.corpus import NonbreakingPrefixes
@@ -30,6 +31,9 @@ class MosesTokenizer(object):
  )
  IsLower = str("".join(perluniprops.chars("IsLower")))
 
+ AlphaChars = frozenset(chain(perluniprops.chars("IsAlpha"), VIRAMAS, NUKTAS))
+ LowerChars = frozenset(perluniprops.chars("IsLower"))
+
  # Remove ASCII junk.
  DEDUPLICATE_SPACE = re.compile(r"\s+"), r" "
  ASCII_JUNK = re.compile(r"[\000-\037]"), r""
@@ -357,10 +361,10 @@ def restore_multidots(self, text):
  return re.sub(r"DOTMULTI", r".", text)
 
  def islower(self, text):
- return not set(text).difference(set(self.IsLower))
+ return bool(set(text) <= self.LowerChars)
 
  def isanyalpha(self, text):
- return any(set(text).intersection(set(self.IsAlpha)))
+ return bool(set(text) & self.AlphaChars)
 
  def has_numeric_only(self, text):
  return bool(re.search(r"[\s]+(\#NUMERIC_ONLY\#)", text))