Show prevalence of rules in the output

CHANGELOG.md

@@ -8,6 +8,7 @@
 - binja: add support for forwarded exports #1646 @xusheng6
 - binja: add support for symtab names #1504 @xusheng6
 - add com class/interface features #322 @Aayush-goel-04
+- Show prevalence of rules in the output #520 @Aayush-Goel-04
 
 ### Breaking Changes
 

capa/render/default.py

@@ -7,6 +7,7 @@
 # See the License for the specific language governing permissions and limitations under the License.
 
 import collections
+from typing import Dict
 
 import tabulate
 
@@ -72,36 +73,67 @@ def rec(match: rd.Match):
 
 def render_capabilities(doc: rd.ResultDocument, ostream: StringIO):
  """
+render capabilities sorted by:
+- prevalence rare -> unknown
+- namespace a -> z
+
  example::
 
- +-------------------------------------------------------+-------------------------------------------------+
- | CAPABILITY | NAMESPACE |
- |-------------------------------------------------------+-------------------------------------------------|
- | check for OutputDebugString error (2 matches) | anti-analysis/anti-debugging/debugger-detection |
- | read and send data from client to server | c2/file-transfer |
- | ... | ... |
- +-------------------------------------------------------+-------------------------------------------------+
+ +-------------------------------------------------------+-------------------------------------------------+------------+
+ | CAPABILITY | NAMESPACE | PREVALENCE |
+ |-------------------------------------------------------+-------------------------------------------------|------------|
+ | check for OutputDebugString error (2 matches) | anti-analysis/anti-debugging/debugger-detection | rare |
+ | ... | ... | ... |
+ |-------------------------------------------------------|-------------------------------------------------|------------|
+ | read and send data from client to server | c2/file-transfer | common |
+ | ... | ... | ... |
+ +-------------------------------------------------------+-------------------------------------------------+------------+
  """
  subrule_matches = find_subrule_matches(doc)
 
- rows = []
+ # seperate rules based on their prevalence
+ common: Dict[str, str] = {"capability": "", "namespace": "", "prevalence": ""}
+ had_common = False
+ rare: Dict[str, str] = {"capability": "", "namespace": "", "prevalence": ""}
+ had_rare = False
+
  for rule in rutils.capability_rules(doc):
  if rule.meta.name in subrule_matches:
- # rules that are also matched by other rules should not get rendered by default.
- # this cuts down on the amount of output while giving approx the same detail.
- # see #224
  continue
 
  count = len(rule.matches)
  if count == 1:
  capability = rutils.bold(rule.meta.name)
  else:
  capability = f"{rutils.bold(rule.meta.name)} ({count} matches)"
- rows.append((capability, rule.meta.namespace))
+
+ namespace = rule.meta.namespace if rule.meta.namespace is not None else ""
+ prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown"
+
+ if "rare" in prevalence:
+ rare["capability"] += capability + "\n"
+ rare["namespace"] += namespace + "\n"
+ rare["prevalence"] += prevalence + "\n"
+ had_rare = True
+ else:
+ common["capability"] += capability + "\n"
+ common["namespace"] += namespace + "\n"
+ common["prevalence"] += prevalence + "\n"
+ had_common = True
+
+ rows = []
+ if had_rare:
+ rows.append((rare["capability"], rare["namespace"], rare["prevalence"]))
+ if had_common:
+ rows.append((common["capability"], common["namespace"], common["prevalence"]))
 
  if rows:
  ostream.write(
- tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline")
+ tabulate.tabulate(
+ rows,
+ headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)],
+ tablefmt="mixed_grid",
+ )
  )
  ostream.write("\n")
  else:

capa/render/result_document.py

@@ -5,6 +5,8 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
+import gzip
+import json
 import datetime
 import collections
 from typing import Dict, List, Tuple, Union, Literal, Optional
@@ -22,6 +24,13 @@
 from capa.engine import MatchResults
 from capa.helpers import assert_never
 
+try:
+ from functools import lru_cache
+except ImportError:
+ # need to type ignore this due to mypy bug here (duplicate name):
+ # https://github.com/python/mypy/issues/1153
+ from backports.functools_lru_cache import lru_cache # type: ignore
+
 
 class FrozenModel(BaseModel):
  model_config = ConfigDict(frozen=True, extra="forbid")
@@ -501,9 +510,23 @@ class MaecMetadata(FrozenModel):
  model_config = ConfigDict(frozen=True, populate_by_name=True)
 
 
+@lru_cache(maxsize=None)
+def load_rules_prevalence() -> Dict[str, str]:
+ CD = Path(__file__).resolve().parent.parent.parent
+ file = CD / "assets" / "rules_prevalence_data" / "rules_prevalence.json.gz"
+ if not file.exists():
+ raise FileNotFoundError(f"File '{file}' not found.")
+ try:
+ with gzip.open(file, "rb") as gzfile:
+ return json.loads(gzfile.read().decode("utf-8"))
+ except Exception as e:
+ raise RuntimeError(f"An error occurred while loading '{file}': {e}")
+
+
 class RuleMetadata(FrozenModel):
  name: str
  namespace: Optional[str] = None
+ prevalence: str
  authors: Tuple[str, ...]
  scope: capa.rules.Scope
  attack: Tuple[AttackSpec, ...] = Field(alias="att&ck")
@@ -521,6 +544,7 @@ def from_capa(cls, rule: capa.rules.Rule) -> "RuleMetadata":
  return cls(
  name=rule.meta.get("name"),
  namespace=rule.meta.get("namespace"),
+ prevalence=load_rules_prevalence().get(rule.meta.get("name"), "unknown"),
  authors=rule.meta.get("authors"),
  scope=capa.rules.Scope(rule.meta.get("scope")),
  attack=tuple(map(AttackSpec.from_str, rule.meta.get("att&ck", []))),

capa/render/utils.py

@@ -5,7 +5,6 @@
 # Unless required by applicable law or agreed to in writing, software distributed under the License
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
-
 import io
 from typing import Union, Iterator
 

capa/render/verbose.py

@@ -139,6 +139,9 @@ def render_rules(ostream, doc: rd.ResultDocument):
 
  rows.append((key, v))
 
+ prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown"
+ rows.insert(1, ("prevalence", prevalence))
+
  if rule.meta.scope != capa.rules.FILE_SCOPE:
  locations = [m[0] for m in doc.rules[rule.meta.name].matches]
  rows.append(("matches", "\n".join(map(format_address, locations))))

capa/render/vverbose.py

@@ -305,6 +305,9 @@ def render_rules(ostream, doc: rd.ResultDocument):
  # library rules should not have a namespace
  rows.append(("namespace", rule.meta.namespace))
 
+ prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown"
+ rows.append(("prevalence", prevalence))
+
  if rule.meta.maec.analysis_conclusion or rule.meta.maec.analysis_conclusion_ov:
  rows.append(
  (