Merge pull request #897 from Codium-ai/tr/self_reflect

Tr/self reflect

README.md

@@ -43,6 +43,7 @@ CodiumAI PR-Agent aims to help efficiently review and handle pull requests, by p
 ### May 12, 2024
 Inspired by [AlphaCodium](https://github.com/Codium-ai/AlphaCodium) flow engineering scheme, PR-Agent now performs **self-reflection** on the code suggestions it provides,
 enabling to remove invalid suggestions, and score the valid ones. The suggestions will be presented sorted by their score, enabling to focus on the most important ones first.
+You can also choose to remove suggestions below a certain importance score threshold, by setting the `pr_code_suggestions.suggestions_score_threshold` [configuration](https://pr-agent-docs.codium.ai/tools/improve/#configuration-options).
 
 <kbd><img src="https://codium.ai/images/pr_agent/self_reflection1.png" width="512"></kbd>
 

docs/docs/tools/improve.md

@@ -90,6 +90,10 @@ Hence, the total number of suggestions is proportional to the number of chunks,
  <td><b>self_reflect_on_suggestions</b></td>
  <td>If set to true, the improve tool will calculate an importance score for each suggestion [1-10], and sort the suggestion labels group based on this score. Default is true.</td>
  </tr>
+ <tr>
+ <td><b>suggestions_score_threshold</b></td>
+ <td> Any suggestion with importance score less than this threshold will be removed. Default is 0. Highly recommend not to set this value above 7-8, since above it may clip relevant suggestions that can be useful. </td>
+ </tr>
  <tr>
  <td><b>enable_help_text</b></td>
  <td>If set to true, the tool will display a help text in the comment. Default is true.</td>

pr_agent/settings/configuration.toml

@@ -85,7 +85,9 @@ extra_instructions = ""
 rank_suggestions = false
 enable_help_text=false
 persistent_comment=false
+# suggestions scoring
 self_reflect_on_suggestions=true
+suggestions_score_threshold=0 # [0-10]. highly recommend not to set this value above 8, since above it may clip highly relevant suggestions
 # params for '/improve --extended' mode
 auto_extended_mode=true
 num_code_suggestions_per_chunk=4

pr_agent/tools/pr_code_suggestions.py

@@ -344,14 +344,15 @@ async def _prepare_prediction_extended(self, model: str) -> dict:
  data = {"code_suggestions": []}
  for i, predictions in enumerate(prediction_list):
  if "code_suggestions" in predictions:
+ score_threshold = max(1,get_settings().pr_code_suggestions.suggestions_score_threshold)
  for prediction in predictions["code_suggestions"]:
  try:
  if get_settings().pr_code_suggestions.self_reflect_on_suggestions:
  score = int(prediction["score"])
- if score > 0:
+ if score >= score_threshold:
  data["code_suggestions"].append(prediction)
  else:
- get_logger().info(f"Removing suggestions {i}, because score is {score}",
+ get_logger().info(f"Removing suggestions {i}, because score is {score}, and score_threshold is {score_threshold}",
  artifact=prediction)
  else:
  get_logger().error(f"Error getting PR diff, no code suggestions found in call {i + 1}")
@@ -517,11 +518,12 @@ def generate_summarized_suggestions(self, data: Dict) -> str:
  pr_body += f"Why: {suggestion['score_why']}\n\n"
  pr_body += f"</details>"
 
+ pr_body += f"</details>"
+
  # # add another column for 'score'
  if get_settings().pr_code_suggestions.self_reflect_on_suggestions:
  pr_body += f"</td><td align=center>{suggestion['score']}\n\n"
 
- pr_body += f"</details>"
  pr_body += f"</td></tr>"