[#944] Improve performance (#2108)

Currently, the authorship credit analysis process exceeds an hour in
duration, which is significantly longer than the mere 5 minutes required
when the feature is deactivated.

Let's speed up the performance by implementing a caching mechanism and
refining the dynamic programming algorithm utilized for computing the
Levenshtein distance.

src/main/java/reposense/authorship/analyzer/AuthorshipAnalyzer.java

@@ -2,6 +2,8 @@
 
 import java.nio.file.Paths;
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -41,6 +43,10 @@ public class AuthorshipAnalyzer {
  private static final String ADDED_LINE_SYMBOL = "+";
  private static final String DELETED_LINE_SYMBOL = "-";
 
+ private static final ConcurrentHashMap<String, String[]> GIT_LOG_CACHE = new ConcurrentHashMap<>();
+ private static final ConcurrentHashMap<String, ArrayList<ArrayList<String>>> GIT_DIFF_CACHE =
+ new ConcurrentHashMap<>();
+
  /**
  * Analyzes the authorship of {@code lineContent} in {@code filePath} based on {@code originalityThreshold}.
  * Returns {@code true} if {@code currentAuthor} should be assigned full credit, {@code false} otherwise.
@@ -85,32 +91,73 @@ public static boolean analyzeAuthorship(RepoConfiguration config, String filePat
  */
  private static CandidateLine getDeletedLineWithLowestOriginality(RepoConfiguration config, String filePath,
  String lineContent, String commitHash) {
- String gitLogResults = GitLog.getParentCommits(config.getRepoRoot(), commitHash);
- String[] parentCommits = gitLogResults.split(" ");
-
  CandidateLine lowestOriginalityLine = null;
 
+ String gitLogCacheKey = config.getRepoRoot() + commitHash;
+ String[] parentCommits;
+ if (GIT_LOG_CACHE.containsKey(gitLogCacheKey)) {
+ parentCommits = GIT_LOG_CACHE.get(gitLogCacheKey);
+ } else {
+ String gitLogResults = GitLog.getParentCommits(config.getRepoRoot(), commitHash);
+ parentCommits = gitLogResults.split(" ");
+ GIT_LOG_CACHE.put(gitLogCacheKey, parentCommits);
+ }
+
  for (String parentCommit : parentCommits) {
- // Generate diff between commit and parent commit
- String gitDiffResult = GitDiff.diffCommits(config.getRepoRoot(), parentCommit, commitHash);
- String[] fileDiffResultList = gitDiffResult.split(DIFF_FILE_CHUNK_SEPARATOR);
+ String gitDiffCacheKey = config.getRepoRoot() + parentCommit + commitHash;
+ ArrayList<String> fileDiffResultList;
+ ArrayList<String> preImageFilePathList;
+ ArrayList<String> postImageFilePathList;
+
+ if (GIT_DIFF_CACHE.containsKey(gitDiffCacheKey)) {
+ ArrayList<ArrayList<String>> cacheResults = GIT_DIFF_CACHE.get(gitDiffCacheKey);
+ fileDiffResultList = cacheResults.get(0);
+ preImageFilePathList = cacheResults.get(1);
+ postImageFilePathList = cacheResults.get(2);
+ } else {
+ fileDiffResultList = new ArrayList<>();
+ preImageFilePathList = new ArrayList<>();
+ postImageFilePathList = new ArrayList<>();
+
+ // Generate diff between commit and parent commit
+ String gitDiffResult = GitDiff.diffCommits(config.getRepoRoot(), parentCommit, commitHash);
+ String[] fileDiffResults = gitDiffResult.split(DIFF_FILE_CHUNK_SEPARATOR);
+
+ for (String fileDiffResult : fileDiffResults) {
+ Matcher filePathMatcher = FILE_CHANGED_PATTERN.matcher(fileDiffResult);
+ if (!filePathMatcher.find()) {
+ continue;
+ }
 
- for (String fileDiffResult : fileDiffResultList) {
- Matcher filePathMatcher = FILE_CHANGED_PATTERN.matcher(fileDiffResult);
- if (!filePathMatcher.find()) {
- continue;
+ // If file was added in the commit
+ String preImageFilePath = filePathMatcher.group(PRE_IMAGE_FILE_PATH_GROUP_NAME);
+ if (preImageFilePath.equals(FILE_ADDED_SYMBOL)) {
+ continue;
+ }
+
+ String postImageFilePath = filePathMatcher.group(POST_IMAGE_FILE_PATH_GROUP_NAME);
+
+ fileDiffResultList.add(fileDiffResult);
+ preImageFilePathList.add(preImageFilePath);
+ postImageFilePathList.add(postImageFilePath);
  }
 
- String preImageFilePath = filePathMatcher.group(PRE_IMAGE_FILE_PATH_GROUP_NAME);
- String postImageFilePath = filePathMatcher.group(POST_IMAGE_FILE_PATH_GROUP_NAME);
+ ArrayList<ArrayList<String>> cacheResults = new ArrayList<>();
+ cacheResults.add(fileDiffResultList);
+ cacheResults.add(preImageFilePathList);
+ cacheResults.add(postImageFilePathList);
+
+ GIT_DIFF_CACHE.put(gitDiffCacheKey, cacheResults);
+ }
 
- // If file was added in the commit or file name does not match
- if (preImageFilePath.equals(FILE_ADDED_SYMBOL) || !postImageFilePath.equals(filePath)) {
+ for (int i = 0; i < fileDiffResultList.size(); i++) {
+ // If file name does not match
+ if (!postImageFilePathList.get(i).equals(filePath)) {
  continue;
  }
 
  CandidateLine candidateLine = getDeletedLineWithLowestOriginalityInDiff(
- fileDiffResult, lineContent, parentCommit, preImageFilePath);
+ fileDiffResultList.get(i), lineContent, parentCommit, preImageFilePathList.get(i));
  if (candidateLine == null) {
  continue;
  }
@@ -152,7 +199,11 @@ private static CandidateLine getDeletedLineWithLowestOriginalityInDiff(String fi
 
  if (lineChanged.startsWith(DELETED_LINE_SYMBOL)) {
  String deletedLineContent = lineChanged.substring(DELETED_LINE_SYMBOL.length());
- double originalityScore = computeOriginalityScore(lineContent, deletedLineContent);
+ double lowestOriginalityScore = lowestOriginalityLine == null
+ ? Integer.MAX_VALUE
+ : lowestOriginalityLine.getOriginalityScore();
+ double originalityScore = computeOriginalityScore(lineContent, deletedLineContent,
+ lowestOriginalityScore);
 
  if (lowestOriginalityLine == null
  || originalityScore < lowestOriginalityLine.getOriginalityScore()) {
@@ -192,8 +243,8 @@ private static int getPreImageStartingLineNumber(String linesChangedHeader) {
  /**
  * Calculates the originality score of {@code s} with {@code baseString}.
  */
- private static double computeOriginalityScore(String s, String baseString) {
- double levenshteinDistance = StringsUtil.getLevenshteinDistance(s, baseString);
+ private static double computeOriginalityScore(String s, String baseString, double limit) {
+ double levenshteinDistance = StringsUtil.getLevenshteinDistance(s, baseString, limit * baseString.length());
  return levenshteinDistance / baseString.length();
  }
 

src/main/java/reposense/util/StringsUtil.java

@@ -94,35 +94,72 @@ public static boolean isNumeric(String string) {
 
  /**
  * Calculates the Levenshtein Distance between two strings using Dynamic Programming.
+ * Insertion, deletion, and substitution are all of cost 1.
+ * This version improves the space complexity down to O(min(s, t))
+ * <p></p>
+ * The dp will stop if the {@code limit} is reached, this means that if the final distance is 7 and the limit is set
+ * to 3, the algorithm ends early once it reaches 3. This is possible as we are using this method to find the string
+ * with the lowest Levenshtein distance.
+ * <p></p>
+ * Returns {@code Integer.MAX_VALUE} if limit is reached, else returns the computed Levenshtein distance.
  */
- public static int getLevenshteinDistance(String s, String t) {
- // dp[i][j] stores the distance between s.substring(0, i) and t.substring(0, j) -> distance(s[:i], t[:j])
- int[][] dp = new int[s.length() + 1][t.length() + 1];
+ public static int getLevenshteinDistance(String s, String t, double limit) {
+ // Early termination if either string is empty, lev dist is just the length of the other string.
+ if (s.isEmpty()) {
+ return t.length();
+ }
+
+ if (t.isEmpty()) {
+ return s.length();
+ }
 
- // Distance between a string and an empty string is the length of the string
- for (int i = 0; i <= s.length(); i++) {
- dp[i][0] = i;
+ // The final lev dist is at least k where k = difference in length = number of insert/delete.
+ if (Math.abs(s.length() - t.length()) >= limit) {
+ return Integer.MAX_VALUE;
  }
 
+ if (s.length() < t.length()) {
+ // Swap s and t to ensure s is always the longer string
+ String temp = s;
+ s = t;
+ t = temp;
+ }
+
+ int[] dp = new int[t.length() + 1];
  for (int i = 0; i <= t.length(); i++) {
- dp[0][i] = i;
+ dp[i] = i;
  }
 
  for (int i = 1; i <= s.length(); i++) {
+ // Store the value of the previous row's column
+ int prev = dp[0];
+ dp[0] = i;
+
+ // If for this row, all the values are at least k, then the final lev dist computed will also be at least k.
+ // hasLower will check for values smaller than the limit, and terminate early if limit is reached.
+ boolean hasLower = false;
+
  for (int j = 1; j <= t.length(); j++) {
- // If s[i-1] and t[j-1] are equal, distance(s[:i], t[:j]) equals to distance(s[:i-1], t[:j-1])
+ int temp = dp[j];
+
  if (s.charAt(i - 1) == t.charAt(j - 1)) {
- dp[i][j] = dp[i - 1][j - 1];
+ dp[j] = prev;
  } else {
- // distance(s[:i], t[:j]) is the minimum of:
- // 1) distance(s[:i-1], t[:j]) + 1 -> add s[i]
- // 2) distance(s[:i], t[:j-1]) + 1 -> add t[j]
- // 3) distance(s[:i-1], t[:j-1]) + 1 -> substitute s[i] with t[j]
- dp[i][j] = Math.min(dp[i - 1][j], Math.min(dp[i][j - 1], dp[i - 1][j - 1])) + 1;
+ dp[j] = Math.min(prev, Math.min(dp[j - 1], dp[j])) + 1;
+ }
+
+ prev = temp;
+
+ if (dp[j] < limit) {
+ hasLower = true;
  }
  }
+
+ if (!hasLower) {
+ return Integer.MAX_VALUE;
+ }
  }
 
- return dp[s.length()][t.length()];
+ return dp[t.length()];
  }
 }

src/test/java/reposense/util/StringsUtilTest.java

@@ -91,36 +91,52 @@ public void addQuotesForFilePath_specialBashCharacters_success() {
 
  @Test
  public void getLevenshteinDistance_success() {
- Assertions.assertEquals(4, StringsUtil.getLevenshteinDistance("potato", "tomatoes"));
+ Assertions.assertEquals(4, StringsUtil.getLevenshteinDistance("potato", "tomatoes", Integer.MAX_VALUE));
  }
 
  @Test
  public void getLevenshteinDistance_insertion_success() {
- Assertions.assertEquals(2, StringsUtil.getLevenshteinDistance("abcd", "abcdef"));
+ Assertions.assertEquals(2, StringsUtil.getLevenshteinDistance("abcd", "abcdef", Integer.MAX_VALUE));
  }
 
  @Test
  public void getLevenshteinDistance_deletion_success() {
- Assertions.assertEquals(3, StringsUtil.getLevenshteinDistance("abcde", "ab"));
+ Assertions.assertEquals(3, StringsUtil.getLevenshteinDistance("abcde", "ab", Integer.MAX_VALUE));
  }
 
  @Test
  public void getLevenshteinDistance_substitution_success() {
- Assertions.assertEquals(4, StringsUtil.getLevenshteinDistance("xxxxefg", "abcdefg"));
+ Assertions.assertEquals(4, StringsUtil.getLevenshteinDistance("xxxxefg", "abcdefg", Integer.MAX_VALUE));
  }
 
  @Test
  public void getLevenshteinDistance_identicalStrings_success() {
- Assertions.assertEquals(0, StringsUtil.getLevenshteinDistance("abcdefg", "abcdefg"));
+ Assertions.assertEquals(0, StringsUtil.getLevenshteinDistance("abcdefg", "abcdefg", Integer.MAX_VALUE));
  }
 
  @Test
  public void getLevenshteinDistance_emptyStrings_success() {
- Assertions.assertEquals(0, StringsUtil.getLevenshteinDistance("", ""));
+ Assertions.assertEquals(0, StringsUtil.getLevenshteinDistance("", "", Integer.MAX_VALUE));
  }
 
  @Test
  public void getLevenshteinDistance_emptyString_success() {
- Assertions.assertEquals(6, StringsUtil.getLevenshteinDistance("abcdef", ""));
+ Assertions.assertEquals(6, StringsUtil.getLevenshteinDistance("abcdef", "", Integer.MAX_VALUE));
+ }
+
+ @Test
+ public void getLevenshteinDistance_belowLimit_success() {
+ Assertions.assertEquals(4, StringsUtil.getLevenshteinDistance("xxxxefg", "abcdefg", 4.0001));
+ Assertions.assertEquals(4, StringsUtil.getLevenshteinDistance("xxxxefg", "abcdefg", 123.456));
+ Assertions.assertEquals(4, StringsUtil.getLevenshteinDistance("xxxxefg", "abcdefg", Integer.MAX_VALUE));
+ }
+
+ @Test
+ public void getLevenshteinDistance_exceedLimit_success() {
+ Assertions.assertEquals(Integer.MAX_VALUE, StringsUtil.getLevenshteinDistance("xxxxefg", "abcdefg", 4.000));
+ Assertions.assertEquals(Integer.MAX_VALUE, StringsUtil.getLevenshteinDistance("xxxxefg", "abcdefg", 3.99999));
+ Assertions.assertEquals(Integer.MAX_VALUE, StringsUtil.getLevenshteinDistance("xxxxefg", "abcdefg", 0.89014));
+
+ Assertions.assertEquals(Integer.MAX_VALUE, StringsUtil.getLevenshteinDistance("a", "1234567", 2.0));
  }
 }