FileContent Detector should account for allowed patterns in talismanRC

.talismanrc

@@ -11,7 +11,7 @@ fileignoreconfig:
  ignore_detectors:
  - filecontent
 - filename: detector/filecontent/filecontent_detector_test.go
- checksum: affb25839a87476dcef4f4169ccb9b54b2d2f2437cef3aca24f4d3b69d5886c5
+ checksum: 05d00bb99452d37ab45de28c9e074357e5be22a80d95dad1b2cdd01074b25b2a
 - filename: detector/filename/filename_detector.go
  checksum: 5404565683a7e812fa98ff2d14237c4d1ba7dc5b4aca2dd3ba663b33dc8ddae7
 - filename: detector/filename/filename_detector_test.go
@@ -32,6 +32,8 @@ fileignoreconfig:
  checksum: bf0da9b8b6f779f502564166323a903a49b56b9d8df2597729a5b96c8f066074
 - filename: install.sh
  checksum: c909f6a1caefba3f196d489f9262608044be596a44793c2173ec55b98ecec649
+- filename: talismanrc/talismanrc_test.go
+ checksum: eab40d0745dc215267da86ef7f926be77c4d46e9a248dadbc7e52aa186e82853
 scopeconfig:
 - scope: go
 version: "1.0"

detector/filecontent/filecontent_detector.go

@@ -119,6 +119,7 @@ func (fc *FileContentDetector) Test(comparator helpers.ChecksumCompare, currentA
  data := []byte(content)
  addition.Data = data
  }
+ addition.Data = []byte(talismanRC.FilterAllowedPatternsFromAddition(addition))
  for _, ct := range contentTypes {
  contents <- content{
  name: addition.Name,

detector/filecontent/filecontent_detector_test.go

@@ -2,6 +2,7 @@ package filecontent
 
 import (
  "fmt"
+ "regexp"
  "strings"
  "talisman/detector/helpers"
  "talisman/detector/severity"
@@ -20,10 +21,10 @@ var emptyTalismanRC = &talismanrc.TalismanRC{IgnoreConfigs: []talismanrc.IgnoreC
 var defaultChecksumCompareUtility = helpers.
  NewChecksumCompare(nil, utility.MakeHasher("default", "."), emptyTalismanRC)
 var dummyCallback = func() {}
+var filename = "filename"
 
 func TestShouldNotFlagSafeText(t *testing.T) {
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("prettySafe"))}
 
  NewFileContentDetector(emptyTalismanRC).
@@ -33,7 +34,6 @@ func TestShouldNotFlagSafeText(t *testing.T) {
 
 func TestShouldIgnoreFileIfNeeded(t *testing.T) {
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("prettySafe"))}
  talismanRCIWithFilenameIgnore := &talismanrc.TalismanRC{
  IgnoreConfigs: []talismanrc.IgnoreConfig{
@@ -59,7 +59,6 @@ func TestShouldNotFlag4CharSafeText(t *testing.T) {
  the encoded value of i· rather just a plain abcd input see
  stackoverflow.com/questions/8571501/how-to-check-whether-the-string-is-base64-encoded-or-not#comment23919648_8571649*/
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte("abcd"))}
 
  NewFileContentDetector(emptyTalismanRC).
@@ -71,7 +70,6 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) {
  const lowEntropyString string = "YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWEK"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
  content := []byte(lowEntropyString)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
 
  NewFileContentDetector(emptyTalismanRC).
@@ -82,7 +80,6 @@ func TestShouldNotFlagLowEntropyBase64Text(t *testing.T) {
 func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) {
  const awsSecretAccessKey string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsSecretAccessKey))}
  filePath := additions[0].Path
 
@@ -99,7 +96,6 @@ func TestShouldFlagPotentialAWSSecretKeys(t *testing.T) {
 func TestShouldFlagPotentialSecretWithoutTrimmingWhenLengthLessThan50Characters(t *testing.T) {
  const secret string = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9asdfa"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(secret))}
  filePath := additions[0].Path
 
@@ -117,7 +113,6 @@ func TestShouldFlagPotentialJWT(t *testing.T) {
  "OiJDaHJpcyBTZXZpbGxlamEiLCJhZG1pbiI6dHJ1ZX0.03f329983b86f7d9a9f5fef85305880101d5e302afafa20154d094b229f757"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
  content := []byte(jwt)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
  filePath := additions[0].Path
 
@@ -141,7 +136,6 @@ func TestShouldFlagPotentialSecretsWithinJavaCode(t *testing.T) {
  "}"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
  content := []byte(dangerousJavaCode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)}
  filePath := additions[0].Path
 
@@ -162,7 +156,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) {
  " }\r\n\r\n" +
  "}"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(safeJavaCode))}
 
  NewFileContentDetector(emptyTalismanRC).
@@ -173,7 +166,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeJavaCode(t *testing.T) {
 func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) {
  safeLongMethodName := "TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithRidiculousHighEntropyWordsMightExist"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(safeLongMethodName))}
 
  NewFileContentDetector(emptyTalismanRC).
@@ -184,7 +176,6 @@ func TestShouldNotFlagPotentialSecretsWithinSafeLongMethodName(t *testing.T) {
 func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) {
  const hex string = "68656C6C6F20776F726C6421"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hex))}
  filePath := additions[0].Path
 
@@ -198,7 +189,6 @@ func TestShouldFlagPotentialSecretsEncodedInHex(t *testing.T) {
 func TestShouldNotFlagPotentialCreditCardNumberIfAboveThreshold(t *testing.T) {
  const creditCardNumber string = "340000000000009"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))}
  talismanRCWithThreshold := &talismanrc.TalismanRC{Threshold: severity.High}
  checksumCompareWithThreshold := helpers.
@@ -210,12 +200,44 @@ func TestShouldNotFlagPotentialCreditCardNumberIfAboveThreshold(t *testing.T) {
  assert.False(t, results.HasFailures(), "Expected no base64 detection when threshold is higher")
 }
 
+func TestShouldNotFlagPotentialSecretsIfIgnored(t *testing.T) {
+ const hex string = "68656C6C6F20776F726C6421"
+ talismanRCWithIgnores := &talismanrc.TalismanRC{
+ AllowedPatterns: []*regexp.Regexp{regexp.MustCompile("[0-9a-fA-F]*")}}
+ results := helpers.NewDetectionResults(talismanrc.HookMode)
+ additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hex))}
+
+ NewFileContentDetector(emptyTalismanRC).
+ Test(defaultChecksumCompareUtility, additions, talismanRCWithIgnores, results, dummyCallback)
+
+ assert.False(t, results.HasFailures(), "Expected file ignore allowed pattern for hex text")
+}
+
+func TestResultsShouldNotFlagCreditCardNumberIfSpecifiedInFileIgnores(t *testing.T) {
+ const creditCardNumber string = "340000000000009"
+ results := helpers.NewDetectionResults(talismanrc.HookMode)
+ fileIgnoreConfig := &talismanrc.FileIgnoreConfig{
+ FileName: filename, Checksum: "",
+ AllowedPatterns: []string{creditCardNumber},
+ }
+ talismanRCWithFileIgnore := &talismanrc.TalismanRC{
+ IgnoreConfigs: []talismanrc.IgnoreConfig{fileIgnoreConfig},
+ }
+ additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))}
+
+ NewFileContentDetector(emptyTalismanRC).
+ Test(defaultChecksumCompareUtility, additions, talismanRCWithFileIgnore, results, dummyCallback)
+
+ assert.False(t, results.HasFailures(), "Expected the creditcard number to be ignored based on talisman RC")
+
+}
+
+
 func TestResultsShouldContainHexTextsIfHexAndBase64ExistInFile(t *testing.T) {
  const hex string = "68656C6C6F20776F726C6421"
  const base64 string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
  const hexAndBase64 = hex + "\n" + base64
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hexAndBase64))}
  filePath := additions[0].Path
 
@@ -232,7 +254,6 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T)
  const base64 string = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
  const hexAndBase64 = hex + "\n" + base64
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(hexAndBase64))}
  filePath := additions[0].Path
 
@@ -248,7 +269,6 @@ func TestResultsShouldContainBase64TextsIfHexAndBase64ExistInFile(t *testing.T)
 func TestResultsShouldContainCreditCardNumberIfCreditCardNumberExistInFile(t *testing.T) {
  const creditCardNumber string = "340000000000009"
  results := helpers.NewDetectionResults(talismanrc.HookMode)
- filename := "filename"
  additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(creditCardNumber))}
  filePath := additions[0].Path
 

detector/pattern/pattern_detector.go

@@ -50,7 +50,7 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current
  ignoredFilePaths <- addition.Path
  return
  }
- detections := detector.secretsPattern.check(processAllowedPatterns(addition, ignoreConfig), ignoreConfig.Threshold)
+ detections := detector.secretsPattern.check(ignoreConfig.FilterAllowedPatternsFromAddition(addition), ignoreConfig.Threshold)
  matches <- match{name: addition.Name, path: addition.Path, detections: detections, commits: addition.Commits}
  }(addition)
  }
@@ -77,24 +77,6 @@ func (detector PatternDetector) Test(comparator helpers.ChecksumCompare, current
  }
 }
 
-func processAllowedPatterns(addition gitrepo.Addition, tRC *talismanrc.TalismanRC) string {
- additionPathAsString := string(addition.Path)
- // Processing global allowed patterns
- for _, pattern := range tRC.AllowedPatterns {
- addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
- }
-
- // Processing allowed patterns based on file path
- for _, ignoreConfig := range tRC.IgnoreConfigs {
- if ignoreConfig.GetFileName() == additionPathAsString {
- for _, pattern := range ignoreConfig.GetAllowedPatterns() {
- addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
- }
- }
- }
- return string(addition.Data)
-}
-
 func (detector PatternDetector) processIgnore(ignoredFilePath gitrepo.FilePath, result *helpers.DetectionResults) {
  log.WithFields(log.Fields{
  "filePath": ignoredFilePath,

talismanrc/talismanrc.go

@@ -164,6 +164,25 @@ func (tRC *TalismanRC) Deny(addition gitrepo.Addition, detectorName string) bool
  return false
 }
 
+//Strip git addition 
+func(tRC *TalismanRC) FilterAllowedPatternsFromAddition(addition gitrepo.Addition) string {
+ additionPathAsString := string(addition.Path)
+ // Processing global allowed patterns
+ for _, pattern := range tRC.AllowedPatterns {
+ addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
+ }
+
+ // Processing allowed patterns based on file path
+ for _, ignoreConfig := range tRC.IgnoreConfigs {
+ if ignoreConfig.GetFileName() == additionPathAsString {
+ for _, pattern := range ignoreConfig.GetAllowedPatterns() {
+ addition.Data = pattern.ReplaceAll(addition.Data, []byte(""))
+ }
+ }
+ }
+ return string(addition.Data)
+}
+
 func (tRC *TalismanRC) effectiveRules(detectorName string) []string {
  var result []string
  for _, ignore := range tRC.IgnoreConfigs {

talismanrc/talismanrc_test.go

@@ -4,6 +4,7 @@ import (
  "fmt"
  "io/ioutil"
  "os"
+ "regexp"
  "testing"
 
  "talisman/detector/severity"
@@ -47,6 +48,31 @@ func TestShouldIgnoreUnformattedFiles(t *testing.T) {
  setRepoFileReader(defaultRepoFileReader)
 }
 
+func TestShouldFilterAllowedPatternsFromAddition(t *testing.T) {
+ const hex string = "68656C6C6F20776F726C6421"
+ const fileContent string = "Prefix content" + hex
+ gitRepoAddition1 := testAdditionWithData("file1", []byte(fileContent))
+ talismanrc := &TalismanRC{AllowedPatterns: []*regexp.Regexp{regexp.MustCompile(hex)}}
+
+ fileContentFiltered := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition1)
+
+ assert.Equal(t, fileContentFiltered, "Prefix content")
+}
+
+func TestShouldFilterAllowedPatternsFromAdditionBasedOnFileConfig(t *testing.T) {
+ const hexContent string = "68656C6C6F20776F726C6421"
+ const fileContent string = "Prefix content" + hexContent
+ gitRepoAddition1 := testAdditionWithData("file1", []byte(fileContent))
+ gitRepoAddition2 := testAdditionWithData("file2", []byte(fileContent))
+ talismanrc := createTalismanRCWithFileIgnores("file1", "somedetector", []string{hexContent})
+
+ fileContentFiltered1 := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition1)
+ fileContentFiltered2 := talismanrc.FilterAllowedPatternsFromAddition(gitRepoAddition2)
+
+ assert.Equal(t, fileContentFiltered1, "Prefix content")
+ assert.Equal(t, fileContentFiltered2, fileContent)
+}
+
 func TestShouldConvertThresholdToValue(t *testing.T) {
  talismanRCContents := []byte("threshold: high")
  assert.Equal(t, newPersistedRC(talismanRCContents).Threshold, severity.High)
@@ -142,27 +168,34 @@ func assertDenies(line, ignoreDetector string, path string, t *testing.T) {
 }
 
 func assertDeniesDetector(line, ignoreDetector string, path string, detectorName string, t *testing.T) {
- assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector).Deny(testAddition(path), detectorName), "%s is expected to deny a file named %s.", line, path)
+ assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector, []string{}).Deny(testAddition(path), detectorName), "%s is expected to deny a file named %s.", line, path)
 }
 
 func assertAccepts(line, ignoreDetector string, path string, t *testing.T, detectorNames ...string) {
  assertAcceptsDetector(line, ignoreDetector, path, "someDetector", t)
 }
 
 func assertAcceptsDetector(line, ignoreDetector string, path string, detectorName string, t *testing.T) {
- assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector).Accept(testAddition(path), detectorName), "%s is expected to accept a file named %s.", line, path)
+ assert.True(t, createTalismanRCWithFileIgnores(line, ignoreDetector, []string{}).Accept(testAddition(path), detectorName), "%s is expected to accept a file named %s.", line, path)
 }
 
 func testAddition(path string) gitrepo.Addition {
  return gitrepo.NewAddition(path, make([]byte, 0))
 }
 
-func createTalismanRCWithFileIgnores(filename string, detector string) *TalismanRC {
+func testAdditionWithData(path string, content []byte) gitrepo.Addition {
+ return gitrepo.NewAddition(path, content)
+}
+
+func createTalismanRCWithFileIgnores(filename string, detector string, allowedPatterns []string) *TalismanRC {
  fileIgnoreConfig := &FileIgnoreConfig{}
  fileIgnoreConfig.FileName = filename
  if detector != "" {
  fileIgnoreConfig.IgnoreDetectors = []string{detector}
  }
+ if len(allowedPatterns) != 0 {
+ fileIgnoreConfig.AllowedPatterns = allowedPatterns
+ }
 
  return &TalismanRC{IgnoreConfigs: []IgnoreConfig{fileIgnoreConfig}}
 }