Initial commit of modular publishing system

core/event.go

@@ -0,0 +1,48 @@
+package core
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+)
+
+// A MatchEvent holds a list of matches for a particular Url and signature
+type MatchEvent struct {
+ Url string
+ Matches []string
+ Signature string
+ File string
+ Stars int
+ Source GitResourceType
+}
+
+// Line returns a string containing the following: Url, Signature, File, and
+// Matches
+func (m MatchEvent) String() string {
+ var b strings.Builder
+
+ b.WriteString(fmt.Sprintf("Url: %s ", m.Url))
+ b.WriteString(fmt.Sprintf("Signature: %s ", m.Signature))
+ b.WriteString(fmt.Sprintf("File: %s ", m.File))
+ b.WriteString(fmt.Sprintf("Matches: %s\n", strings.Join(m.Matches, ", ")))
+
+ return b.String()
+}
+
+// Line returns a slice of strings containing the following: Url, Signature,
+// File, and Matches
+func (m MatchEvent) Line() []string {
+ return []string{m.Url, m.Signature, m.File, strings.Join(m.Matches, ", ")}
+}
+
+// Json returns a JSON formatted string that includes all of the data in a
+// MatchEvent.
+func (m MatchEvent) Json() string {
+ b, err := json.Marshal(m)
+ if err != nil {
+ LogIfError("unable to create JSON, %s", err)
+ return ""
+ }
+
+ return string(b)
+}

core/options.go

@@ -19,6 +19,7 @@ type Options struct {
  ProcessGists *bool
  TempDirectory *string
  CsvPath *string
+ Delimiter *string
  SearchQuery *string
  Local *string
  Live *string
@@ -39,6 +40,7 @@ func ParseOptions() (*Options, error) {
  ProcessGists: flag.Bool("process-gists", true, "Will watch and process Gists. Set to false to disable."),
  TempDirectory: flag.String("temp-directory", filepath.Join(os.TempDir(), Name), "Directory to process and store repositories/matches"),
  CsvPath: flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable"),
+ Delimiter: flag.String("delimiter", ",", "Delimiter for CSV file."),
  SearchQuery: flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)"),
  Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively. No need to have Githib tokens with local run."),
  Live: flag.String("live", "", "Your shhgit live endpoint"),

core/publish.go

@@ -0,0 +1,105 @@
+package core
+
+import (
+ "bytes"
+ "encoding/csv"
+ "fmt"
+ "net/http"
+ "os"
+ "time"
+)
+
+type Publisher interface {
+ Publish(m MatchEvent) error
+}
+
+// WebSource represents a web-based publisher
+type WebSource struct {
+ client *http.Client
+ endpoint string
+ method string
+ contentType string
+}
+
+// Publish writes a MatchEvent to the WebSource.
+func (w *WebSource) Publish(m MatchEvent) error {
+ var data string
+
+ switch w.contentType {
+ case "application/json":
+ data = m.Json()
+ // Add new cases for additional content types
+ default:
+ // Nothing
+ }
+
+ req, err := http.NewRequest(w.method, w.endpoint, bytes.NewBufferString(data))
+ if err != nil {
+ return err
+ }
+
+ req.Header.Add("Content-Type", w.contentType)
+ _, err = w.client.Do(req)
+ if err != nil {
+ return err
+ }
+
+ // May need to capture the response and check for status codes like 404 or
+ // 503 so that errors can be returned if needed.
+
+ return nil
+}
+
+// Returns a new WebSource that will send MatchEvents to the given enpoint
+// using the given method and content type.
+func NewWebSource(endpoint, method, contentType string) (WebSource, error) {
+ var w WebSource
+
+ if !(method == "POST" || method == "GET") {
+ return w, fmt.Errorf("method must be POST or GET, received %s", method)
+ }
+
+ w.client = &http.Client{Timeout: 10 * time.Second}
+ w.endpoint = endpoint
+ w.method = method
+ w.contentType = contentType
+
+ return w, nil
+}
+
+// DelimitedSource represents a Comma or Tab delimited publisher.
+type DelimitedSource struct {
+ writer *csv.Writer
+}
+
+// Publish writes a MatchEvent to the file.
+func (w *DelimitedSource) Publish(m MatchEvent) error {
+ err := w.writer.Write(m.Line())
+ if err != nil {
+ return err
+ }
+
+ w.writer.Flush()
+
+ return nil
+}
+
+// NewDelimitedSource returns a new comma or tab delimited writer. If a header
+// is provided, it is written to the file.
+func NewDelimitedSource(filename string, delimiter rune, header []string) (DelimitedSource, error) {
+ var d DelimitedSource
+
+ file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return d, err
+ }
+
+ d.writer = csv.NewWriter(file)
+ d.writer.Comma = delimiter
+
+ if header != nil {
+ d.writer.Write(header)
+ }
+
+ return d, nil
+}

core/session.go

@@ -2,7 +2,6 @@ package core
 
 import (
  "context"
- "encoding/csv"
  "fmt"
  "math/rand"
  "os"
@@ -28,7 +27,7 @@ type Session struct {
  Context context.Context
  Clients chan *GitHubClientWrapper
  ExhaustedClients chan *GitHubClientWrapper
- CsvWriter  *csv.Writer
+ Publishers []Publisher
 }
 
 var (
@@ -44,7 +43,7 @@ func (s *Session) Start() {
  s.InitThreads()
  s.InitSignatures()
  s.InitGitHubClients()
- s.InitCsvWriter()
+ s.InitPublishers()
 }
 
 func (s *Session) InitLogger() {
@@ -130,33 +129,32 @@ func (s *Session) InitThreads() {
  runtime.GOMAXPROCS(*s.Options.Threads + 1)
 }
 
-func (s *Session) InitCsvWriter() {
- if *s.Options.CsvPath == "" {
- return
- }
-
- writeHeader := false
- if !PathExists(*s.Options.CsvPath) {
- writeHeader = true
- }
+func (s *Session) InitPublishers() {
+ header := []string{"Repository name", "Signature name", "Matching file", "Matches"}
 
- file, err := os.OpenFile(*s.Options.CsvPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
- LogIfError("Could not create/open CSV file", err)
-
- s.CsvWriter = csv.NewWriter(file)
-
- if writeHeader {
- s.WriteToCsv([]string{"Repository name", "Signature name", "Matching file", "Matches"})
+ // Setup our delimited publisher
+ if *s.Options.CsvPath != "" {
+ // Convert delimiter to a rune slice. We only need the first rune
+ delimiter := []rune(*s.Options.Delimiter)
+ publisher, err := NewDelimitedSource(*s.Options.CsvPath, delimiter[0], header)
+ if err != nil {
+ s.Log.Error("Cannot create CSV publisher: %s", err)
+ }
+ if err == nil {
+ s.Publishers = append(s.Publishers, &publisher)
+ }
  }
-}
 
-func (s *Session) WriteToCsv(line []string) {
- if *s.Options.CsvPath == "" {
- return
+ // Setup our Live publisher
+ if *session.Options.Live != "" {
+ publisher, err := NewWebSource(*session.Options.Live, "POST", "application/json")
+ if err != nil {
+ s.Log.Error("Cannot create Live publisher: %s", err)
+ }
+ if err == nil {
+ s.Publishers = append(s.Publishers, &publisher)
+ }
  }
-
- s.CsvWriter.Write(line)
- s.CsvWriter.Flush()
 }
 
 func GetSession() *Session {

main.go

@@ -3,9 +3,7 @@ package main
 import (
  "bufio"
  "bytes"
- "encoding/json"
  "io/ioutil"
- "net/http"
  "os"
  "path/filepath"
  "regexp"
@@ -15,15 +13,6 @@ import (
  "github.com/fatih/color"
 )
 
-type MatchEvent struct {
- Url string
- Matches []string
- Signature string
- File string
- Stars int
- Source core.GitResourceType
-}
-
 var session = core.GetSession()
 
 func ProcessRepositories() {
@@ -125,8 +114,8 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT
  if matches != nil {
  count := len(matches)
  m := strings.Join(matches, ", ")
+ publish(core.MatchEvent{Url: url, Signature: "Search Query", File: relativeFileName, Matches: matches})
  session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString("Search Query"), relativeFileName, color.YellowString(m))
- session.WriteToCsv([]string{url, "Search Query", relativeFileName, m})
  }
  } else {
  for _, signature := range session.Signatures {
@@ -137,15 +126,13 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT
  if matches = signature.GetContentsMatches(file.Contents); matches != nil {
  count := len(matches)
  m := strings.Join(matches, ", ")
- publish(&MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars})
+ publish(core.MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars})
  session.Log.Important("[%s] %d %s for %s in file %s: %s", url, count, core.Pluralize(count, "match", "matches"), color.GreenString(signature.Name()), relativeFileName, color.YellowString(m))
- session.WriteToCsv([]string{url, signature.Name(), relativeFileName, m})
  }
  } else {
  if *session.Options.PathChecks {
- publish(&MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars})
+ publish(core.MatchEvent{Source: source, Url: url, Matches: matches, Signature: signature.Name(), File: relativeFileName, Stars: stars})
  session.Log.Important("[%s] Matching file %s for %s", url, color.YellowString(relativeFileName), color.GreenString(signature.Name()))
- session.WriteToCsv([]string{url, signature.Name(), relativeFileName, ""})
  }
 
  if *session.Options.EntropyThreshold > 0 && file.CanCheckEntropy() {
@@ -167,9 +154,8 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT
  }
 
  if !blacklistedMatch {
- publish(&MatchEvent{Source: source, Url: url, Matches: []string{line}, Signature: "High entropy string", File: relativeFileName, Stars: stars})
+ publish(core.MatchEvent{Source: source, Url: url, Matches: []string{line}, Signature: "High entropy string", File: relativeFileName, Stars: stars})
  session.Log.Important("[%s] Potential secret in %s = %s", url, color.YellowString(relativeFileName), color.GreenString(line))
- session.WriteToCsv([]string{url, "High entropy string", relativeFileName, line})
  }
  }
  }
@@ -187,11 +173,12 @@ func checkSignatures(dir string, url string, stars int, source core.GitResourceT
  return
 }
 
-func publish(event *MatchEvent) {
- // todo: implement a modular plugin system to handle the various outputs (console, live, csv, webhooks, etc)
- if len(*session.Options.Live) > 0 {
- data, _ := json.Marshal(event)
- http.Post(*session.Options.Live, "application/json", bytes.NewBuffer(data))
+func publish(event core.MatchEvent) {
+ for _, publisher := range session.Publishers {
+ err := publisher.Publish(event)
+ if err != nil {
+ session.Log.Error("Cannot publish: %s", err)
+ }
  }
 }