Remove globals from core to improve usage as a library

core/config.go

@@ -38,7 +38,7 @@ func ParseConfig(options *Options) (*Config, error) {
  )
 
  if len(*options.ConfigPath) > 0 {
- data, err = ioutil.ReadFile(path.Join(*options.ConfigPath, "config.yaml"))
+ data, err = ioutil.ReadFile(path.Join(*options.ConfigPath, *options.ConfigName))
  if err != nil {
  return config, err
  }
@@ -47,10 +47,10 @@ func ParseConfig(options *Options) (*Config, error) {
  // Helps e.g. with Drone where workdir is different than shhgit dir
  ex, err := os.Executable()
  dir := filepath.Dir(ex)
- data, err = ioutil.ReadFile(path.Join(dir, "config.yaml"))
+ data, err = ioutil.ReadFile(path.Join(dir, *options.ConfigName))
  if err != nil {
  dir, _ = os.Getwd()
- data, err = ioutil.ReadFile(path.Join(dir, "config.yaml"))
+ data, err = ioutil.ReadFile(path.Join(dir, *options.ConfigName))
  if err != nil {
  return config, err
  }

core/log.go

@@ -30,6 +30,7 @@ var LogColors = map[int]*color.Color{
 
 type Logger struct {
  sync.Mutex
+ s *Session
 
  debug bool
  silent bool
@@ -61,10 +62,10 @@ func (l *Logger) Log(level int, format string, args ...interface{}) {
  fmt.Printf("\r"+format+"\n", args...)
  }
 
- if level > WARN && session.Config.Webhook != "" {
+ if level > WARN && l.s.Config.Webhook != "" {
  text := colorStrip(fmt.Sprintf(format, args...))
- payload := fmt.Sprintf(session.Config.WebhookPayload, text)
- http.Post(session.Config.Webhook, "application/json", strings.NewReader(payload))
+ payload := fmt.Sprintf(l.s.Config.WebhookPayload, text)
+ http.Post(l.s.Config.Webhook, "application/json", strings.NewReader(payload))
  }
 
  if level == FATAL {

core/match.go

@@ -28,16 +28,16 @@ func NewMatchFile(path string) MatchFile {
  }
 }
 
-func IsSkippableFile(path string) bool {
+func IsSkippableFile(s *Session, path string) bool {
  extension := strings.ToLower(filepath.Ext(path))
 
- for _, skippableExt := range session.Config.BlacklistedExtensions {
+ for _, skippableExt := range s.Config.BlacklistedExtensions {
  if extension == skippableExt {
  return true
  }
  }
 
- for _, skippablePathIndicator := range session.Config.BlacklistedPaths {
+ for _, skippablePathIndicator := range s.Config.BlacklistedPaths {
  skippablePathIndicator = strings.Replace(skippablePathIndicator, "{sep}", string(os.PathSeparator), -1)
  if strings.Contains(path, skippablePathIndicator) {
  return true
@@ -47,12 +47,12 @@ func IsSkippableFile(path string) bool {
  return false
 }
 
-func (match MatchFile) CanCheckEntropy() bool {
+func (match MatchFile) CanCheckEntropy(s *Session) bool {
  if match.Filename == "id_rsa" {
  return false
  }
 
- for _, skippableExt := range session.Config.BlacklistedEntropyExtensions {
+ for _, skippableExt := range s.Config.BlacklistedEntropyExtensions {
  if match.Extension == skippableExt {
  return false
  }
@@ -61,12 +61,12 @@ func (match MatchFile) CanCheckEntropy() bool {
  return true
 }
 
-func GetMatchingFiles(dir string) []MatchFile {
+func GetMatchingFiles(s *Session, dir string) []MatchFile {
  fileList := make([]MatchFile, 0)
- maxFileSize := *session.Options.MaximumFileSize * 1024
+ maxFileSize := *s.Options.MaximumFileSize * 1024
 
  filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
- if err != nil || f.IsDir() || uint(f.Size()) > maxFileSize || IsSkippableFile(path) {
+ if err != nil || f.IsDir() || uint(f.Size()) > maxFileSize || IsSkippableFile(s, path) {
  return nil
  }
  fileList = append(fileList, NewMatchFile(path))

core/options.go

@@ -1,7 +1,6 @@
 package core
 
 import (
- "flag"
  "os"
  "path/filepath"
 )
@@ -18,34 +17,95 @@ type Options struct {
  PathChecks *bool
  ProcessGists *bool
  TempDirectory *string
- CsvPath *string
+ CSVPath *string
  SearchQuery *string
  Local *string
  Live *string
  ConfigPath *string
+ ConfigName *string
 }
 
-func ParseOptions() (*Options, error) {
- options := &Options{
- Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"),
- Silent: flag.Bool("silent", false, "Suppress all output except for errors"),
- Debug: flag.Bool("debug", false, "Print debugging information"),
- MaximumRepositorySize: flag.Uint("maximum-repository-size", 5120, "Maximum repository size to process in KB"),
- MaximumFileSize: flag.Uint("maximum-file-size", 256, "Maximum file size to process in KB"),
- CloneRepositoryTimeout: flag.Uint("clone-repository-timeout", 10, "Maximum time it should take to clone a repository in seconds. Increase this if you have a slower connection"),
- EntropyThreshold: flag.Float64("entropy-threshold", 5.0, "Set to 0 to disable entropy checks"),
- MinimumStars: flag.Uint("minimum-stars", 0, "Only process repositories with this many stars. Default 0 will ignore star count"),
- PathChecks: flag.Bool("path-checks", true, "Set to false to disable checking of filepaths, i.e. just match regex patterns of file contents"),
- ProcessGists: flag.Bool("process-gists", true, "Will watch and process Gists. Set to false to disable."),
- TempDirectory: flag.String("temp-directory", filepath.Join(os.TempDir(), Name), "Directory to process and store repositories/matches"),
- CsvPath: flag.String("csv-path", "", "CSV file path to log found secrets to. Leave blank to disable"),
- SearchQuery: flag.String("search-query", "", "Specify a search string to ignore signatures and filter on files containing this string (regex compatible)"),
- Local: flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively. No need to have GitHub tokens with local run."),
- Live: flag.String("live", "", "Your shhgit live endpoint"),
- ConfigPath: flag.String("config-path", "", "Searches for config.yaml from given directory. If not set, tries to find if from shhgit binary's and current directory"),
+func (o *Options) Merge(new *Options) {
+ if new.Threads != nil {
+ o.Threads = new.Threads
  }
+ if new.Silent != nil {
+ o.Silent = new.Silent
+ }
+ if new.Debug != nil {
+ o.Debug = new.Debug
+ }
+ if new.MaximumRepositorySize != nil {
+ o.MaximumRepositorySize = new.MaximumRepositorySize
+ }
+ if new.MaximumFileSize != nil {
+ o.MaximumFileSize = new.MaximumFileSize
+ }
+ if new.CloneRepositoryTimeout != nil {
+ o.CloneRepositoryTimeout = new.CloneRepositoryTimeout
+ }
+ if new.EntropyThreshold != nil {
+ o.EntropyThreshold = new.EntropyThreshold
+ }
+ if new.MinimumStars != nil {
+ o.MinimumStars = new.MinimumStars
+ }
+ if new.PathChecks != nil {
+ o.PathChecks = new.PathChecks
+ }
+ if new.ProcessGists != nil {
+ o.ProcessGists = new.ProcessGists
+ }
+ if new.TempDirectory != nil {
+ o.TempDirectory = new.TempDirectory
+ }
+ if new.CSVPath != nil {
+ o.CSVPath = new.CSVPath
+ }
+ if new.SearchQuery != nil {
+ o.SearchQuery = new.SearchQuery
+ }
+ if new.Local != nil {
+ o.Local = new.Local
+ }
+ if new.Live != nil {
+ o.Live = new.Live
+ }
+ if new.ConfigPath != nil {
+ o.ConfigPath = new.ConfigPath
+ }
+ if new.ConfigName != nil {
+ o.ConfigName = new.ConfigName
+ }
+}
 
- flag.Parse()
+var (
+ // Defaults that don't represent Go struct defaults
+ DefaultMaximumRepositorySize = uint(5120)
+ DefaultMaximumFileSize = uint(256)
+ DefaultCloneRepositoryTimeout = uint(10)
+ DefaultEntropy = float64(5.0)
+ DefaultPathChecks = true
+ DefaultProcessGists = true
+ DefaultSilent = false
+ DefaultMinimumStars = uint(0)
+ DefaultDebug = false
+ DefaultTempDirectory = filepath.Join(os.TempDir(), Name)
+ DefaultThreads = 0
+ DefaultCSVPath = ""
 
- return options, nil
-}
+ DefaultOptions = Options{
+ MaximumRepositorySize: &DefaultMaximumRepositorySize,
+ MaximumFileSize: &DefaultMaximumFileSize,
+ CloneRepositoryTimeout: &DefaultCloneRepositoryTimeout,
+ EntropyThreshold: &DefaultEntropy,
+ PathChecks: &DefaultPathChecks,
+ ProcessGists: &DefaultProcessGists,
+ TempDirectory: &DefaultTempDirectory,
+ Silent: &DefaultSilent,
+ Debug: &DefaultDebug,
+ Threads: &DefaultThreads,
+ MinimumStars: &DefaultMinimumStars,
+ CSVPath: &DefaultCSVPath,
+ }
+)

core/session.go

@@ -31,12 +31,6 @@ type Session struct {
  CsvWriter *csv.Writer
 }
 
-var (
- session *Session
- sessionSync sync.Once
- err error
-)
-
 func (s *Session) Start() {
  rand.Seed(time.Now().Unix())
 
@@ -48,7 +42,7 @@ func (s *Session) Start() {
 }
 
 func (s *Session) InitLogger() {
- s.Log = &Logger{}
+ s.Log = &Logger{s: s}
  s.Log.SetDebug(*s.Options.Debug)
  s.Log.SetSilent(*s.Options.Silent)
 }
@@ -131,17 +125,19 @@ func (s *Session) InitThreads() {
 }
 
 func (s *Session) InitCsvWriter() {
- if *s.Options.CsvPath == "" {
+ if *s.Options.CSVPath == "" {
  return
  }
 
  writeHeader := false
- if !PathExists(*s.Options.CsvPath) {
+ if !PathExists(*s.Options.CSVPath) {
  writeHeader = true
  }
 
- file, err := os.OpenFile(*s.Options.CsvPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
- LogIfError("Could not create/open CSV file", err)
+ file, err := os.OpenFile(*s.Options.CSVPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ s.Log.Error("Could not create/open CSV file: %v", err)
+ }
 
  s.CsvWriter = csv.NewWriter(file)
 
@@ -151,35 +147,30 @@ func (s *Session) InitCsvWriter() {
 }
 
 func (s *Session) WriteToCsv(line []string) {
- if *s.Options.CsvPath == "" {
+ if *s.Options.CSVPath == "" {
  return
  }
 
  s.CsvWriter.Write(line)
  s.CsvWriter.Flush()
 }
 
-func GetSession() *Session {
- sessionSync.Do(func() {
- session = &Session{
- Context: context.Background(),
- Repositories: make(chan GitResource, 1000),
- Gists: make(chan string, 100),
- Comments: make(chan string, 1000),
- }
-
- if session.Options, err = ParseOptions(); err != nil {
- fmt.Println(err)
- os.Exit(1)
- }
-
- if session.Config, err = ParseConfig(session.Options); err != nil {
- fmt.Println(err)
- os.Exit(1)
- }
-
- session.Start()
- })
+func NewSession(ctx context.Context, o *Options) (*Session, error) {
+ s := &Session{
+ // TODO: Remove (contexts should not be embedded in structs)
+ Context: ctx,
+ Repositories: make(chan GitResource, 1000),
+ Gists: make(chan string, 100),
+ Comments: make(chan string, 1000),
+ Options: &DefaultOptions,
+ }
 
- return session
+ s.Options.Merge(o)
+ sc, err := ParseConfig(s.Options)
+ if err != nil {
+ return nil, fmt.Errorf("parse config: %w", err)
+ }
+ s.Config = sc
+ s.Start()
+ return s, nil
 }

core/signatures.go

@@ -18,8 +18,9 @@ const (
 
 type Signature interface {
  Name() string
- Match(file MatchFile) (bool, string)
- GetContentsMatches(contents []byte) []string
+ Match(MatchFile) (bool, string)
+ GetContentsMatches(*Session, []byte) []string
+ StringSubMatches(*Session, []byte) [][]string
 }
 
 type SimpleSignature struct {
@@ -57,7 +58,11 @@ func (s SimpleSignature) Match(file MatchFile) (bool, string) {
  return (s.match == *haystack), matchPart
 }
 
-func (s SimpleSignature) GetContentsMatches(contents []byte) []string {
+func (s SimpleSignature) GetContentsMatches(sess *Session, contents []byte) []string {
+ return nil
+}
+
+func (s SimpleSignature) StringSubMatches(sess *Session, contents []byte) [][]string {
  return nil
 }
 
@@ -90,14 +95,14 @@ func (s PatternSignature) Match(file MatchFile) (bool, string) {
  return s.match.MatchString(*haystack), matchPart
 }
 
-func (s PatternSignature) GetContentsMatches(contents []byte) []string {
+func (s PatternSignature) GetContentsMatches(sess *Session, contents []byte) []string {
  matches := make([]string, 0)
 
  for _, match := range s.match.FindAllSubmatch(contents, -1) {
  match := string(match[0])
  blacklistedMatch := false
 
- for _, blacklistedString := range session.Config.BlacklistedStrings {
+ for _, blacklistedString := range sess.Config.BlacklistedStrings {
  if strings.Contains(strings.ToLower(match), strings.ToLower(blacklistedString)) {
  blacklistedMatch = true
  }
@@ -111,6 +116,27 @@ func (s PatternSignature) GetContentsMatches(contents []byte) []string {
  return matches
 }
 
+func (s PatternSignature) StringSubMatches(sess *Session, contents []byte) [][]string {
+ matches := [][]string{}
+
+ for _, match := range s.match.FindAllStringSubmatch(string(contents), -1) {
+ fullMatch := string(match[0])
+ blacklistedMatch := false
+
+ for _, blacklistedString := range sess.Config.BlacklistedStrings {
+ if strings.Contains(strings.ToLower(fullMatch), strings.ToLower(blacklistedString)) {
+ blacklistedMatch = true
+ }
+ }
+
+ if !blacklistedMatch {
+ matches = append(matches, match)
+ }
+ }
+
+ return matches
+}
+
 func (s PatternSignature) Name() string {
  return s.name
 }