Merge remote-tracking branch 'origin'

v2/go.mod

@@ -11,9 +11,9 @@ require (
  github.com/projectdiscovery/dnsx v1.1.6
  github.com/projectdiscovery/fdmax v0.0.4
  github.com/projectdiscovery/gologger v1.1.12
- github.com/projectdiscovery/ratelimit v0.0.23
- github.com/projectdiscovery/retryablehttp-go v1.0.42
- github.com/projectdiscovery/utils v0.0.72
+ github.com/projectdiscovery/ratelimit v0.0.26
+ github.com/projectdiscovery/retryablehttp-go v1.0.44
+ github.com/projectdiscovery/utils v0.0.76
  github.com/rs/xid v1.5.0
  github.com/stretchr/testify v1.8.4
  github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
@@ -24,6 +24,7 @@ require (
 require (
  aead.dev/minisign v0.2.0 // indirect
  github.com/Masterminds/semver/v3 v3.2.1 // indirect
+ github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
  github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
  github.com/VividCortex/ewma v1.2.0 // indirect
  github.com/akrylysov/pogreb v0.10.1 // indirect
@@ -38,6 +39,7 @@ require (
  github.com/denisbrodbeck/machineid v1.0.1 // indirect
  github.com/dimchansky/utfbom v1.1.1 // indirect
  github.com/dlclark/regexp2 v1.8.1 // indirect
+ github.com/docker/go-units v0.5.0 // indirect
  github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
  github.com/fatih/color v1.15.0 // indirect
  github.com/gaukas/godicttls v0.0.4 // indirect
@@ -63,9 +65,9 @@ require (
  github.com/pierrec/lz4/v4 v4.1.2 // indirect
  github.com/projectdiscovery/blackrock v0.0.1 // indirect
  github.com/projectdiscovery/cdncheck v1.0.9 // indirect
- github.com/projectdiscovery/fastdialer v0.0.51 // indirect
- github.com/projectdiscovery/hmap v0.0.33 // indirect
- github.com/projectdiscovery/networkpolicy v0.0.6 // indirect
+ github.com/projectdiscovery/fastdialer v0.0.56 // indirect
+ github.com/projectdiscovery/hmap v0.0.36 // indirect
+ github.com/projectdiscovery/networkpolicy v0.0.7 // indirect
  github.com/quic-go/quic-go v0.37.7 // indirect
  github.com/refraction-networking/utls v1.5.4 // indirect
  github.com/rivo/uniseg v0.4.4 // indirect
@@ -110,7 +112,7 @@ require (
  github.com/pkg/errors v0.9.1
  github.com/pmezard/go-difflib v1.0.0 // indirect
  github.com/projectdiscovery/goflags v0.1.34
- github.com/projectdiscovery/retryabledns v1.0.50 // indirect
+ github.com/projectdiscovery/retryabledns v1.0.53 // indirect
  golang.org/x/net v0.17.0 // indirect
- golang.org/x/sys v0.15.0 // indirect
+ golang.org/x/sys v0.16.0 // indirect
 )

v2/go.sum

@@ -3,6 +3,8 @@ aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE=
+github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4=
 github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
 github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
@@ -46,6 +48,8 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/
 github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0=
 github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY=
 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
@@ -179,26 +183,26 @@ github.com/projectdiscovery/chaos-client v0.5.1 h1:NFxU7cxBY7AL1OKxQQRLRvJrkXlV+
 github.com/projectdiscovery/chaos-client v0.5.1/go.mod h1:uU7X/hVEKpOP8OZ8meDAAetMZ4Up0lz9oCcPGe86Yx4=
 github.com/projectdiscovery/dnsx v1.1.6 h1:QdKVKC0n/fpgaB4q3s6A2wn+qqg75CY0XxNkU9sVjws=
 github.com/projectdiscovery/dnsx v1.1.6/go.mod h1:9rkLQzJHxQ26qiD1PhfoJDrhqCVN8lKLsxiAON1uDxM=
-github.com/projectdiscovery/fastdialer v0.0.51 h1:LsRry/aSzUfgSCakJve05d6Ut83w1n1NcGS5tyUqsEY=
-github.com/projectdiscovery/fastdialer v0.0.51/go.mod h1:OqJbaFL/a6kX7107K6OjZ3usi2MStZ7dQop73DUOUJU=
+github.com/projectdiscovery/fastdialer v0.0.56 h1:SUw193mzYaDFtBpQOs4+EHa6r1NnBYj3xwSqK1j9Chk=
+github.com/projectdiscovery/fastdialer v0.0.56/go.mod h1:o0Oeb78Bn6mpr+dTLV4Jb7hnEFQjjA7lUeDe8Ga6zq4=
 github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc=
 github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I=
 github.com/projectdiscovery/goflags v0.1.34 h1:s64GffNF7lsdMz4V4ZS2vfo+x+FxaMB0/TpqZC2q6FI=
 github.com/projectdiscovery/goflags v0.1.34/go.mod h1:Br+lowrSpQuQOwDon5w1ZLq9BPJvxZhBUpUTc98UXZs=
 github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A=
 github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw=
-github.com/projectdiscovery/hmap v0.0.33 h1:kDkw4xVE8uyko6Cv3Cd9MZsHByn9BtXK3y7PeLKVBs4=
-github.com/projectdiscovery/hmap v0.0.33/go.mod h1:IlKSbnFKtn68STLiNwc5Kbu4GaR6aIsGaHbpFOYNFGY=
-github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o=
-github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk=
-github.com/projectdiscovery/ratelimit v0.0.23 h1:Fz2A57UW6GK0L0huOGVXd97EhASrJV41SC1NrGImShU=
-github.com/projectdiscovery/ratelimit v0.0.23/go.mod h1:042iuvdggjUnsgAIzyxM3iLFveMaXnGTRwlCpfd03I0=
-github.com/projectdiscovery/retryabledns v1.0.50 h1:0nM3x29G5LAZ7urfl0jSs501RQ5q57SkPwkdY19ECn8=
-github.com/projectdiscovery/retryabledns v1.0.50/go.mod h1:CbQhVC9JjtqU/89gz25gs6UgpQKYwFN2RoWoW5a/M9Q=
-github.com/projectdiscovery/retryablehttp-go v1.0.42 h1:NW76U/r0pWNi6iudBqggG69sN8aguuXLLbGRkLvniyo=
-github.com/projectdiscovery/retryablehttp-go v1.0.42/go.mod h1:NWR4amTNHwM+ALk1QL1HiyzhFejRTMCHapM+oSoNSv8=
-github.com/projectdiscovery/utils v0.0.72 h1:sJ1lBcaWO6dJ65F+fVhSJbguhgWjixgy9mjj7jKBUW8=
-github.com/projectdiscovery/utils v0.0.72/go.mod h1:VPnijH51D8wB1VJiEujUp7UZ+TUTKN8PpoW82nivUVY=
+github.com/projectdiscovery/hmap v0.0.36 h1:hnaiw+NZKQzoP0TXGjf5QmAtMgmrR0LHtR4xINJXWr0=
+github.com/projectdiscovery/hmap v0.0.36/go.mod h1:q6UNS9PoJqZHq5s3BjFTKdYRTZvmFr1JvhMKjNGdGgo=
+github.com/projectdiscovery/networkpolicy v0.0.7 h1:AwHqBRXBqDQgnWzBMuoJtHBNEYBw+NFp/4qIK688x7o=
+github.com/projectdiscovery/networkpolicy v0.0.7/go.mod h1:CK0CnFoLF1Nou6mY7P4WODSAxhPN8g8g7XpapgEP8tI=
+github.com/projectdiscovery/ratelimit v0.0.26 h1:sxZCh72lMpQ1YNnJOWrJ+uZE9GFWdVE58LOArOc6c+4=
+github.com/projectdiscovery/ratelimit v0.0.26/go.mod h1:2NHqfqqb9xAnqW+Ztd8AzzNi+JP38Kcdhb8cnbfX9sI=
+github.com/projectdiscovery/retryabledns v1.0.53 h1:eX4f7Afi2INmEaDj8F5x5T/VfEj62Q5qS+z3kcRuMBk=
+github.com/projectdiscovery/retryabledns v1.0.53/go.mod h1:FbjRnVnTkyGujjnQGDVzMomIYA4apN+AR6VrlJ8toHk=
+github.com/projectdiscovery/retryablehttp-go v1.0.44 h1:hicCe2h6daHt4muPovmffZE3YKBqGioreO6EpIGZ87g=
+github.com/projectdiscovery/retryablehttp-go v1.0.44/go.mod h1:7ECXK2cH2/G4sstf8hacyrMdPPJ/3wCAO5tFPZ4iO4s=
+github.com/projectdiscovery/utils v0.0.76 h1:6azn0Zju0taw5Y9qAjpGPxyqwJf2AI4VJjtIzPBcRzQ=
+github.com/projectdiscovery/utils v0.0.76/go.mod h1:ERIYcW+h5jKIYyYkfdOpNPIUtH8Ogz4q5Wq3gx/71Zw=
 github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU=
 github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
 github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o=
@@ -346,8 +350,8 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

v2/pkg/runner/banners.go

@@ -17,7 +17,7 @@ const banner = `
 const ToolName = `subfinder`
 
 // Version is the current version of subfinder
-const version = `v2.6.4`
+const version = `v2.6.5`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

v2/pkg/runner/enumerate.go

@@ -49,6 +49,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
  uniqueMap := make(map[string]resolve.HostEntry)
  // Create a map to track sources for each host
  sourceMap := make(map[string]map[string]struct{})
+ skippedCounts := make(map[string]int)
  // Process the results in a separate goroutine
  go func() {
  for result := range passiveResults {
@@ -58,6 +59,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
  case subscraping.Subdomain:
  // Validate the subdomain found and remove wildcards from
  if !strings.HasSuffix(result.Value, "."+domain) {
+ skippedCounts[result.Source]++
  continue
  }
  subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "")
@@ -77,6 +79,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
  // Check if the subdomain is a duplicate. If not,
  // send the subdomain for resolution.
  if _, ok := uniqueMap[subdomain]; ok {
+ skippedCounts[result.Source]++
  continue
  }
 
@@ -164,7 +167,17 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
 
  if r.options.Statistics {
  gologger.Info().Msgf("Printing source statistics for %s", domain)
- printStatistics(r.passiveAgent.GetStatistics())
+ statistics := r.passiveAgent.GetStatistics()
+ // This is a hack to remove the skipped count from the statistics
+ // as we don't want to show it in the statistics.
+ // TODO: Design a better way to do this.
+ for source, count := range skippedCounts {
+ if stat, ok := statistics[source]; ok {
+ stat.Results -= count
+ statistics[source] = stat
+ }
+ }
+ printStatistics(statistics)
  }
 
  return nil

v2/pkg/subscraping/sources/securitytrails/securitytrails.go

@@ -12,6 +12,7 @@ import (
  jsoniter "github.com/json-iterator/go"
 
  "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
+ "github.com/projectdiscovery/utils/ptr"
 )
 
 type response struct {
@@ -66,7 +67,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
  resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers)
  }
 
- if err != nil && resp.StatusCode == 403 {
+ if err != nil && ptr.Safe(resp).StatusCode == 403 {
  resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers)
  }