[backport] feat(go/vulndb): support the new OSV format (#165)

* chore(mod): tidy (#163)

* refactor: use WalkDir

* feat(go/vulndb): support a new OSV format

go.mod

@@ -5,17 +5,18 @@ go 1.15
 require (
  github.com/BurntSushi/toml v0.3.1
  github.com/aquasecurity/bolt-fixtures v0.0.0-20200903104109-d34e7f983986
- github.com/aquasecurity/vuln-list-update v0.0.0-20191016075347-3d158c2bf9a2
  github.com/briandowns/spinner v1.12.0
  github.com/fatih/color v1.10.0
  github.com/google/go-github/v38 v38.1.0
  github.com/hashicorp/go-version v1.2.1
  github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
  github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936
- github.com/stretchr/testify v1.6.1
+ github.com/mattn/go-runewidth v0.0.4 // indirect
+ github.com/stretchr/testify v1.7.0
  github.com/urfave/cli v1.22.5
  go.etcd.io/bbolt v1.3.5
- golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5
+ golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
+ golang.org/x/vuln v0.0.0-20211215213114-5e054cb3e47e
  golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
  gopkg.in/cheggaaa/pb.v1 v1.0.28
  gopkg.in/yaml.v2 v2.4.0

pkg/utils/utils.go

@@ -3,6 +3,7 @@ package utils
 import (
  "bytes"
  "io"
+ "io/fs"
  "log"
  "os"
  "os/exec"
@@ -22,14 +23,18 @@ func CacheDir() string {
 }
 
 func FileWalk(root string, walkFn func(r io.Reader, path string) error) error {
- err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+ err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
  if err != nil {
  return err
- }
- if info.IsDir() {
+ } else if d.IsDir() {
  return nil
  }
 
+ info, err := d.Info()
+ if err != nil {
+ return xerrors.Errorf("file info error: %w", err)
+ }
+
  if info.Size() == 0 {
  log.Printf("invalid size: %s\n", path)
  return nil

pkg/vulnsrc/govulndb/govulndb.go

@@ -1,14 +1,15 @@
 package govulndb
 
 import (
- "bytes"
  "encoding/json"
  "fmt"
  "io"
  "log"
  "path/filepath"
+ "strings"
 
  bolt "go.etcd.io/bbolt"
+ "golang.org/x/vuln/osv"
  "golang.org/x/xerrors"
 
  "github.com/aquasecurity/trivy-db/pkg/db"
@@ -41,16 +42,16 @@ func (vs VulnSrc) Update(dir string) error {
  rootDir := filepath.Join(dir, "vuln-list", govulndbDir)
 
  var items []Entry
- buffer := &bytes.Buffer{}
  err := utils.FileWalk(rootDir, func(r io.Reader, path string) error {
- item := Entry{}
- if _, err := buffer.ReadFrom(r); err != nil {
- return xerrors.Errorf("failed to read file (%s): %w", path, err)
+ var item Entry
+ if err := json.NewDecoder(r).Decode(&item); err != nil {
+ return xerrors.Errorf("JSON decode error (%s): %w", path, err)
  }
- if err := json.Unmarshal(buffer.Bytes(), &item); err != nil {
- return xerrors.Errorf("JSON error (%s): %w", path, err)
+
+ // Standard libraries are not listed in go.sum, etc.
+ if item.Module == "stdlib" {
+ return nil
  }
- buffer.Reset()
  items = append(items, item)
  return nil
  })
@@ -89,41 +90,52 @@ func (vs VulnSrc) commit(tx *bolt.Tx, item Entry) error {
  vulnIDs = []string{item.ID}
  }
 
- var patchedVersions, vulnerableVersions []string
- for _, affect := range item.Affects.Ranges {
- // patched versions
- patchedVersions = append(patchedVersions, affect.Fixed)
+ // Take a single affected entry
+ affected := findAffected(item.Module, item.Affected)
+ if len(affected.Ranges) == 0 {
+ return xerrors.Errorf("invalid entry: %s %s", item.ID, item.Module)
+ }
 
- if affect.Fixed == "" {
- continue
+ var patchedVersions, vulnerableVersions, references []string
+ for _, affects := range affected.Ranges {
+ var vulnerable string
+ for _, event := range affects.Events {
+ switch {
+ case event.Introduced != "":
+ // e.g. {"introduced": "1.2.0}, {"introduced": "2.2.0}
+ if vulnerable != "" {
+ vulnerableVersions = append(vulnerableVersions, vulnerable)
+ }
+ vulnerable = fmt.Sprintf(">=%s", event.Introduced)
+ case event.Fixed != "":
+ // patched versions
+ patchedVersions = append(patchedVersions, event.Fixed)
+
+ // e.g. {"introduced": "1.2.0}, {"fixed": "1.2.5}
+ vulnerable = fmt.Sprintf("%s, <%s", vulnerable, event.Fixed)
+ }
  }
-
- // vulnerable versions
- vulnerable := fmt.Sprintf("< %s", affect.Fixed)
- if affect.Introduced != "" {
- vulnerable = fmt.Sprintf(">= %s, %s", affect.Introduced, vulnerable)
+ if vulnerable != "" {
+ vulnerableVersions = append(vulnerableVersions, vulnerable)
  }
+ }
+
+ if affected.DatabaseSpecific.URL != "" {
+ references = append(references, affected.DatabaseSpecific.URL)
+ }
 
- vulnerableVersions = append(vulnerableVersions, vulnerable)
+ // Update references
+ for _, ref := range item.References {
+ references = append(references, ref.URL)
  }
 
  a := types.Advisory{
  PatchedVersions: patchedVersions,
  VulnerableVersions: vulnerableVersions,
  }
 
+ // A module name must be filled.
  pkgName := item.Module
- if pkgName == "" {
- pkgName = item.Package.Name
- }
-
- var references []string
- for _, ref := range item.References {
- references = append(references, ref.URL)
- }
- if item.EcosystemSpecific.URL != "" {
- references = append(references, item.EcosystemSpecific.URL)
- }
 
  prefixedBucketName, _ := bucket.Name(vulnerability.Go, bucketName)
  for _, vulnID := range vulnIDs {
@@ -133,11 +145,8 @@ func (vs VulnSrc) commit(tx *bolt.Tx, item Entry) error {
  }
 
  vuln := types.VulnerabilityDetail{
- ID: vulnID,
- Description: item.Details,
- References: references,
- PublishedDate: &item.Published,
- LastModifiedDate: &item.Modified,
+ Description: item.Details,
+ References: references,
  }
  if err = vs.dbc.PutVulnerabilityDetail(tx, vulnID, prefixedBucketName, vuln); err != nil {
  return xerrors.Errorf("failed to put vulnerability detail (%s): %w", vulnID, err)
@@ -150,3 +159,33 @@ func (vs VulnSrc) commit(tx *bolt.Tx, item Entry) error {
 
  return nil
 }
+
+func findAffected(module string, affectedList []osv.Affected) osv.Affected {
+ // Multiple packages may be included in "affected".
+ // We have to select the appropriate package matching the module name
+ // because those packages may have different versioning.
+ //
+ // e.g. GO-2020-0017
+ // module => github.com/dgrijalva/jwt-go/v4
+ // packages => github.com/dgrijalva/jwt-go and github.com/dgrijalva/jwt-go/v4
+ //
+ // We should ignore "github.com/dgrijalva/jwt-go" in the above case.
+ for _, a := range affectedList {
+ if a.Package.Name == module {
+ return a
+ }
+ }
+
+ // If there is no package that exactly matches the module name,
+ // we'll choose a package that contains the module name.
+ // e.g. GO-2021-0101
+ // module => github.com/apache/thrift
+ // package => github.com/apache/thrift/lib/go/thrift
+ for _, a := range affectedList {
+ if strings.HasPrefix(a.Package.Name, module) {
+ return a
+ }
+ }
+
+ return osv.Affected{}
+}

pkg/vulnsrc/govulndb/govulndb_test.go

@@ -9,7 +9,6 @@ import (
  "github.com/aquasecurity/trivy-db/pkg/db"
  "github.com/aquasecurity/trivy-db/pkg/dbtest"
  "github.com/aquasecurity/trivy-db/pkg/types"
- "github.com/aquasecurity/trivy-db/pkg/utils"
  "github.com/aquasecurity/trivy-db/pkg/vulnsrc/govulndb"
 )
 
@@ -29,60 +28,47 @@ func TestVulnSrc_Update(t *testing.T) {
  dir: "testdata/happy",
  want: []wantKV{
  {
- key: []string{"advisory-detail", "CVE-2020-29242", "go::vulndb", "github.com/dhowden/tag"},
+ key: []string{"advisory-detail", "CVE-2019-0210", "go::vulndb", "github.com/apache/thrift"},
  value: types.Advisory{
- PatchedVersions: []string{"v0.0.0-20201120070457-d52dcb253c63"},
- VulnerableVersions: []string{"< v0.0.0-20201120070457-d52dcb253c63"},
+ PatchedVersions: []string{"0.13.0"},
+ VulnerableVersions: []string{">=0.0.0-20151001171628-53dd39833a08, <0.13.0"},
  },
  },
  {
- key: []string{"vulnerability-detail", "CVE-2020-29242", "go::vulndb"},
+ key: []string{"vulnerability-detail", "CVE-2019-0210", "go::vulndb"},
  value: types.VulnerabilityDetail{
- ID: "CVE-2020-29242",
- Description: "Due to improper bounds checking a number of methods can trigger a panic due to attempted\nout-of-bounds reads. If the package is used to parse user supplied input this may be\nused as a vector for a denial of service attack.\n",
+ Description: "Due to an improper bounds check, parsing maliciously crafted messages can cause panics. If\nthis package is used to parse untrusted input, this may be used as a vector for a denial of\nservice attack.\n",
  References: []string{
- "https://github.com/dhowden/tag/commit/d52dcb253c63a153632bfee5f269dd411dcd8e96",
- "https://github.com/dhowden/tag/commit/a92213460e4838490ce3066ef11dc823cdc1740e",
- "https://github.com/dhowden/tag/commit/4b595ed4fac79f467594aa92f8953f90f817116e",
- "https://github.com/dhowden/tag/commit/6b18201aa5c5535511802ddfb4e4117686b4866d",
- "https://go.googlesource.com/vulndb/+/refs/heads/main/reports/GO-2021-0097.toml",
+ "https://go.googlesource.com/vulndb/+/refs/heads/master/reports/GO-2021-0101.yaml",
+ "https://github.com/apache/thrift/commit/264a3f318ed3e9e51573f67f963c8509786bcec2",
+ "https://github.com/advisories/GHSA-jq7p-26h5-w78r",
  },
- PublishedDate: utils.MustTimeParse("2021-04-14T12:00:00Z"),
- LastModifiedDate: utils.MustTimeParse("2021-04-14T12:00:00Z"),
  },
  },
- },
- },
- {
- name: "missing module",
- dir: "testdata/no-module",
- want: []wantKV{
  {
- key: []string{"advisory-detail", "GO-2021-0090", "go::vulndb", "github.com/tendermint/tendermint/types"},
+ key: []string{"advisory-detail", "CVE-2020-26160", "go::vulndb", "github.com/dgrijalva/jwt-go/v4"},
  value: types.Advisory{
- PatchedVersions: []string{"v0.34.0-dev1.0.20200702134149-480b995a3172"},
- VulnerableVersions: []string{">= v0.33.0, < v0.34.0-dev1.0.20200702134149-480b995a3172"},
+ PatchedVersions: []string{"4.0.0-preview1"},
+ VulnerableVersions: []string{">=0, <4.0.0-preview1"},
  },
  },
  {
- key: []string{"vulnerability-detail", "GO-2021-0090", "go::vulndb"},
+ key: []string{"vulnerability-detail", "CVE-2020-26160", "go::vulndb"},
  value: types.VulnerabilityDetail{
- ID: "GO-2021-0090",
- Description: "Proposed commits may contain signatures for blocks not contained within the commit. Instead of skipping\nthese signatures, they cause failure during verification. A malicious proposer can use this to force\nconsensus failures.\n",
+ Description: "If a JWT contains an audience claim with an array of strings, rather\nthan a single string, and `MapClaims.VerifyAudience` is called with\n`req` set to `false`, then audience verification will be bypassed,\nallowing an invalid set of audiences to be provided.\n",
  References: []string{
- "https://github.com/tendermint/tendermint/pull/5426",
- "https://go.googlesource.com/vulndb/+/refs/heads/main/reports/GO-2021-0090.toml",
+ "https://go.googlesource.com/vulndb/+/refs/heads/master/reports/GO-2020-0017.yaml",
+ "https://github.com/dgrijalva/jwt-go/commit/ec0a89a131e3e8567adcb21254a5cd20a70ea4ab",
+ "https://github.com/dgrijalva/jwt-go/issues/422",
  },
- PublishedDate: utils.MustTimeParse("2021-04-14T12:00:00Z"),
- LastModifiedDate: utils.MustTimeParse("2021-04-14T12:00:00Z"),
  },
  },
  },
  },
  {
  name: "broken JSON",
  dir: "testdata/broken",
- wantErr: "JSON error",
+ wantErr: "JSON decode error",
  },
  {
  name: "sad path",

pkg/vulnsrc/govulndb/testdata/happy/vuln-list/go/GO-2020-0017.json

@@ -0,0 +1,73 @@
+{
+ "module": "github.com/dgrijalva/jwt-go/v4",
+ "id": "GO-2020-0017",
+ "published": "2021-04-14T12:00:00Z",
+ "modified": "2021-04-14T12:00:00Z",
+ "aliases": [
+ "CVE-2020-26160"
+ ],
+ "details": "If a JWT contains an audience claim with an array of strings, rather\nthan a single string, and `MapClaims.VerifyAudience` is called with\n`req` set to `false`, then audience verification will be bypassed,\nallowing an invalid set of audiences to be provided.\n",
+ "affected": [
+ {
+ "package": {
+ "name": "github.com/dgrijalva/jwt-go",
+ "ecosystem": "Go"
+ },
+ "ranges": [
+ {
+ "type": "SEMVER",
+ "events": [
+ {
+ "introduced": "0.0.0-20150717181359-44718f8a89b0"
+ }
+ ]
+ }
+ ],
+ "database_specific": {
+ "url": "https://go.googlesource.com/vulndb/+/refs/heads/master/reports/GO-2020-0017.yaml"
+ },
+ "ecosystem_specific": {
+ "symbols": [
+ "MapClaims.VerifyAudience"
+ ]
+ }
+ },
+ {
+ "package": {
+ "name": "github.com/dgrijalva/jwt-go/v4",
+ "ecosystem": "Go"
+ },
+ "ranges": [
+ {
+ "type": "SEMVER",
+ "events": [
+ {
+ "introduced": "0"
+ },
+ {
+ "fixed": "4.0.0-preview1"
+ }
+ ]
+ }
+ ],
+ "database_specific": {
+ "url": "https://go.googlesource.com/vulndb/+/refs/heads/master/reports/GO-2020-0017.yaml"
+ },
+ "ecosystem_specific": {
+ "symbols": [
+ "MapClaims.VerifyAudience"
+ ]
+ }
+ }
+ ],
+ "references": [
+ {
+ "type": "FIX",
+ "url": "https://github.com/dgrijalva/jwt-go/commit/ec0a89a131e3e8567adcb21254a5cd20a70ea4ab"
+ },
+ {
+ "type": "WEB",
+ "url": "https://github.com/dgrijalva/jwt-go/issues/422"
+ }
+ ]
+}