cmd/vulnreport: various vulnreport changes for batch automation

* When creating or unexcluding a report, add a note to the report that
an error occurred in addition to logging a warning.
* When unexcluding reports, fix and lint the report before writing it.
* In Fix, attempt to add an advisory if the report needs one.

All of these changes are intended to make it easier to batch create &
unexclude reports, and notice when there are issues in the automated
process.

Change-Id: I33ec94a619cf0b89112a682386439a11381b6c57
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/562246
Reviewed-by: Damien Neil <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: tatianab

cmd/vulnreport/create.go

@@ -220,18 +220,23 @@ func reportFromAliases(ctx context.Context, id, modulePath string, aliases []str
 	if ac != nil {
 		suggestions, err := suggestions(ctx, ac, r, 1)
 		if err != nil {
+			r.AddNote(report.NoteTypeCreate, "failed to get AI-generated suggestions")
 			log.Warnf("failed to get AI-generated suggestions for %s: %v\n", r.ID, err)
-		} else if len(suggestions) == 0 {
-			log.Warnf("failed to get AI-generated suggestions for %s (none generated)\n", r.ID)
 		} else {
 			log.Infof("applying AI-generated suggestion for %s", r.ID)
 			applySuggestion(r, suggestions[0])
 		}
 	}
 
 	if *populateSymbols {
+		log.Infof("attempting to auto-populate symbols for %s (this may take a while...)", r.ID)
 		if err := symbols.Populate(r); err != nil {
+			r.AddNote(report.NoteTypeCreate, "failed to auto-populate symbols")
 			log.Warnf("could not auto-populate symbols: %s", err)
+		} else {
+			if err := checkReportSymbols(r); err != nil {
+				log.Warnf("auto-populated symbols have error(s): %s", err)
+			}
 		}
 	}
 

cmd/vulnreport/unexclude.go

@@ -89,6 +89,11 @@ func (u *unexclude) run(ctx context.Context, filename string) (err error) {
 	// Remove description because this is a "basic" report.
 	newR.Description = ""
 
+	r.Fix(u.pc)
+	if hasLints := newR.LintAsNotes(u.pc); hasLints {
+		log.Warnf("unexcluded report %s has lint errors that need to be fixed manually", filename)
+	}
+
 	if err := os.Remove(filename); err != nil {
 		log.Errf("could not remove excluded report: %v", err)
 	}

internal/genericosv/report.go

@@ -36,10 +36,7 @@ func (osv *Entry) ToReport(goID string, pc *proxy.Client) *report.Report {
 		case ghsa.IsGHSA(alias):
 			r.GHSAs = append(r.GHSAs, alias)
 		default:
-			r.Notes = append(r.Notes, &report.Note{
-				Body: fmt.Sprintf("found alias %s that is not a GHSA or CVE", alias),
-				Type: report.NoteTypeCreate,
-			})
+			r.AddNote(report.NoteTypeCreate, "found alias %s that is not a GHSA or CVE", alias)
 		}
 	}
 	addAlias(osv.ID)

internal/report/fix.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"golang.org/x/exp/slices"
+	"golang.org/x/vulndb/internal/osv"
 	"golang.org/x/vulndb/internal/proxy"
 	"golang.org/x/vulndb/internal/version"
 )
@@ -21,6 +22,7 @@ func (r *Report) Fix(pc *proxy.Client) {
 		m.FixVersions(pc)
 	}
 	r.FixText()
+	r.FixReferences()
 }
 
 func (r *Report) FixText() {
@@ -32,9 +34,25 @@ func (r *Report) FixText() {
 	if r.CVEMetadata != nil {
 		fixLines(&r.CVEMetadata.Description)
 	}
+}
+
+func (r *Report) FixReferences() {
 	for _, ref := range r.References {
 		ref.URL = fixURL(ref.URL)
 	}
+	if r.missingAdvisory(r.countAdvisories()) {
+		r.addAdvisory()
+	}
+}
+
+func (r *Report) addAdvisory() {
+	// For now, only add an advisory if there is a CVE.
+	if len(r.CVEs) > 0 {
+		r.References = append(r.References, &Reference{
+			Type: osv.ReferenceTypeAdvisory,
+			URL:  fmt.Sprintf("%s%s", NISTPrefix, r.CVEs[0]),
+		})
+	}
 }
 
 // FixVersions replaces each version with its canonical form (if possible),

internal/report/lint.go

@@ -219,17 +219,14 @@ func (ref *Reference) lint(l *linter, r *Report) {
 }
 
 func (r *Report) lintReferences(l *linter) {
-	advisoryCount := 0
 	for i, ref := range r.References {
 		rl := l.Group(name("references", i, ref.URL))
 		ref.lint(rl, r)
-		if ref.Type == osv.ReferenceTypeAdvisory {
-			advisoryCount++
-		}
 	}
 
 	// Find missing references.
 	rl := l.Group("references")
+	advisoryCount := r.countAdvisories()
 	if advisoryCount > 1 {
 		rl.Errorf("too many advisories (found %d, want <=1)", advisoryCount)
 	}
@@ -258,12 +255,27 @@ func (r *Report) lintReferences(l *linter) {
 				rl.Errorf("must contain an announcement link matching regex %q", announceRegex)
 			}
 		}
-		if advisoryCount == 0 && r.Description == "" && r.CVEMetadata == nil {
+		if r.missingAdvisory(advisoryCount) {
 			rl.Error("missing advisory (required because report has no description)")
 		}
 	}
 }
 
+func (r *Report) countAdvisories() int {
+	advisoryCount := 0
+	for _, ref := range r.References {
+		if ref.Type == osv.ReferenceTypeAdvisory {
+			advisoryCount++
+		}
+	}
+	return advisoryCount
+}
+
+func (r *Report) missingAdvisory(advisoryCount int) bool {
+	return !r.IsExcluded() && !r.IsFirstParty() &&
+		advisoryCount == 0 && r.Description == "" && r.CVEMetadata == nil
+}
+
 func (d *Description) lint(l *linter, r *Report) {
 	desc := d.String()
 
@@ -417,17 +429,21 @@ func (r *Report) LintAsNotes(pc *proxy.Client) bool {
 	if lints := r.Lint(pc); len(lints) > 0 {
 		slices.Sort(lints)
 		for _, lint := range lints {
-			r.Notes = append(r.Notes, &Note{
-				Body: lint,
-				Type: NoteTypeLint,
-			})
+			r.AddNote(NoteTypeLint, lint)
 		}
 		return true
 	}
 
 	return false
 }
 
+func (r *Report) AddNote(t NoteType, format string, v ...any) {
+	r.Notes = append(r.Notes, &Note{
+		Body: fmt.Sprintf(format, v...),
+		Type: t,
+	})
+}
+
 // LintOffline performs all lint checks that don't require a network connection.
 func (r *Report) LintOffline() []string {
 	return r.lint(nil)