client: only fetch necessary packages when cache missing

When a httpSource is used without a Cache implementation, only fetch
packages which are represented in the index. Previously if there
wasn't a Cache implementation every pacakge was enumerated, ignoring
the contents of the retrieved index.

Change-Id: I8e8e0ce412b3ded188afd6bb109d96efb7e7f27c
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/333455
Trust: Roland Shoemaker <[email protected]>
Trust: Zvonimir Pavlinovic <[email protected]>
Run-TryBot: Roland Shoemaker <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Zvonimir Pavlinovic <[email protected]>

Author: sayjun0505

client/client.go

@@ -158,12 +158,12 @@ func (hs *httpSource) Get(packages []string) ([]*osv.Entry, error) {
 	}
 
 	var stillNeed []string
-	if hs.cache != nil {
-		for _, p := range packages {
-			lastModified, present := index[p]
-			if !present {
-				continue
-			}
+	for _, p := range packages {
+		lastModified, present := index[p]
+		if !present {
+			continue
+		}
+		if hs.cache != nil {
 			if cached, err := hs.cache.ReadEntries(hs.dbName, p); err != nil {
 				return nil, err
 			} else if len(cached) != 0 {
@@ -179,10 +179,8 @@ func (hs *httpSource) Get(packages []string) ([]*osv.Entry, error) {
 					continue
 				}
 			}
-			stillNeed = append(stillNeed, p)
 		}
-	} else {
-		stillNeed = packages
+		stillNeed = append(stillNeed, p)
 	}
 
 	for _, p := range stillNeed {

client/client_test.go

@@ -5,12 +5,15 @@
 package client
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
+	"reflect"
 	"runtime"
 	"testing"
 	"time"
@@ -156,3 +159,30 @@ func TestClient(t *testing.T) {
 		}
 	}
 }
+
+func TestCorrectFetchesNoCache(t *testing.T) {
+	fetches := map[string]int{}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fetches[r.URL.Path]++
+		if r.URL.Path == "/index.json" {
+			j, _ := json.Marshal(osv.DBIndex{
+				"a": time.Now(),
+				"b": time.Now(),
+			})
+			w.Write(j)
+		} else {
+			w.Write([]byte("[]"))
+		}
+	}))
+	defer ts.Close()
+
+	hs := &httpSource{url: ts.URL, c: new(http.Client)}
+	_, err := hs.Get([]string{"a", "b", "c"})
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	expectedFetches := map[string]int{"/index.json": 1, "/a.json": 1, "/b.json": 1}
+	if !reflect.DeepEqual(fetches, expectedFetches) {
+		t.Errorf("unexpected fetches, got %v, want %v", fetches, expectedFetches)
+	}
+}