internal/frontend: limit number of paths allowed by frontend fetch

A upperbound of 10 candidateModulePaths is added for the number of fetch
attempts that a single frontend fetch request will make. Based on our
current dataset, there aren't any module paths that currently have more
than 7 path elements.

Change-Id: Ia098e538abb5a22ba23c9d6c147ffd1da4f1fbb7
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/248377
Run-TryBot: Julie Qiu <[email protected]>
TryBot-Result: kokoro <[email protected]>
Reviewed-by: Jamal Carvalho <[email protected]>
Reviewed-by: Robert Findley <[email protected]>

Author: julieqiu

internal/frontend/fetch.go

@@ -443,6 +443,11 @@ var vcsHostsWithThreeElementRepoName = map[string]bool{
 	"golang.org":    true,
 }
 
+// maxPathsToFetch is the number of modulePaths that are fetched from a single
+// fetch request. The longest module path we've seen in our database had 7 path
+// elements. maxPathsToFetch is set to 10 as a buffer.
+var maxPathsToFetch = 10
+
 // candidateModulePaths returns the potential module paths that could contain
 // the fullPath. The paths are returned in reverse length order.
 func candidateModulePaths(fullPath string) (_ []string, err error) {
@@ -462,6 +467,9 @@ func candidateModulePaths(fullPath string) (_ []string, err error) {
 		parts = parts[2:]
 	}
 	for _, part := range parts {
+		if len(modulePaths) == maxPathsToFetch {
+			return modulePaths, nil
+		}
 		path += part
 		if err := module.CheckImportPath(path); err != nil {
 			continue

internal/frontend/fetch_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"golang.org/x/pkgsite/internal"
 	"golang.org/x/pkgsite/internal/derrors"
 	"golang.org/x/pkgsite/internal/experiment"
@@ -177,3 +178,36 @@ func TestFetchPathAlreadyExists(t *testing.T) {
 		})
 	}
 }
+
+func TestCandidateModulePaths(t *testing.T) {
+	maxPathsToFetch = 7
+	for _, test := range []struct {
+		name, fullPath string
+		want           []string
+	}{
+		{"custom path", "my.module/foo", []string{
+			"my.module/foo",
+			"my.module",
+		}},
+		{"github.com module path", sample.ModulePath, []string{sample.ModulePath}},
+		{"more than 7 possible paths", sample.ModulePath + "/1/2/3/4/5/6/7/8/9", []string{
+			sample.ModulePath + "/1/2/3/4/5/6",
+			sample.ModulePath + "/1/2/3/4/5",
+			sample.ModulePath + "/1/2/3/4",
+			sample.ModulePath + "/1/2/3",
+			sample.ModulePath + "/1/2",
+			sample.ModulePath + "/1",
+			sample.ModulePath,
+		}},
+	} {
+		t.Run(test.fullPath, func(t *testing.T) {
+			got, err := candidateModulePaths(test.fullPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(test.want, got); diff != "" {
+				t.Fatalf("mismatch (-want, +got):\n%s", diff)
+			}
+		})
+	}
+}