Skip to content

Commit 9f4ea6c

Browse files
rscrsc
committed
cmd/go: add go mod download
go mod download provides a way to force downloading of a particular module version into the download cache and also to locate its cached files. Forcing downloads is useful for warming caches, such as in base docker images. Finding the cached files allows caching proxies to use go mod download as the way to obtain module files on cache miss. Fixes #26577. Fixes #26610. Change-Id: Ib8065bcce07c9f5105868ec1d87887ef4871f07e Reviewed-on: https://go-review.googlesource.com/128355 Run-TryBot: Russ Cox <[email protected]> TryBot-Result: Gobot Gobot <[email protected]> Reviewed-by: Bryan C. Mills <[email protected]>
1 parent 89e13c8 commit 9f4ea6c

File tree

7 files changed

+291
-32
lines changed

7 files changed

+291
-32
lines changed

‎src/cmd/go/internal/modcmd/download.go

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Copyright 2018 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package modcmd
6+
7+
import (
8+
"cmd/go/internal/base"
9+
"cmd/go/internal/modfetch"
10+
"cmd/go/internal/modload"
11+
"cmd/go/internal/module"
12+
"cmd/go/internal/par"
13+
"encoding/json"
14+
"os"
15+
)
16+
17+
var cmdDownload = &base.Command{
18+
UsageLine: "go mod download [-dir] [-json] [modules]",
19+
Short: "download modules to local cache",
20+
Long: `
21+
Download downloads the named modules, which can be module patterns selecting
22+
dependencies of the main module or module queries of the form path@version.
23+
With no arguments, download applies to all dependencies of the main module.
24+
25+
The go command will automatically download modules as needed during ordinary
26+
execution. The "go mod download" command is useful mainly for pre-filling
27+
the local cache or to compute the answers for a Go module proxy.
28+
29+
By default, download reports errors to standard error but is otherwise silent.
30+
The -json flag causes download to print a sequence of JSON objects
31+
to standard output, describing each downloaded module (or failure),
32+
corresponding to this Go struct:
33+
34+
type Module struct {
35+
Path string // module path
36+
Version string // module version
37+
Error string // error loading module
38+
Info string // absolute path to cached .info file
39+
GoMod string // absolute path to cached .mod file
40+
Zip string // absolute path to cached .zip file
41+
Dir string // absolute path to cached source root directory
42+
}
43+
44+
See 'go help module' for more about module queries.
45+
`,
46+
}
47+
48+
var downloadJSON = cmdDownload.Flag.Bool("json", false, "")
49+
50+
func init() {
51+
cmdDownload.Run = runDownload // break init cycle
52+
}
53+
54+
type moduleJSON struct {
55+
Path string `json:",omitempty"`
56+
Version string `json:",omitempty"`
57+
Error string `json:",omitempty"`
58+
Info string `json:",omitempty"`
59+
GoMod string `json:",omitempty"`
60+
Zip string `json:",omitempty"`
61+
Dir string `json:",omitempty"`
62+
}
63+
64+
func runDownload(cmd *base.Command, args []string) {
65+
if len(args) == 0 {
66+
args = []string{"all"}
67+
}
68+
69+
var mods []*moduleJSON
70+
var work par.Work
71+
listU := false
72+
listVersions := false
73+
for _, info := range modload.ListModules(args, listU, listVersions) {
74+
if info.Replace != nil {
75+
info = info.Replace
76+
}
77+
if info.Version == "" {
78+
continue
79+
}
80+
m := &moduleJSON{
81+
Path: info.Path,
82+
Version: info.Version,
83+
}
84+
mods = append(mods, m)
85+
work.Add(m)
86+
}
87+
88+
work.Do(10, func(item interface{}) {
89+
m := item.(*moduleJSON)
90+
var err error
91+
m.Info, err = modfetch.InfoFile(m.Path, m.Version)
92+
if err != nil {
93+
m.Error = err.Error()
94+
return
95+
}
96+
m.GoMod, err = modfetch.GoModFile(m.Path, m.Version)
97+
if err != nil {
98+
m.Error = err.Error()
99+
return
100+
}
101+
mod := module.Version{Path: m.Path, Version: m.Version}
102+
m.Zip, err = modfetch.DownloadZip(mod)
103+
if err != nil {
104+
m.Error = err.Error()
105+
return
106+
}
107+
m.Dir, err = modfetch.Download(mod)
108+
if err != nil {
109+
m.Error = err.Error()
110+
return
111+
}
112+
})
113+
114+
if *downloadJSON {
115+
for _, m := range mods {
116+
b, err := json.MarshalIndent(m, "", "\t")
117+
if err != nil {
118+
base.Fatalf("%v", err)
119+
}
120+
os.Stdout.Write(append(b, '\n'))
121+
}
122+
}
123+
}

‎src/cmd/go/internal/modcmd/mod.go

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ See 'go help modules' for an overview of module functionality.
1919
`,
2020

2121
Commands: []*base.Command{
22+
cmdDownload,
2223
cmdEdit,
2324
cmdFix,
2425
cmdGraph,

‎src/cmd/go/internal/modfetch/cache.go

+41
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,23 @@ func Stat(path, rev string) (*RevInfo, error) {
232232
return repo.Stat(rev)
233233
}
234234

235+
// InfoFile is like Stat but returns the name of the file containing
236+
// the cached information.
237+
func InfoFile(path, version string) (string, error) {
238+
if !semver.IsValid(version) {
239+
return "", fmt.Errorf("invalid version %q", version)
240+
}
241+
if _, err := Stat(path, version); err != nil {
242+
return "", err
243+
}
244+
// Stat should have populated the disk cache for us.
245+
file, _, err := readDiskStat(path, version)
246+
if err != nil {
247+
return "", err
248+
}
249+
return file, nil
250+
}
251+
235252
// GoMod is like Lookup(path).GoMod(rev) but avoids the
236253
// repository path resolution in Lookup if the result is
237254
// already cached on local disk.
@@ -256,6 +273,23 @@ func GoMod(path, rev string) ([]byte, error) {
256273
return repo.GoMod(rev)
257274
}
258275

276+
// GoModFile is like GoMod but returns the name of the file containing
277+
// the cached information.
278+
func GoModFile(path, version string) (string, error) {
279+
if !semver.IsValid(version) {
280+
return "", fmt.Errorf("invalid version %q", version)
281+
}
282+
if _, err := GoMod(path, version); err != nil {
283+
return "", err
284+
}
285+
// GoMod should have populated the disk cache for us.
286+
file, _, err := readDiskGoMod(path, version)
287+
if err != nil {
288+
return "", err
289+
}
290+
return file, nil
291+
}
292+
259293
var errNotCached = fmt.Errorf("not in cache")
260294

261295
// readDiskStat reads a cached stat result from disk,
@@ -274,6 +308,13 @@ func readDiskStat(path, rev string) (file string, info *RevInfo, err error) {
274308
if err := json.Unmarshal(data, info); err != nil {
275309
return file, nil, errNotCached
276310
}
311+
// The disk might have stale .info files that have Name and Short fields set.
312+
// We want to canonicalize to .info files with those fields omitted.
313+
// Remarshal and update the cache file if needed.
314+
data2, err := json.Marshal(info)
315+
if err == nil && !bytes.Equal(data2, data) {
316+
writeDiskCache(file, data)
317+
}
277318
return file, info, nil
278319
}
279320

‎src/cmd/go/internal/modfetch/fetch.go

+42-15
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"sync"
1818

1919
"cmd/go/internal/base"
20+
"cmd/go/internal/cfg"
2021
"cmd/go/internal/dirhash"
2122
"cmd/go/internal/module"
2223
"cmd/go/internal/par"
@@ -46,24 +47,10 @@ func Download(mod module.Version) (dir string, err error) {
4647
return cached{"", err}
4748
}
4849
if files, _ := ioutil.ReadDir(dir); len(files) == 0 {
49-
zipfile, err := CachePath(mod, "zip")
50+
zipfile, err := DownloadZip(mod)
5051
if err != nil {
5152
return cached{"", err}
5253
}
53-
if _, err := os.Stat(zipfile); err == nil {
54-
// Use it.
55-
// This should only happen if the mod/cache directory is preinitialized
56-
// or if pkg/mod/path was removed but not pkg/mod/cache/download.
57-
fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version)
58-
} else {
59-
if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
60-
return cached{"", err}
61-
}
62-
fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
63-
if err := downloadZip(mod, zipfile); err != nil {
64-
return cached{"", err}
65-
}
66-
}
6754
modpath := mod.Path + "@" + mod.Version
6855
if err := Unzip(dir, zipfile, modpath, 0); err != nil {
6956
fmt.Fprintf(os.Stderr, "-> %s\n", err)
@@ -76,6 +63,46 @@ func Download(mod module.Version) (dir string, err error) {
7663
return c.dir, c.err
7764
}
7865

66+
var downloadZipCache par.Cache
67+
68+
// DownloadZip downloads the specific module version to the
69+
// local zip cache and returns the name of the zip file.
70+
func DownloadZip(mod module.Version) (zipfile string, err error) {
71+
// The par.Cache here avoids duplicate work but also
72+
// avoids conflicts from simultaneous calls by multiple goroutines
73+
// for the same version.
74+
type cached struct {
75+
zipfile string
76+
err error
77+
}
78+
c := downloadZipCache.Do(mod, func() interface{} {
79+
zipfile, err := CachePath(mod, "zip")
80+
if err != nil {
81+
return cached{"", err}
82+
}
83+
if _, err := os.Stat(zipfile); err == nil {
84+
// Use it.
85+
// This should only happen if the mod/cache directory is preinitialized
86+
// or if pkg/mod/path was removed but not pkg/mod/cache/download.
87+
if cfg.CmdName != "mod download" {
88+
fmt.Fprintf(os.Stderr, "go: extracting %s %s\n", mod.Path, mod.Version)
89+
}
90+
} else {
91+
if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
92+
return cached{"", err}
93+
}
94+
if cfg.CmdName != "mod download" {
95+
fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
96+
}
97+
if err := downloadZip(mod, zipfile); err != nil {
98+
return cached{"", err}
99+
}
100+
}
101+
return cached{zipfile, nil}
102+
}).(cached)
103+
return c.zipfile, c.err
104+
}
105+
79106
func downloadZip(mod module.Version, target string) error {
80107
repo, err := Lookup(mod.Path)
81108
if err != nil {

‎src/cmd/go/internal/modfetch/repo.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,12 @@ type Repo interface {
5555
// A Rev describes a single revision in a module repository.
5656
type RevInfo struct {
5757
Version string // version string
58-
Name string // complete ID in underlying repository
59-
Short string // shortened ID, for use in pseudo-version
6058
Time time.Time // commit time
59+
60+
// These fields are used for Stat of arbitrary rev,
61+
// but they are not recorded when talking about module versions.
62+
Name string `json:"-"` // complete ID in underlying repository
63+
Short string `json:"-"` // shortened ID, for use in pseudo-version
6164
}
6265

6366
// Re: module paths, import paths, repository roots, and lookups

‎src/cmd/go/internal/modload/build.go

+17-15
Original file line numberDiff line numberDiff line change
@@ -144,23 +144,25 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic {
144144

145145
complete(info)
146146

147-
if r := Replacement(m); r.Path != "" {
148-
info.Replace = &modinfo.ModulePublic{
149-
Path: r.Path,
150-
Version: r.Version,
151-
GoVersion: info.GoVersion,
152-
}
153-
if r.Version == "" {
154-
if filepath.IsAbs(r.Path) {
155-
info.Replace.Dir = r.Path
156-
} else {
157-
info.Replace.Dir = filepath.Join(ModRoot, r.Path)
147+
if fromBuildList {
148+
if r := Replacement(m); r.Path != "" {
149+
info.Replace = &modinfo.ModulePublic{
150+
Path: r.Path,
151+
Version: r.Version,
152+
GoVersion: info.GoVersion,
153+
}
154+
if r.Version == "" {
155+
if filepath.IsAbs(r.Path) {
156+
info.Replace.Dir = r.Path
157+
} else {
158+
info.Replace.Dir = filepath.Join(ModRoot, r.Path)
159+
}
158160
}
161+
complete(info.Replace)
162+
info.Dir = info.Replace.Dir
163+
info.GoMod = filepath.Join(info.Dir, "go.mod")
164+
info.Error = nil // ignore error loading original module version (it has been replaced)
159165
}
160-
complete(info.Replace)
161-
info.Dir = info.Replace.Dir
162-
info.GoMod = filepath.Join(info.Dir, "go.mod")
163-
info.Error = nil // ignore error loading original module version (it has been replaced)
164166
}
165167

166168
return info

‎src/cmd/go/testdata/script/mod_download.txt

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
env GO111MODULE=on
2+
3+
# download with version should print nothing
4+
go mod download rsc.io/[email protected]
5+
! stdout .
6+
7+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.info
8+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.mod
9+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.0.zip
10+
11+
# download -json with version should print JSON
12+
go mod download -json 'rsc.io/quote@<=v1.5.0'
13+
stdout '^\t"Path": "rsc.io/quote"'
14+
stdout '^\t"Version": "v1.5.0"'
15+
stdout '^\t"Info": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.0.info"'
16+
stdout '^\t"GoMod": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.0.mod"'
17+
stdout '^\t"Zip": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.0.zip"'
18+
! stdout '"Error"'
19+
20+
# download queries above should not have added to go.mod.
21+
go list -m all
22+
! stdout rsc.io
23+
24+
# add to go.mod so we can test non-query downloads
25+
go mod edit -require rsc.io/[email protected]
26+
! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info
27+
! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod
28+
! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip
29+
30+
# module loading will page in the info and mod files
31+
go list -m all
32+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info
33+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod
34+
! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip
35+
36+
# download will fetch and unpack the zip file
37+
go mod download
38+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.info
39+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.mod
40+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.2.zip
41+
exists $GOPATH/pkg/mod/rsc.io/[email protected]
42+
43+
go mod download -json
44+
stdout '^\t"Path": "rsc.io/quote"'
45+
stdout '^\t"Version": "v1.5.2"'
46+
stdout '^\t"Info": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.info"'
47+
stdout '^\t"GoMod": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.mod"'
48+
stdout '^\t"Zip": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)cache(\\\\|/)download(\\\\|/)rsc.io(\\\\|/)quote(\\\\|/)@v(\\\\|/)v1.5.2.zip"'
49+
stdout '^\t"Dir": ".*(\\\\|/)pkg(\\\\|/)mod(\\\\|/)rsc.io(\\\\|/)[email protected]"'
50+
51+
# download will follow replacements
52+
go mod edit -require rsc.io/[email protected] -replace rsc.io/[email protected]=rsc.io/[email protected]
53+
go mod download
54+
! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.1.zip
55+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.3-pre1.zip
56+
57+
# download will not follow replacements for explicit module queries
58+
go mod download -json rsc.io/[email protected]
59+
exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.5.1.zip
60+
61+
-- go.mod --
62+
module m

0 commit comments

Comments
 (0)