Skip to content

Commit 43a34cc

Browse files
aledbfaledbf
authored
Improve cache handling and provence generation (#202)
* Improve cache handling and provence generation * Update dependencies * Update to dependencies * Improve CI jobs * Fix remote cache configuration * Cleanup
1 parent 25ab1fd commit 43a34cc

28 files changed

+1812
-993
lines changed

Diff for: ‎.devcontainer/devcontainer.json

+10-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
{
22
"name": "leeway",
3-
"image": "mcr.microsoft.com/devcontainers/go:1.22",
4-
// Features to add to the dev container. More info: https://containers.dev/features.
5-
"features": {}
3+
"build": {
4+
"context": "..",
5+
"dockerfile": "../.gitpod.Dockerfile"
6+
},
7+
"runArgs": [
8+
"--privileged",
9+
"--security-opt=seccomp=unconfined",
10+
"--network=host"
11+
],
12+
"containerUser": "root"
613
}

Diff for: ‎.github/workflows/ci-build.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ on:
99
- 'main'
1010

1111
env:
12-
GO_VERSION: '1.23'
12+
GO_VERSION: '1.24'
1313

1414
jobs:
1515
check-go:

Diff for: ‎.github/workflows/release.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
- name: Setup Golang
2020
uses: actions/setup-go@v5
2121
with:
22-
go-version: "1.23"
22+
go-version: "1.24"
2323
- name: Docker Login
2424
uses: docker/login-action@v1
2525
with:
@@ -31,7 +31,7 @@ jobs:
3131
if: startsWith(github.ref, 'refs/tags/')
3232
with:
3333
distribution: goreleaser
34-
version: v2.3.2
34+
version: "~> v2"
3535
args: release --clean
3636
env:
3737
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Diff for: ‎.gitpod.Dockerfile

+51-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,52 @@
1-
FROM gitpod/workspace-full
1+
FROM ubuntu:jammy
22

3-
USER root
3+
ADD https://raw.githubusercontent.com/gitpod-io/workspace-images/main/base/install-packages /usr/bin/install-packages
4+
RUN chmod +x /usr/bin/install-packages
5+
6+
RUN install-packages \
7+
zip unzip \
8+
jq \
9+
curl \
10+
ca-certificates \
11+
file \
12+
git \
13+
node.js
14+
15+
ENV GO_VERSION=1.24.0
16+
RUN echo "TARGETPLATFORM=${TARGETPLATFORM:-linux/amd64}" && \
17+
case "${TARGETPLATFORM:-linux/amd64}" in \
18+
"linux/arm64") \
19+
echo "GO_PLATFORM=linux-arm64" > /tmp/go_platform.env \
20+
;; \
21+
*) \
22+
echo "GO_PLATFORM=linux-amd64" > /tmp/go_platform.env \
23+
;; \
24+
esac
25+
26+
# Install Go and add it to PATH
27+
RUN . /tmp/go_platform.env && \
28+
curl -fsSL https://dl.google.com/go/go$GO_VERSION.$GO_PLATFORM.tar.gz | tar -C /usr/local -xzs
29+
30+
# Set Go environment variables
31+
ENV GOROOT=/usr/local/go
32+
ENV PATH=$GOROOT/bin:$PATH
33+
ENV GOPATH=/root/go
34+
ENV PATH=$GOPATH/bin:$PATH
35+
36+
# install VS Code Go tools for use with gopls as per https://github.com/golang/vscode-go/blob/master/docs/tools.md
37+
# also https://github.com/golang/vscode-go/blob/27bbf42a1523cadb19fad21e0f9d7c316b625684/src/goTools.ts#L139
38+
RUN go install -v github.com/uudashr/gopkgs/cmd/gopkgs@v2 \
39+
&& go install -v github.com/ramya-rao-a/go-outline@latest \
40+
&& go install -v github.com/cweill/gotests/gotests@latest \
41+
&& go install -v github.com/fatih/gomodifytags@latest \
42+
&& go install -v github.com/josharian/impl@latest \
43+
&& go install -v github.com/haya14busa/goplay/cmd/goplay@latest \
44+
&& go install -v github.com/go-delve/delve/cmd/dlv@latest \
45+
&& go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest \
46+
&& go install -v golang.org/x/tools/gopls@latest \
47+
&& go install -v honnef.co/go/tools/cmd/staticcheck@latest \
48+
&& rm -rf /root/.cache
49+
50+
ENV SHFMT_VERSION=3.10.0
51+
RUN curl -sSL -o /usr/local/bin/shfmt "https://github.com/mvdan/sh/releases/download/v${SHFMT_VERSION}/shfmt_v${SHFMT_VERSION}_linux_amd64" && \
52+
chmod 755 /usr/local/bin/shfmt

Diff for: ‎cmd/build.go

+54-17
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import (
1212
"time"
1313

1414
"github.com/gitpod-io/leeway/pkg/leeway"
15+
"github.com/gitpod-io/leeway/pkg/leeway/cache"
16+
"github.com/gitpod-io/leeway/pkg/leeway/cache/local"
17+
"github.com/gitpod-io/leeway/pkg/leeway/cache/remote"
1518
"github.com/gookit/color"
1619
log "github.com/sirupsen/logrus"
1720
"github.com/spf13/cobra"
@@ -84,7 +87,7 @@ var buildCmd = &cobra.Command{
8487
},
8588
}
8689

87-
func serveBuildResult(ctx context.Context, addr string, localCache *leeway.FilesystemCache, pkg *leeway.Package) {
90+
func serveBuildResult(ctx context.Context, addr string, localCache cache.LocalCache, pkg *leeway.Package) {
8891
br, exists := localCache.Location(pkg)
8992
if !exists {
9093
log.Fatal("build result is not in local cache despite just being built. Something's wrong with the cache.")
@@ -121,7 +124,7 @@ func serveBuildResult(ctx context.Context, addr string, localCache *leeway.Files
121124
}
122125
}
123126

124-
func saveBuildResult(ctx context.Context, loc string, localCache *leeway.FilesystemCache, pkg *leeway.Package) {
127+
func saveBuildResult(ctx context.Context, loc string, localCache cache.LocalCache, pkg *leeway.Package) {
125128
br, exists := localCache.Location(pkg)
126129
if !exists {
127130
log.Fatal("build result is not in local cache despite just being built. Something's wrong with the cache.")
@@ -178,15 +181,15 @@ func addBuildFlags(cmd *cobra.Command) {
178181
cmd.Flags().Bool("report-github", os.Getenv("GITHUB_OUTPUT") != "", "Report package build success/failure to GitHub Actions using the GITHUB_OUTPUT environment variable")
179182
}
180183

181-
func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemCache) {
184+
func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) {
182185
cm, _ := cmd.Flags().GetString("cache")
183186
log.WithField("cacheMode", cm).Debug("configuring caches")
184187
cacheLevel := leeway.CacheLevel(cm)
185188

186189
remoteCache := getRemoteCache()
187190
switch cacheLevel {
188191
case leeway.CacheNone, leeway.CacheLocal:
189-
remoteCache = leeway.NoRemoteCache{}
192+
remoteCache = remote.NewNoRemoteCache()
190193
case leeway.CacheRemotePull:
191194
remoteCache = &pullOnlyRemoteCache{C: remoteCache}
192195
case leeway.CacheRemotePush:
@@ -212,7 +215,7 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemC
212215
}
213216
}
214217
log.WithField("location", localCacheLoc).Debug("set up local cache")
215-
localCache, err := leeway.NewFilesystemCache(localCacheLoc)
218+
localCache, err := local.NewFilesystemCache(localCacheLoc)
216219
if err != nil {
217220
log.Fatal(err)
218221
}
@@ -310,33 +313,67 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, *leeway.FilesystemC
310313
}
311314

312315
type pushOnlyRemoteCache struct {
313-
C leeway.RemoteCache
316+
C cache.RemoteCache
314317
}
315318

316-
func (c *pushOnlyRemoteCache) ExistingPackages(pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
317-
return c.C.ExistingPackages(pkgs)
319+
func (c *pushOnlyRemoteCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) {
320+
return c.C.ExistingPackages(ctx, pkgs)
318321
}
319322

320-
func (c *pushOnlyRemoteCache) Download(dst leeway.Cache, pkgs []*leeway.Package) error {
323+
func (c *pushOnlyRemoteCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) error {
321324
return nil
322325
}
323326

324-
func (c *pushOnlyRemoteCache) Upload(src leeway.Cache, pkgs []*leeway.Package) error {
325-
return c.C.Upload(src, pkgs)
327+
func (c *pushOnlyRemoteCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error {
328+
return c.C.Upload(ctx, src, pkgs)
326329
}
327330

328331
type pullOnlyRemoteCache struct {
329-
C leeway.RemoteCache
332+
C cache.RemoteCache
330333
}
331334

332-
func (c *pullOnlyRemoteCache) ExistingPackages(pkgs []*leeway.Package) (map[*leeway.Package]struct{}, error) {
333-
return c.C.ExistingPackages(pkgs)
335+
func (c *pullOnlyRemoteCache) ExistingPackages(ctx context.Context, pkgs []cache.Package) (map[cache.Package]struct{}, error) {
336+
return c.C.ExistingPackages(ctx, pkgs)
334337
}
335338

336-
func (c *pullOnlyRemoteCache) Download(dst leeway.Cache, pkgs []*leeway.Package) error {
337-
return c.C.Download(dst, pkgs)
339+
func (c *pullOnlyRemoteCache) Download(ctx context.Context, dst cache.LocalCache, pkgs []cache.Package) error {
340+
return c.C.Download(ctx, dst, pkgs)
338341
}
339342

340-
func (c *pullOnlyRemoteCache) Upload(src leeway.Cache, pkgs []*leeway.Package) error {
343+
func (c *pullOnlyRemoteCache) Upload(ctx context.Context, src cache.LocalCache, pkgs []cache.Package) error {
341344
return nil
342345
}
346+
347+
func getRemoteCache() cache.RemoteCache {
348+
remoteCacheBucket := os.Getenv(EnvvarRemoteCacheBucket)
349+
remoteStorage := os.Getenv(EnvvarRemoteCacheStorage)
350+
if remoteCacheBucket != "" {
351+
switch remoteStorage {
352+
case "GCP":
353+
return remote.NewGSUtilCache(
354+
&cache.RemoteConfig{
355+
BucketName: remoteCacheBucket,
356+
},
357+
)
358+
case "AWS":
359+
rc, err := remote.NewS3Cache(
360+
&cache.RemoteConfig{
361+
BucketName: remoteCacheBucket,
362+
},
363+
)
364+
if err != nil {
365+
log.Fatalf("cannot access remote S3 cache: %v", err)
366+
}
367+
368+
return rc
369+
default:
370+
return remote.NewGSUtilCache(
371+
&cache.RemoteConfig{
372+
BucketName: remoteCacheBucket,
373+
},
374+
)
375+
}
376+
}
377+
378+
return remote.NewNoRemoteCache()
379+
}

Diff for: ‎cmd/provenance-assert.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ var provenanceAssertCmd = &cobra.Command{
7373
return nil
7474
}
7575

76-
failures = append(assertions.AssertEnvelope(env), failures...)
76+
failures = append(assertions.AssertBundle(env), failures...)
7777

7878
raw, err := base64.StdEncoding.DecodeString(env.Payload)
7979
if err != nil {

Diff for: ‎cmd/root.go

-27
Original file line numberDiff line numberDiff line change
@@ -173,33 +173,6 @@ func getBuildArgs() (leeway.Arguments, error) {
173173
return res, nil
174174
}
175175

176-
func getRemoteCache() leeway.RemoteCache {
177-
remoteCacheBucket := os.Getenv(EnvvarRemoteCacheBucket)
178-
remoteStorage := os.Getenv(EnvvarRemoteCacheStorage)
179-
if remoteCacheBucket != "" {
180-
switch remoteStorage {
181-
case "GCP":
182-
return leeway.GSUtilRemoteCache{
183-
BucketName: remoteCacheBucket,
184-
}
185-
case "AWS":
186-
rc, err := leeway.NewS3RemoteCache(remoteCacheBucket, nil)
187-
if err != nil {
188-
log.Fatalf("cannot access remote S3 cache: %v", err)
189-
}
190-
191-
return rc
192-
default:
193-
return leeway.GSUtilRemoteCache{
194-
BucketName: remoteCacheBucket,
195-
}
196-
}
197-
198-
}
199-
200-
return leeway.NoRemoteCache{}
201-
}
202-
203176
func addExperimentalCommand(parent, child *cobra.Command) {
204177
if os.Getenv("LEEWAY_EXPERIMENTAL") != "true" {
205178
return

Diff for: ‎go.mod

+36-37
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
module github.com/gitpod-io/leeway
22

3-
go 1.23
3+
go 1.24
44

55
require (
6-
github.com/aws/aws-sdk-go-v2 v1.32.3
7-
github.com/aws/aws-sdk-go-v2/config v1.28.1
8-
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.35
9-
github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
10-
github.com/aws/aws-sdk-go-v2/service/sts v1.32.3
6+
github.com/aws/aws-sdk-go-v2 v1.36.1
7+
github.com/aws/aws-sdk-go-v2/config v1.29.6
8+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.59
9+
github.com/aws/aws-sdk-go-v2/service/s3 v1.75.4
1110
github.com/creack/pty v1.1.23
1211
github.com/disiqueira/gotree v1.0.0
1312
github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd
@@ -16,7 +15,7 @@ require (
1615
github.com/google/uuid v1.6.0
1716
github.com/gookit/color v1.5.4
1817
github.com/imdario/mergo v0.3.13
19-
github.com/in-toto/in-toto-golang v0.3.3
18+
github.com/in-toto/in-toto-golang v0.9.0
2019
github.com/karrick/godirwalk v1.17.0
2120
github.com/minio/highwayhash v1.0.2
2221
github.com/opencontainers/runc v1.1.10
@@ -26,50 +25,50 @@ require (
2625
github.com/sirupsen/logrus v1.9.3
2726
github.com/spf13/cobra v1.8.1
2827
github.com/stretchr/testify v1.9.0
29-
golang.org/x/mod v0.21.0
30-
golang.org/x/sync v0.8.0
28+
golang.org/x/mod v0.23.0
29+
golang.org/x/sync v0.11.0
3130
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da
3231
gopkg.in/yaml.v3 v3.0.1
33-
sigs.k8s.io/bom v0.1.0
32+
sigs.k8s.io/bom v0.6.0
3433
)
3534

3635
require (
37-
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
38-
github.com/aws/aws-sdk-go-v2/credentials v1.17.42 // indirect
39-
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
40-
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
41-
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
42-
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
43-
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.22 // indirect
44-
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
45-
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
46-
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
47-
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
48-
github.com/aws/aws-sdk-go-v2/service/sso v1.24.3 // indirect
49-
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.3 // indirect
50-
github.com/aws/smithy-go v1.22.0 // indirect
36+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
37+
github.com/aws/aws-sdk-go-v2/credentials v1.17.59 // indirect
38+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect
39+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
40+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect
41+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect
42+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect
43+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
44+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 // indirect
45+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect
46+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect
47+
github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect
48+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect
49+
github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect
50+
github.com/aws/smithy-go v1.22.2 // indirect
5151
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
52-
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
53-
github.com/cyphar/filepath-securejoin v0.3.4 // indirect
52+
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
53+
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
5454
github.com/davecgh/go-spew v1.1.1 // indirect
5555
github.com/dlclark/regexp2 v1.11.4 // indirect
5656
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
57-
github.com/godbus/dbus/v5 v5.1.0 // indirect
57+
github.com/godbus/dbus/v5 v5.0.6 // indirect
5858
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
5959
github.com/inconshreveable/mousetrap v1.1.0 // indirect
60-
github.com/kr/pretty v0.3.1 // indirect
61-
github.com/moby/sys/mountinfo v0.7.1 // indirect
62-
github.com/pkg/errors v0.9.1 // indirect
60+
github.com/moby/sys/mountinfo v0.5.0 // indirect
6361
github.com/pmezard/go-difflib v1.0.0 // indirect
64-
github.com/rogpeppe/go-internal v1.11.0 // indirect
65-
github.com/seccomp/libseccomp-golang v0.10.0 // indirect
62+
github.com/rogpeppe/go-internal v1.13.1 // indirect
63+
github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
64+
github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect
6665
github.com/segmentio/backo-go v1.0.0 // indirect
67-
github.com/shibumi/go-pathspec v1.2.0 // indirect
66+
github.com/shibumi/go-pathspec v1.3.0 // indirect
6867
github.com/spf13/pflag v1.0.5 // indirect
6968
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
70-
golang.org/x/crypto v0.31.0 // indirect
71-
golang.org/x/net v0.33.0 // indirect
72-
golang.org/x/sys v0.28.0 // indirect
73-
golang.org/x/text v0.21.0 // indirect
69+
golang.org/x/crypto v0.18.0 // indirect
70+
golang.org/x/net v0.20.0 // indirect
71+
golang.org/x/sys v0.21.0 // indirect
72+
golang.org/x/text v0.14.0 // indirect
7473
sigs.k8s.io/release-utils v0.7.7 // indirect
7574
)

0 commit comments

Comments
 (0)