Skip to content

Commit bff2282

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #981 from shiftstack/upstream-tests
✨Enhancements for running E2E tests locally
2 parents 7dc7c13 + 7746f28 commit bff2282

12 files changed

+136
-24
lines changed

‎Containerfile

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Copyright 2021 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Containerfile for podman
16+
#
17+
# Ideally we would just use Dockerfile to avoid maintaining 2 files. However,
18+
# the Dockerfile is currently using some features [1] that podman doesn't
19+
# support. We should aim to re-merge these files eventually. In the meantime
20+
# we need to keep them consistent.
21+
#
22+
# [1] * RUN --mount=type=...
23+
# * podman can't pull dockerfile:1.1-experimental: unsupported docker v2s2
24+
# media type: "application/vnd.oci.image.layer.v1.tar+gzip"
25+
26+
# Build the manager binary
27+
FROM golang:1.16.0 as builder
28+
WORKDIR /workspace
29+
30+
# Run this with docker build --build_arg goproxy=$(go env GOPROXY) to override the goproxy
31+
ARG goproxy=https://proxy.golang.org
32+
ENV GOPROXY=$goproxy
33+
34+
# Copy the Go Modules manifests
35+
COPY go.mod go.mod
36+
COPY go.sum go.sum
37+
38+
# Cache deps before building and copying source so that we don't need to re-download as much
39+
# and so that source changes don't invalidate our downloaded layer
40+
RUN go mod download
41+
42+
# Copy the sources
43+
COPY ./ ./
44+
45+
# Build
46+
ARG package=.
47+
ARG ARCH
48+
ARG ldflags
49+
50+
# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
51+
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
52+
go build -ldflags "${ldflags} -extldflags '-static'" \
53+
-o manager ${package}
54+
55+
# Production image
56+
FROM gcr.io/distroless/static:nonroot
57+
WORKDIR /
58+
COPY --from=builder /workspace/manager .
59+
# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
60+
USER 65532
61+
ENTRYPOINT ["/manager"]

‎Dockerfile

-5
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,6 @@ RUN --mount=type=cache,target=/go/pkg/mod \
3434
# Copy the sources
3535
COPY ./ ./
3636

37-
# Cache the go build into the the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls
38-
RUN --mount=type=cache,target=/root/.cache/go-build \
39-
--mount=type=cache,target=/go/pkg/mod \
40-
go build .
41-
4237
# Build
4338
ARG package=.
4439
ARG ARCH

‎Makefile

+27-7
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,13 @@ PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
5757
DOCKER_CLI_EXPERIMENTAL=enabled
5858
DOCKER_BUILDKIT=1
5959

60+
PODMAN ?= 0
61+
ifeq ($(PODMAN), 1)
62+
CONTAINERFILE ?= Containerfile
63+
else
64+
CONTAINERFILE ?= Dockerfile
65+
endif
66+
6067
# Release variables
6168

6269
STAGING_REGISTRY := gcr.io/k8s-staging-capi-openstack
@@ -76,6 +83,7 @@ ALL_ARCH ?= amd64 arm arm64 ppc64le s390x
7683
# main controller
7784
IMAGE_NAME ?= capi-openstack-controller
7885
CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME)
86+
CONTROLLER_IMG_TAG ?= $(CONTROLLER_IMG)-$(ARCH):$(TAG)
7987
CONTROLLER_ORIGINAL_IMG := gcr.io/k8s-staging-capi-openstack/capi-openstack-controller
8088
CONTROLLER_NAME := capo-controller-manager
8189
MANIFEST_FILE := infrastructure-components
@@ -99,6 +107,9 @@ LDFLAGS := $(shell source ./hack/version.sh; version::ldflags)
99107
## Testing
100108
## --------------------------------------
101109

110+
# The number of ginkgo tests to run concurrently
111+
E2E_GINKGO_PARALLEL=2
112+
102113
E2E_ARGS ?=
103114

104115
$(ARTIFACTS):
@@ -113,12 +124,21 @@ test: ## Run tests
113124
# E2E_GINKGO_ARGS="-stream -focus='default'" E2E_ARGS="-use-existing-cluster='true'" make test-e2e
114125
E2E_GINKGO_ARGS ?= -stream
115126
.PHONY: test-e2e ## Run e2e tests using clusterctl
116-
test-e2e: $(GINKGO) $(KIND) $(KUSTOMIZE) e2e-image ## Run e2e tests
117-
time $(GINKGO) -trace -progress -v -tags=e2e --nodes=2 $(E2E_GINKGO_ARGS) ./test/e2e/suites/e2e/... -- -config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" --data-folder="$(E2E_DATA_DIR)" $(E2E_ARGS)
127+
test-e2e: $(GINKGO) $(KIND) $(KUSTOMIZE) e2e-image test-e2e-image-prerequisites ## Run e2e tests
128+
time $(GINKGO) -trace -progress -v -tags=e2e --nodes=$(E2E_GINKGO_PARALLEL) $(E2E_GINKGO_ARGS) ./test/e2e/suites/e2e/... -- -config-path="$(E2E_CONF_PATH)" -artifacts-folder="$(ARTIFACTS)" --data-folder="$(E2E_DATA_DIR)" $(E2E_ARGS)
118129

119130
.PHONY: e2e-image
120-
e2e-image: docker-pull-prerequisites
121-
docker build -f Dockerfile --tag="gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e" .
131+
e2e-image: CONTROLLER_IMG_TAG = "gcr.io/k8s-staging-capi-openstack/capi-openstack-controller:e2e"
132+
e2e-image: docker-build
133+
134+
# Pull all the images references in test/e2e/data/e2e_conf.yaml
135+
test-e2e-image-prerequisites:
136+
docker pull gcr.io/k8s-staging-cluster-api/cluster-api-controller:v0.4.0
137+
docker pull gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller:v0.4.0
138+
docker pull gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller:v0.4.0
139+
docker pull quay.io/jetstack/cert-manager-cainjector:v1.1.0
140+
docker pull quay.io/jetstack/cert-manager-webhook:v1.1.0
141+
docker pull quay.io/jetstack/cert-manager-controller:v1.1.0
122142

123143
CONFORMANCE_E2E_ARGS ?= -kubetest.config-file=$(KUBETEST_CONF_PATH)
124144
CONFORMANCE_E2E_ARGS += $(E2E_ARGS)
@@ -207,15 +227,15 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
207227

208228
.PHONY: docker-build
209229
docker-build: docker-pull-prerequisites ## Build the docker image for controller-manager
210-
docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG)
230+
docker build -f $(CONTAINERFILE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" . -t $(CONTROLLER_IMG_TAG)
211231

212232
.PHONY: docker-push
213233
docker-push: ## Push the docker image
214-
docker push $(CONTROLLER_IMG)-$(ARCH):$(TAG)
234+
docker push $(CONTROLLER_IMG_TAG)
215235

216236
.PHONY: docker-pull-prerequisites
217237
docker-pull-prerequisites:
218-
docker pull docker.io/docker/dockerfile:1.1-experimental
238+
[ "$(PODMAN)" -eq 0 ] && docker pull docker.io/docker/dockerfile:1.1-experimental
219239
docker pull docker.io/library/golang:$(GOLANG_VERSION)
220240
docker pull gcr.io/distroless/static:latest
221241

‎docs/book/src/clusteropenstack/configuration.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ When running CAPO with `--v=6` the gophercloud client logs its requests to the O
133133

134134
## External network
135135

136-
External network is automatically found, but you can specify the external network explicitly by `spec.externalNetworkId` of `OpenStackCluster`.
136+
If there is only a single external network it will be detected automatically. If there is more than one external network you can specify which one the cluster should use by setting the environment variable `OPENSTACK_EXTERNAL_NETWORK_ID`.
137137

138138
The public network id can be obtained by using command,
139139

‎templates/cluster-template-external-cloud-provider.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ spec:
3131
nodeCidr: 10.6.0.0/24
3232
dnsNameservers:
3333
- ${OPENSTACK_DNS_NAMESERVERS}
34+
externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID}
3435
---
3536
kind: KubeadmControlPlane
3637
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4

‎templates/cluster-template-without-lb.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ spec:
3030
nodeCidr: 10.6.0.0/24
3131
dnsNameservers:
3232
- ${OPENSTACK_DNS_NAMESERVERS}
33+
externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID}
3334
---
3435
kind: KubeadmControlPlane
3536
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4

‎templates/cluster-template.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ spec:
3131
nodeCidr: 10.6.0.0/24
3232
dnsNameservers:
3333
- ${OPENSTACK_DNS_NAMESERVERS}
34+
externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID}
3435
---
3536
kind: KubeadmControlPlane
3637
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4

‎test/e2e/data/e2e_conf.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ variables:
108108
OPENSTACK_IMAGE_NAME: "ubuntu-2004-kube-v1.18.15"
109109
OPENSTACK_NODE_MACHINE_FLAVOR: "m1.small"
110110
OPENSTACK_SSH_KEY_NAME: "cluster-api-provider-openstack-sigs-k8s-io"
111+
OPENSTACK_EXTERNAL_NETWORK_ID: ""
111112
CONFORMANCE_WORKER_MACHINE_COUNT: "5"
112113
CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1"
113114

‎test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ spec:
4040
flavor: ${OPENSTACK_BASTION_MACHINE_FLAVOR}
4141
image: ${OPENSTACK_BASTION_IMAGE_NAME}
4242
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
43+
externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID}
4344
---
4445
kind: KubeadmControlPlane
4546
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4

‎test/e2e/data/infrastructure-openstack/cluster-template-without-lb.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ spec:
3838
flavor: ${OPENSTACK_BASTION_MACHINE_FLAVOR}
3939
image: ${OPENSTACK_BASTION_IMAGE_NAME}
4040
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
41+
externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID}
4142
---
4243
kind: KubeadmControlPlane
4344
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4

‎test/e2e/data/infrastructure-openstack/cluster-template.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ spec:
3939
flavor: ${OPENSTACK_BASTION_MACHINE_FLAVOR}
4040
image: ${OPENSTACK_BASTION_IMAGE_NAME}
4141
sshKeyName: ${OPENSTACK_SSH_KEY_NAME}
42+
externalNetworkId: ${OPENSTACK_EXTERNAL_NETWORK_ID}
4243
---
4344
kind: KubeadmControlPlane
4445
apiVersion: controlplane.cluster.x-k8s.io/v1alpha4

‎test/e2e/shared/openstack.go

+40-11
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,8 @@ type AuthOpts struct {
228228
TenantName string `ini:"tenant-name"`
229229
DomainID string `ini:"domain-id"`
230230
DomainName string `ini:"domain-name"`
231+
232+
// In-tree cloud provider will fail to start if these are present
231233
// TenantDomainID string `ini:"tenant-domain-id"`
232234
// TenantDomainName string `ini:"tenant-domain-name"`
233235
// UserDomainID string `ini:"user-domain-id"`
@@ -251,18 +253,45 @@ func getEncodedOpenStackCloudYAML(cloudYAML string) string {
251253
func getEncodedOpenStackCloudProviderConf(cloudYAML, cloudName string) string {
252254
clouds := getParsedOpenStackCloudYAML(cloudYAML)
253255
cloud := clouds.Clouds[cloudName]
256+
257+
authopts := AuthOpts{
258+
AuthURL: cloud.AuthInfo.AuthURL,
259+
UserID: cloud.AuthInfo.UserID,
260+
Username: cloud.AuthInfo.Username,
261+
Password: cloud.AuthInfo.Password,
262+
TenantID: cloud.AuthInfo.ProjectID,
263+
TenantName: cloud.AuthInfo.ProjectName,
264+
Region: cloud.RegionName,
265+
}
266+
267+
// In-tree OpenStack cloud provider does not support
268+
// {Tenant,User}Domain{ID,Name}, but external cloud provider does.
269+
// Here we manually set Domain{ID,Name} depending on the most specific config available
270+
switch {
271+
case cloud.AuthInfo.UserDomainID != "":
272+
authopts.DomainID = cloud.AuthInfo.UserDomainID
273+
case cloud.AuthInfo.UserDomainName != "":
274+
authopts.DomainName = cloud.AuthInfo.UserDomainName
275+
case cloud.AuthInfo.ProjectDomainID != "":
276+
authopts.DomainID = cloud.AuthInfo.UserDomainID
277+
case cloud.AuthInfo.ProjectDomainName != "":
278+
authopts.DomainName = cloud.AuthInfo.ProjectDomainName
279+
case cloud.AuthInfo.DomainID != "":
280+
authopts.DomainID = cloud.AuthInfo.DomainID
281+
case cloud.AuthInfo.DomainName != "":
282+
authopts.DomainName = cloud.AuthInfo.DomainName
283+
}
284+
285+
// Regardless of the path to a CA cert specified in the input
286+
// clouds.yaml, we will deploy the cert to /etc/certs/cacert in the
287+
// target cluster as specified in KubeadmControlPlane and KubeadmConfig
288+
// for the control plane and workers respectively in the E2E cluster templates.
289+
if cloud.CACertFile != "" {
290+
authopts.CAFile = "/etc/certs/cacert"
291+
}
292+
254293
cloudProviderConf := &Config{
255-
Global: AuthOpts{
256-
AuthURL: cloud.AuthInfo.AuthURL,
257-
UserID: cloud.AuthInfo.UserID,
258-
Username: cloud.AuthInfo.Username,
259-
Password: cloud.AuthInfo.Password,
260-
TenantID: cloud.AuthInfo.ProjectID,
261-
TenantName: cloud.AuthInfo.ProjectName,
262-
DomainID: cloud.AuthInfo.DomainID,
263-
DomainName: cloud.AuthInfo.DomainName,
264-
Region: cloud.RegionName,
265-
},
294+
Global: authopts,
266295
}
267296

268297
cfg := ini.Empty()

0 commit comments

Comments
 (0)