Skip to content

Commit 411250a

Browse files
authored
DVO-230: fix makefile for build tag job (#370)
* (draft) Add new targets replacing the boilerplate ones * (draft) Fix some paths and added missing targets * (draft) Add generate bundle contents step * (prev) Add OLM bundle image creation * (prev) Add clone SRE versions repository * (prev) Add previous operator version logic * (prev) Add OLM bundle previous version replace field * (prev) Add OLM bundle validation * (prev) Add OLM catalog build and validation * (prev) Fix existing script to retrieve grpcurl * (prev) Fix Dockerfile builder source image * (prev) Fixing versions repo accesses and writing * (prev) Add last steps of tagging and pushing built images * (prev) Add dev credentials for Jenkins tests * (prev) Temp fix to build_tag script for testing on Jenkins * (prev) Fix dependency managemend and commands accesses * (prev) Fix missing custom configuration for docker/podman * (prev) Fix container configuration * (prev) Fix container configuration export podman auth * (prev) Fix container configuration * (prev) Fix container configuration * (test) Podman pull from makefile * (test) REGISTRY USER check * (test) Different creds variables * (test) New pull from alt * (test) podman login included temporarily on script * (test) Complete run * (prev) Fix docker versioning * (prev) Fix testing entrypoint * Revoke test token (it was already revoked on quay) * Refactor precheck func * Refactor preparation funcs together * Fix linting and minor changes * Remove testing code * Minor refactor * Remove last local dev items * Removing no longer used targets and scripts * Add missing targets for the CI * Fix mishap with test variables * Add codecov.sh from boilerplate osd operators * Fix CI jobs * Refactor and add validate target * Add minor comment
1 parent b9d0bdd commit 411250a

File tree

5 files changed

+402
-21
lines changed

5 files changed

+402
-21
lines changed

Diff for: Makefile

+98-19
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,105 @@
1-
GOLANGCI_OPTIONAL_CONFIG = .golangci.yml
1+
OPERATOR_NAME = deployment-validation-operator
2+
# Image repository vars
3+
REGISTRY_USER ?= ${QUAY_USER}
4+
REGISTRY_TOKEN ?= ${QUAY_TOKEN}
5+
IMAGE_REGISTRY ?= quay.io
26
IMAGE_REPOSITORY ?= app-sre
3-
REGISTRY_USER = $(QUAY_USER)
4-
REGISTRY_TOKEN = $(QUAY_TOKEN)
7+
IMAGE_NAME ?= ${OPERATOR_NAME}
8+
OPERATOR_IMAGE = ${IMAGE_REGISTRY}/${IMAGE_REPOSITORY}/${IMAGE_NAME}
9+
10+
OLM_CHANNEL ?= alpha
11+
OLM_BUNDLE_IMAGE = ${OPERATOR_IMAGE}-bundle
12+
OLM_CATALOG_IMAGE = ${OPERATOR_IMAGE}-catalog
13+
14+
VERSION_MAJOR ?= 0
15+
VERSION_MINOR ?= 1
16+
COMMIT_COUNT=$(shell git rev-list --count HEAD)
17+
CURRENT_COMMIT=$(shell git rev-parse --short=7 HEAD)
18+
OPERATOR_VERSION=${VERSION_MAJOR}.${VERSION_MINOR}.${COMMIT_COUNT}-g${CURRENT_COMMIT}
19+
OPERATOR_IMAGE_TAG ?= ${OPERATOR_VERSION}
20+
21+
CONTAINER_ENGINE_CONFIG_DIR = .docker
22+
CONTAINER_ENGINE = $(shell command -v podman 2>/dev/null || echo docker --config=$(CONTAINER_ENGINE_CONFIG_DIR))
23+
24+
.PHONY: go-mod-update
25+
go-mod-update:
26+
go mod vendor
27+
28+
GOOS ?= linux
29+
GOENV=GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=1
30+
GOBUILDFLAGS=-gcflags="all=-trimpath=${GOPATH}" -asmflags="all=-trimpath=${GOPATH}"
31+
.PHONY: go-build
32+
go-build: go-mod-update
33+
@echo "## Building the binary..."
34+
${GOENV} go build ${GOBUILDFLAGS} -o build/_output/bin/$(OPERATOR_NAME) .
35+
36+
## Used by CI pipeline ci/prow/lint
37+
GOLANGCI_OPTIONAL_CONFIG = .golangci.yml
38+
GOLANGCI_LINT_CACHE =/tmp/golangci-cache
39+
.PHONY: lint
40+
lint: go-mod-update
41+
@echo "## Running the golangci-lint tool..."
42+
GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE} golangci-lint run -c ${GOLANGCI_OPTIONAL_CONFIG} ./...
43+
44+
## Used by CI pipeline ci/prow/test
45+
TEST_TARGETS = $(shell ${GOENV} go list -e ./... | grep -E -v "/(vendor)/")
46+
.PHONY: test
47+
test: go-mod-update
48+
@echo "## Running the code unit tests..."
49+
${GOENV} go test ${TEST_TARGETS}
50+
51+
## These targets: coverage and test-coverage; are used by the CI pipeline ci/prow/coverage
52+
.PHONY: coverage
53+
coverage:
54+
@echo "## Running code coverage..."
55+
ci/codecov.sh
56+
57+
TESTOPTS :=
58+
.PHONY: test-coverage
59+
test-coverage: go-mod-update
60+
@echo "## Running the code unit tests with coverage..."
61+
${GOENV} go test ${TESTOPTS} ${TEST_TARGETS}
562

6-
# This include must go below the above definitions
7-
include boilerplate/generated-includes.mk
63+
## Used by CI pipeline ci/prow/validate
64+
.PHONY: validate
65+
validate:
66+
@echo "## Perform validation that the folder does not contain extra artifacts..."
67+
test 0 -eq $$(git status --porcelain | wc -l) || (echo "Base folder contains unknown artifacts" >&2 && git --no-pager diff && exit 1)
868

9-
OPERATOR_IMAGE_URI_TEST = $(IMAGE_REGISTRY)/$(IMAGE_REPOSITORY)/$(IMAGE_NAME):test
69+
.PHONY: quay-login
70+
quay-login:
71+
@echo "## Login to quay.io..."
72+
mkdir -p ${CONTAINER_ENGINE_CONFIG_DIR}
73+
export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json
74+
@${CONTAINER_ENGINE} login -u="${REGISTRY_USER}" -p="${REGISTRY_TOKEN}" quay.io
1075

11-
.PHONY: boilerplate-update
12-
boilerplate-update:
13-
@boilerplate/update
76+
.PHONY: docker-build
77+
docker-build:
78+
@echo "## Building the container image..."
79+
${CONTAINER_ENGINE} build --pull -f build/Dockerfile -t ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG} .
80+
${CONTAINER_ENGINE} tag ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG} ${OPERATOR_IMAGE}:latest
1481

15-
.PHONY: docker-test
16-
docker-test:
17-
${CONTAINER_ENGINE} build . -f $(OPERATOR_DOCKERFILE).test -t $(OPERATOR_IMAGE_URI_TEST)
18-
${CONTAINER_ENGINE} run -t $(OPERATOR_IMAGE_URI_TEST)
82+
.PHONY: docker-push
83+
docker-push:
84+
@echo "## Pushing the container image..."
85+
${CONTAINER_ENGINE} push ${OPERATOR_IMAGE}:${OPERATOR_IMAGE_TAG}
86+
${CONTAINER_ENGINE} push ${OPERATOR_IMAGE}:latest
1987

20-
.PHONY: e2e-test
21-
e2e-test:
22-
ginkgo run --tags e2e test/e2e/
88+
## This target is run by build_tag.sh script, triggered by a Jenkins job
89+
.PHONY: docker-publish
90+
docker-publish: quay-login docker-build docker-push
2391

24-
# We are early adopters of the OPM build/push process. Remove this
25-
# override once boilerplate uses that path by default.
26-
build-push: opm-build-push ;
92+
## This target is run by the master branch Jenkins Job
93+
.PHONY: build-push
94+
build-push: docker-publish
95+
CONTAINER_ENGINE="${CONTAINER_ENGINE}" \
96+
CONTAINER_ENGINE_CONFIG_DIR="${CONTAINER_ENGINE_CONFIG_DIR}" \
97+
CURRENT_COMMIT="${CURRENT_COMMIT}" \
98+
OLM_BUNDLE_IMAGE="${OLM_BUNDLE_IMAGE}" \
99+
OLM_CATALOG_IMAGE="${OLM_CATALOG_IMAGE}" \
100+
OLM_CHANNEL="${OLM_CHANNEL}" \
101+
OPERATOR_NAME="${OPERATOR_NAME}" \
102+
OPERATOR_VERSION="${OPERATOR_VERSION}" \
103+
OPERATOR_IMAGE="${OPERATOR_IMAGE}" \
104+
OPERATOR_IMAGE_TAG="${OPERATOR_IMAGE_TAG}" \
105+
build/build_opm_catalog.sh

Diff for: build/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ FROM quay.io/app-sre/boilerplate:image-v5.0.1 AS builder
33
RUN mkdir -p /workdir
44
COPY . /workdir
55
WORKDIR /workdir
6-
RUN make
6+
RUN make go-build
77

88
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.10-1086
99

Diff for: build/build_opm_catalog.sh

+246
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
REPO_ROOT=$(git rev-parse --show-toplevel)
6+
SCRIPT_BUNDLE_CONTENTS="$REPO_ROOT/hack/generate-operator-bundle-contents.py"
7+
BASE_FOLDER=""
8+
DIR_BUNDLE=""
9+
DIR_EXEC=""
10+
DIR_MANIFESTS=""
11+
12+
GOOS=$(go env GOOS)
13+
OPM_VERSION="v1.23.2"
14+
COMMAND_OPM=""
15+
GRPCURL_VERSION="1.7.0"
16+
COMMAND_GRPCURL=""
17+
18+
export REGISTRY_AUTH_FILE=${CONTAINER_ENGINE_CONFIG_DIR}/config.json
19+
20+
OLM_BUNDLE_VERSIONS_REPO="gitlab.cee.redhat.com/service/saas-operator-versions.git"
21+
OLM_BUNDLE_VERSIONS_REPO_FOLDER="versions_repo"
22+
VERSIONS_FILE="deployment-validation-operator/deployment-validation-operator-versions.txt"
23+
PREV_VERSION=""
24+
25+
OLM_BUNDLE_IMAGE_VERSION="${OLM_BUNDLE_IMAGE}:g${CURRENT_COMMIT}"
26+
OLM_BUNDLE_IMAGE_LATEST="${OLM_BUNDLE_IMAGE}:latest"
27+
28+
OLM_CATALOG_IMAGE_VERSION="${OLM_CATALOG_IMAGE}:${CURRENT_COMMIT}"
29+
OLM_CATALOG_IMAGE_LATEST="${OLM_CATALOG_IMAGE}:latest"
30+
31+
function log() {
32+
echo "$(date "+%Y-%m-%d %H:%M:%S") -- ${1}"
33+
}
34+
35+
function precheck_required_files() {
36+
if [[ ! -x "$SCRIPT_BUNDLE_CONTENTS" ]]; then
37+
log "The script $SCRIPT_BUNDLE_CONTENTS cannot be run. Exiting."
38+
return 1
39+
fi
40+
return 0
41+
}
42+
43+
function prepare_temporary_folders() {
44+
BASE_FOLDER=$(mktemp -d --suffix "-$(basename "$0")")
45+
DIR_BUNDLE=$(mktemp -d -p "$BASE_FOLDER" bundle.XXXX)
46+
DIR_MANIFESTS=$(mktemp -d -p "$DIR_BUNDLE" manifests.XXXX)
47+
DIR_EXEC=$(mktemp -d -p "$BASE_FOLDER" bin.XXXX)
48+
}
49+
50+
function download_dependencies() {
51+
cd "$DIR_EXEC"
52+
53+
local opm_url="https://github.com/operator-framework/operator-registry/releases/download/$OPM_VERSION/$GOOS-amd64-opm"
54+
curl -sfL "${opm_url}" -o opm
55+
chmod +x opm
56+
COMMAND_OPM="$DIR_EXEC/opm"
57+
58+
local grpcurl_url="https://github.com/fullstorydev/grpcurl/releases/download/v$GRPCURL_VERSION/grpcurl_${GRPCURL_VERSION}_${GOOS}_x86_64.tar.gz"
59+
curl -sfL "$grpcurl_url" | tar -xz -O grpcurl > "grpcurl"
60+
chmod +x grpcurl
61+
COMMAND_GRPCURL="$DIR_EXEC/grpcurl"
62+
63+
cd ~-
64+
}
65+
66+
67+
function clone_versions_repo() {
68+
local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER"
69+
log " path: $folder"
70+
71+
if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then
72+
log "Using APP_SRE_BOT_PUSH_TOKEN credentials to authenticate"
73+
git clone "https://app:${APP_SRE_BOT_PUSH_TOKEN}@$OLM_BUNDLE_VERSIONS_REPO" "$folder" --quiet
74+
else
75+
git clone "https://$OLM_BUNDLE_VERSIONS_REPO" "$folder" --quiet
76+
fi
77+
}
78+
79+
function set_previous_operator_version() {
80+
local filename="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER/$VERSIONS_FILE"
81+
82+
if [[ ! -a "$filename" ]]; then
83+
log "No file $VERSIONS_FILE exist. Exiting."
84+
exit 1
85+
fi
86+
PREV_VERSION=$(tail -n 1 "$filename" | awk '{print $1}')
87+
}
88+
89+
function setup_environment() {
90+
log "Generating temporary folders to contain artifacts"
91+
prepare_temporary_folders
92+
log " base path: $BASE_FOLDER"
93+
94+
log "Downloading needed commands: opm and grpcurl"
95+
download_dependencies
96+
log " path: $DIR_EXEC"
97+
98+
log "Cloning $OLM_BUNDLE_VERSIONS_REPO"
99+
clone_versions_repo
100+
101+
log "Determining previous operator version checking $VERSIONS_FILE file"
102+
set_previous_operator_version
103+
log " previous version: $PREV_VERSION"
104+
}
105+
106+
function build_opm_bundle() {
107+
# set venv with needed dependencies
108+
python3 -m venv .venv; source .venv/bin/activate; pip install pyyaml
109+
110+
log "Generating patched bundle contents"
111+
$SCRIPT_BUNDLE_CONTENTS --name "$OPERATOR_NAME" \
112+
--current-version "$OPERATOR_VERSION" \
113+
--image "$OPERATOR_IMAGE" \
114+
--image-tag "$OPERATOR_IMAGE_TAG" \
115+
--output-dir "$DIR_MANIFESTS" \
116+
--replaces "$PREV_VERSION"
117+
118+
log "Creating bundle image $OLM_BUNDLE_IMAGE_VERSION"
119+
cd "$DIR_BUNDLE"
120+
${COMMAND_OPM} alpha bundle build --directory "$DIR_MANIFESTS" \
121+
--channels "$OLM_CHANNEL" \
122+
--default "$OLM_CHANNEL" \
123+
--package "$OPERATOR_NAME" \
124+
--tag "$OLM_BUNDLE_IMAGE_VERSION" \
125+
--image-builder "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" \
126+
--overwrite \
127+
1>&2
128+
cd ~-
129+
}
130+
131+
function validate_opm_bundle() {
132+
log "Pushing bundle image $OLM_BUNDLE_IMAGE_VERSION"
133+
$CONTAINER_ENGINE push "$OLM_BUNDLE_IMAGE_VERSION"
134+
135+
log "Validating bundle $OLM_BUNDLE_IMAGE_VERSION"
136+
${COMMAND_OPM} alpha bundle validate --tag "$OLM_BUNDLE_IMAGE_VERSION" \
137+
--image-builder "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')"
138+
}
139+
140+
function build_opm_catalog() {
141+
local FROM_INDEX=""
142+
local PREV_COMMIT=${PREV_VERSION#*g} # remove versioning and the g commit hash prefix
143+
# check if the previous catalog image is available
144+
if [ "$(${CONTAINER_ENGINE} pull "${OLM_CATALOG_IMAGE}":"${PREV_COMMIT}" &> /dev/null; echo $?)" -eq 0 ]; then
145+
FROM_INDEX="--from-index ${OLM_CATALOG_IMAGE}:${PREV_COMMIT}"
146+
log "Index argument is $FROM_INDEX"
147+
fi
148+
149+
log "Creating catalog image $OLM_CATALOG_IMAGE_VERSION using opm"
150+
151+
${COMMAND_OPM} index add --bundles "$OLM_BUNDLE_IMAGE_VERSION" \
152+
--tag "$OLM_CATALOG_IMAGE_VERSION" \
153+
--build-tool "$(basename "$CONTAINER_ENGINE" | awk '{print $1}')" \
154+
$FROM_INDEX
155+
}
156+
157+
function validate_opm_catalog() {
158+
log "Checking that catalog we have built returns the correct version $OPERATOR_VERSION"
159+
160+
local free_port=""
161+
local container_id=""
162+
local catalog_current_version=""
163+
164+
free_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
165+
166+
log "Running $OLM_CATALOG_IMAGE_VERSION and exposing $free_port"
167+
container_id=$(${CONTAINER_ENGINE} run -d -p "$free_port:50051" "$OLM_CATALOG_IMAGE_VERSION")
168+
169+
log "Getting current version from running catalog"
170+
catalog_current_version=$(
171+
${COMMAND_GRPCURL} -plaintext -d '{"name": "'"$OPERATOR_NAME"'"}' \
172+
"localhost:$free_port" api.Registry/GetPackage | \
173+
jq -r '.channels[] | select(.name=="'"$OLM_CHANNEL"'") | .csvName' | \
174+
sed "s/$OPERATOR_NAME\.//"
175+
)
176+
log " catalog version: $catalog_current_version"
177+
178+
log "Removing docker container $container_id"
179+
${CONTAINER_ENGINE} rm -f "$container_id"
180+
181+
if [[ "$catalog_current_version" != "v$OPERATOR_VERSION" ]]; then
182+
log "Version from catalog $catalog_current_version != v$OPERATOR_VERSION"
183+
return 1
184+
fi
185+
}
186+
187+
function update_versions_repo() {
188+
log "Adding the current version $OPERATOR_VERSION to the bundle versions file in $OLM_BUNDLE_VERSIONS_REPO"
189+
local folder="$BASE_FOLDER/$OLM_BUNDLE_VERSIONS_REPO_FOLDER"
190+
191+
cd "$folder"
192+
193+
echo "$OPERATOR_VERSION" >> "$VERSIONS_FILE"
194+
git add .
195+
message="add version $OPERATOR_VERSION
196+
197+
replaces $PREV_VERSION"
198+
git commit -m "$message"
199+
200+
log "Pushing the repository changes to $OLM_BUNDLE_VERSIONS_REPO into master branch"
201+
git push origin master
202+
cd ~-
203+
}
204+
205+
function tag_and_push_images() {
206+
log "Tagging bundle image $OLM_BUNDLE_IMAGE_VERSION as $OLM_BUNDLE_IMAGE_LATEST"
207+
${CONTAINER_ENGINE} tag "$OLM_BUNDLE_IMAGE_VERSION" "$OLM_BUNDLE_IMAGE_LATEST"
208+
209+
log "Tagging catalog image $OLM_CATALOG_IMAGE_VERSION as $OLM_CATALOG_IMAGE_LATEST"
210+
${CONTAINER_ENGINE} tag "$OLM_CATALOG_IMAGE_VERSION" "$OLM_CATALOG_IMAGE_LATEST"
211+
212+
log "Pushing catalog image $OLM_CATALOG_IMAGE_VERSION"
213+
${CONTAINER_ENGINE} push "$OLM_CATALOG_IMAGE_VERSION"
214+
215+
log "Pushing bundle image $OLM_CATALOG_IMAGE_LATEST"
216+
${CONTAINER_ENGINE} push "$OLM_CATALOG_IMAGE_LATEST"
217+
218+
log "Pushing bundle image $OLM_BUNDLE_IMAGE_LATEST"
219+
${CONTAINER_ENGINE} push "$OLM_BUNDLE_IMAGE_LATEST"
220+
}
221+
222+
function main() {
223+
log "Building $OPERATOR_NAME version $OPERATOR_VERSION"
224+
225+
precheck_required_files || return 1
226+
227+
setup_environment
228+
229+
build_opm_bundle
230+
validate_opm_bundle
231+
232+
build_opm_catalog
233+
validate_opm_catalog
234+
235+
if [[ -n "${APP_SRE_BOT_PUSH_TOKEN:-}" ]]; then
236+
update_versions_repo
237+
else
238+
log "APP_SRE_BOT_PUSH_TOKEN credentials were not found"
239+
log "it will be necessary to manually update $OLM_BUNDLE_VERSIONS_REPO repo"
240+
fi
241+
tag_and_push_images
242+
}
243+
244+
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
245+
main
246+
fi

Diff for: build_tag.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#!/bin/bash
22

3+
## This script is the entry point for the Jenkins job: deployment-validation-operator build tag
4+
## It builds a new image and pushes it to the dv-operator repository on quay.io.
35
make \
46
OPERATOR_IMAGE_TAG=$(echo $GIT_BRANCH|cut -d"/" -f3) \
57
IMAGE_REPOSITORY="deployment-validation-operator" \
68
IMAGE_NAME="dv-operator" \
79
REGISTRY_USER=$(echo $QUAY_USER) \
810
REGISTRY_TOKEN=$(echo $QUAY_TOKEN) \
9-
docker-push
11+
docker-publish

0 commit comments

Comments
 (0)