Skip to content

Commit 79cd815

Browse files
authored
add GCP Filestore WIF tests (#56184)
1 parent 25befd5 commit 79cd815

19 files changed

+484
-0
lines changed

ci-operator/config/openshift/gcp-filestore-csi-driver-operator/openshift-gcp-filestore-csi-driver-operator-main.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,14 @@ tests:
180180
env:
181181
PROJECT_NAME: gcp-filestore-csi-driver-operator
182182
workflow: openshift-ci-security
183+
- always_run: false
184+
as: e2e-gcp-manual-oidc-filestore
185+
optional: true
186+
steps:
187+
cluster_profile: gcp
188+
dependencies:
189+
OO_INDEX: ci-index-gcp-filestore-csi-driver-operator-bundle
190+
workflow: openshift-e2e-gcp-manual-oidc-workload-identity-filestore
183191
zz_generated_metadata:
184192
branch: main
185193
org: openshift

ci-operator/jobs/openshift/gcp-filestore-csi-driver-operator/openshift-gcp-filestore-csi-driver-operator-main-presubmits.yaml

+73
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,79 @@ presubmits:
199199
secret:
200200
secretName: result-aggregator
201201
trigger: (?m)^/test( | .* )e2e-gcp-csi-n4,?($|\s.*)
202+
- agent: kubernetes
203+
always_run: false
204+
branches:
205+
- ^main$
206+
- ^main-
207+
cluster: build02
208+
context: ci/prow/e2e-gcp-manual-oidc-filestore
209+
decorate: true
210+
labels:
211+
ci-operator.openshift.io/cloud: gcp
212+
ci-operator.openshift.io/cloud-cluster-profile: gcp
213+
ci.openshift.io/generator: prowgen
214+
pj-rehearse.openshift.io/can-be-rehearsed: "true"
215+
name: pull-ci-openshift-gcp-filestore-csi-driver-operator-main-e2e-gcp-manual-oidc-filestore
216+
optional: true
217+
rerun_command: /test e2e-gcp-manual-oidc-filestore
218+
spec:
219+
containers:
220+
- args:
221+
- --gcs-upload-secret=/secrets/gcs/service-account.json
222+
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
223+
- --lease-server-credentials-file=/etc/boskos/credentials
224+
- --report-credentials-file=/etc/report/credentials
225+
- --secret-dir=/secrets/ci-pull-credentials
226+
- --target=e2e-gcp-manual-oidc-filestore
227+
command:
228+
- ci-operator
229+
image: ci-operator:latest
230+
imagePullPolicy: Always
231+
name: ""
232+
resources:
233+
requests:
234+
cpu: 10m
235+
volumeMounts:
236+
- mountPath: /etc/boskos
237+
name: boskos
238+
readOnly: true
239+
- mountPath: /secrets/ci-pull-credentials
240+
name: ci-pull-credentials
241+
readOnly: true
242+
- mountPath: /secrets/gcs
243+
name: gcs-credentials
244+
readOnly: true
245+
- mountPath: /secrets/manifest-tool
246+
name: manifest-tool-local-pusher
247+
readOnly: true
248+
- mountPath: /etc/pull-secret
249+
name: pull-secret
250+
readOnly: true
251+
- mountPath: /etc/report
252+
name: result-aggregator
253+
readOnly: true
254+
serviceAccountName: ci-operator
255+
volumes:
256+
- name: boskos
257+
secret:
258+
items:
259+
- key: credentials
260+
path: credentials
261+
secretName: boskos-credentials
262+
- name: ci-pull-credentials
263+
secret:
264+
secretName: ci-pull-credentials
265+
- name: manifest-tool-local-pusher
266+
secret:
267+
secretName: manifest-tool-local-pusher
268+
- name: pull-secret
269+
secret:
270+
secretName: registry-pull-credentials
271+
- name: result-aggregator
272+
secret:
273+
secretName: result-aggregator
274+
trigger: (?m)^/test( | .* )e2e-gcp-manual-oidc-filestore,?($|\s.*)
202275
- agent: kubernetes
203276
always_run: true
204277
branches:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
approvers:
2+
- dobsonj
3+
- gnufied
4+
- jsafrane
5+
- RomanBednar
6+
- tsmetana
7+
- mpatlasov
8+
reviewers:
9+
- dobsonj
10+
- gnufied
11+
- jsafrane
12+
- RomanBednar
13+
- tsmetana
14+
- mpatlasov
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
set -o errexit
3+
set -o nounset
4+
set -o pipefail
5+
6+
# logger function prints standard logs
7+
logger() {
8+
local level="$1"
9+
local message="$2"
10+
local timestamp
11+
12+
# Generate a timestamp for the log entry
13+
timestamp=$(date +"%Y-%m-%d %H:%M:%S")
14+
15+
# Print the log message with the level and timestamp
16+
echo "[$timestamp] [$level] $message"
17+
}
18+
19+
function run_command() {
20+
local CMD="$1"
21+
echo "Running Command: ${CMD}"
22+
eval "${CMD}"
23+
}
24+
25+
logger "INFO" "Starting GCP Filestore Workload Identity Federation cleanup"
26+
27+
if [ -f "${SHARED_DIR}/gcp-filestore-service-account-email" ]; then
28+
SERVICE_ACCOUNT_EMAIL=$(cat "${SHARED_DIR}"/gcp-filestore-service-account-email)
29+
else
30+
logger "INFO" "Service account email file not found in ${SHARED_DIR} - nothing to clean up."
31+
exit 0
32+
fi
33+
34+
# For disconnected or otherwise unreachable environments, we want to
35+
# have steps use an HTTP(S) proxy to reach the API server. This proxy
36+
# configuration file should export HTTP_PROXY, HTTPS_PROXY, and NO_PROXY
37+
# environment variables, as well as their lowercase equivalents (note
38+
# that libcurl doesn't recognize the uppercase variables).
39+
if test -f "${SHARED_DIR}/proxy-conf.sh"
40+
then
41+
# shellcheck disable=SC1091
42+
source "${SHARED_DIR}/proxy-conf.sh"
43+
logger "INFO" "Loaded proxy configuration from ${SHARED_DIR}/proxy-conf.sh"
44+
fi
45+
46+
GOOGLE_PROJECT_ID="$(<${CLUSTER_PROFILE_DIR}/openshift_gcp_project)"
47+
export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json"
48+
sa_email=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE})
49+
if ! gcloud auth list | grep -E "\*\s+${sa_email}"
50+
then
51+
logger "INFO" "Activating service account: ${sa_email}"
52+
cmd="gcloud auth activate-service-account --key-file=\"${GCP_SHARED_CREDENTIALS_FILE}\""
53+
run_command "$cmd"
54+
cmd="gcloud config set project \"${GOOGLE_PROJECT_ID}\""
55+
run_command "$cmd"
56+
logger "INFO" "Service account activated and project set to ${GOOGLE_PROJECT_ID}"
57+
fi
58+
59+
# Ref: TBD (no Red Hat docs available yet). Google doc: https://cloud.google.com/iam/docs/workload-identity-federation-with-kubernetes#create_the_workload_identity_pool_and_provider
60+
logger "INFO" "Starting cleanup of GCP Filestore cloud infrastructure for Workload Identity Federation"
61+
62+
## TODO: replace cleanup steps with ccoctl automation if this ever gets implemented
63+
## TODO: alternatively, this could be documented later in the docs, make sure the code below is aligned with the official procedure
64+
65+
# Delete the Google cloud service account
66+
logger "INFO" "Deleting Google cloud service account: ${SERVICE_ACCOUNT_EMAIL}"
67+
cmd="gcloud --quiet iam service-accounts delete \"$SERVICE_ACCOUNT_EMAIL\""
68+
run_command "$cmd"
69+
logger "INFO" "Service account removed"
70+
71+
logger "INFO" "GCP Filestore Workload Identity Federation cleanup completed"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"path": "ipi/conf/gcp/filestore-workload-identity-federation-cleanup/ipi-conf-gcp-filestore-workload-identity-federation-cleanup-ref.yaml",
3+
"owners": {
4+
"approvers": [
5+
"dobsonj",
6+
"gnufied",
7+
"jsafrane",
8+
"RomanBednar",
9+
"tsmetana",
10+
"mpatlasov"
11+
],
12+
"reviewers": [
13+
"dobsonj",
14+
"gnufied",
15+
"jsafrane",
16+
"RomanBednar",
17+
"tsmetana",
18+
"mpatlasov"
19+
]
20+
}
21+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
ref:
2+
as: ipi-conf-gcp-filestore-workload-identity-federation-cleanup
3+
from_image:
4+
namespace: ocp
5+
name: "4.12"
6+
tag: upi-installer
7+
# inject oc binary
8+
cli: latest
9+
commands: ipi-conf-gcp-filestore-workload-identity-federation-cleanup-commands.sh
10+
resources:
11+
requests:
12+
cpu: 10m
13+
memory: 100Mi
14+
documentation: |-
15+
The script cleans up GCP Filestore Workload Identity Federation resources and service accounts.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
approvers:
2+
- dobsonj
3+
- gnufied
4+
- jsafrane
5+
- RomanBednar
6+
- tsmetana
7+
- mpatlasov
8+
reviewers:
9+
- dobsonj
10+
- gnufied
11+
- jsafrane
12+
- RomanBednar
13+
- tsmetana
14+
- mpatlasov
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#!/bin/bash
2+
set -o errexit
3+
set -o nounset
4+
set -o pipefail
5+
6+
INFRA_NAME=${NAMESPACE}-${UNIQUE_HASH}
7+
8+
# logger function prints standard logs
9+
logger() {
10+
local level="$1"
11+
local message="$2"
12+
local timestamp
13+
14+
# Generate a timestamp for the log entry
15+
timestamp=$(date +"%Y-%m-%d %H:%M:%S")
16+
17+
# Print the log message with the level and timestamp
18+
echo "[$timestamp] [$level] $message"
19+
}
20+
21+
function run_command() {
22+
local CMD="$1"
23+
echo "Running Command: ${CMD}" >&2
24+
eval "${CMD}"
25+
}
26+
27+
logger "INFO" "Starting GCP Filestore Workload Identity Federation configuration"
28+
29+
# For disconnected or otherwise unreachable environments, we want to
30+
# have steps use an HTTP(S) proxy to reach the API server. This proxy
31+
# configuration file should export HTTP_PROXY, HTTPS_PROXY, and NO_PROXY
32+
# environment variables, as well as their lowercase equivalents (note
33+
# that libcurl doesn't recognize the uppercase variables).
34+
if test -f "${SHARED_DIR}/proxy-conf.sh"
35+
then
36+
# shellcheck disable=SC1091
37+
source "${SHARED_DIR}/proxy-conf.sh"
38+
logger "INFO" "Loaded proxy configuration from ${SHARED_DIR}/proxy-conf.sh"
39+
fi
40+
41+
GOOGLE_PROJECT_ID="$(<${CLUSTER_PROFILE_DIR}/openshift_gcp_project)"
42+
export GCP_SHARED_CREDENTIALS_FILE="${CLUSTER_PROFILE_DIR}/gce.json"
43+
GCP_SERVICE_ACCOUNT=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE})
44+
SA_SUFFIX=${GCP_SERVICE_ACCOUNT#*@}
45+
SA_EMAIL=$(jq -r .client_email ${GCP_SHARED_CREDENTIALS_FILE})
46+
47+
if ! gcloud auth list | grep -E "\*\s+${SA_EMAIL}"
48+
then
49+
logger "INFO" "Authenticating with GCP service account"
50+
CMD="gcloud auth activate-service-account --key-file=\"${GCP_SHARED_CREDENTIALS_FILE}\""
51+
run_command "${CMD}"
52+
CMD="gcloud config set project \"${GOOGLE_PROJECT_ID}\""
53+
run_command "${CMD}"
54+
logger "INFO" "Successfully authenticated with GCP service account"
55+
fi
56+
57+
# Ref: TBD (no Red Hat docs available yet). Google doc: https://cloud.google.com/iam/docs/workload-identity-federation-with-kubernetes#create_the_workload_identity_pool_and_provider
58+
logger "INFO" "Create GCP Filestore cloud infrastructure for Workload Identity Federation"
59+
60+
## TODO: replace steps to manually create the service account and bindings with ccoctl automation if this ever gets implemented
61+
## TODO: alternatively, this could be documented later in the docs, make sure the code below is aligned with the official procedure
62+
63+
# Create Google cloud service account for GCP Filestore Operator (name length must be between 6 and 30)
64+
SERVICE_ACCOUNT_NAME="gcp-filestore-sa-${UNIQUE_HASH}"-`echo $RANDOM`
65+
SERVICE_ACCOUNT_EMAIL="${SERVICE_ACCOUNT_NAME}@${SA_SUFFIX}"
66+
67+
logger "INFO" "Creating GCP IAM service account: ${SERVICE_ACCOUNT_EMAIL}"
68+
CMD="gcloud iam service-accounts create \"$SERVICE_ACCOUNT_NAME\" --display-name=\"$SERVICE_ACCOUNT_NAME\""
69+
run_command "$CMD"
70+
71+
# Obtain project number, pool ID, and provider ID
72+
# We assume the pool ID is the same as the infrastructure name and reuse it. If this won't work well in the future we can create a new pool for the operator.
73+
logger "INFO" "Obtaining project details and identity pool information"
74+
CMD="gcloud projects describe \"$GOOGLE_PROJECT_ID\" --format=\"value(projectNumber)\""
75+
PROJECT_NUMBER=$(run_command "${CMD}")
76+
POOL_ID=${INFRA_NAME}
77+
PROVIDER_ID=${INFRA_NAME}
78+
logger "INFO" "Project number: ${PROJECT_NUMBER}, Pool ID: ${POOL_ID}, Provider ID: ${PROVIDER_ID}"
79+
80+
# Set roles for the service account - this should match roles CredentialsRequest of Filestore Operator
81+
logger "INFO" "Setting IAM roles for the service account"
82+
CMD="gcloud projects add-iam-policy-binding \"$GOOGLE_PROJECT_ID\" --member=\"serviceAccount:$SERVICE_ACCOUNT_EMAIL\" --role=\"roles/file.editor\" --condition=None"
83+
run_command "${CMD}"
84+
CMD="gcloud projects add-iam-policy-binding \"$GOOGLE_PROJECT_ID\" --member=\"serviceAccount:$SERVICE_ACCOUNT_EMAIL\" --role=\"roles/resourcemanager.tagUser\" --condition=None"
85+
run_command "${CMD}"
86+
logger "INFO" "IAM roles set successfully"
87+
88+
# Allow OpenShift service accounts to impersonate Google cloud service account
89+
logger "INFO" "Configuring Workload Identity Federation for OpenShift service accounts"
90+
CMD="gcloud iam service-accounts add-iam-policy-binding \"$SERVICE_ACCOUNT_EMAIL\" --member=\"principal://iam.googleapis.com/projects/$PROJECT_NUMBER/locations/global/workloadIdentityPools/$POOL_ID/subject/system:serviceaccount:openshift-cluster-csi-drivers:gcp-filestore-csi-driver-controller-sa\" --role=roles/iam.workloadIdentityUser"
91+
run_command "${CMD}"
92+
CMD="gcloud iam service-accounts add-iam-policy-binding \"$SERVICE_ACCOUNT_EMAIL\" --member=\"principal://iam.googleapis.com/projects/$PROJECT_NUMBER/locations/global/workloadIdentityPools/$POOL_ID/subject/system:serviceaccount:openshift-cluster-csi-drivers:gcp-filestore-csi-driver-operator\" --role=roles/iam.workloadIdentityUser"
93+
run_command "${CMD}"
94+
logger "INFO" "Workload Identity Federation configured successfully"
95+
96+
# Store GCP WIF variables to a known location to be used later in chain as OO_CONFIG_ENVVARS used by `optional-operators-subscribe` step
97+
logger "INFO" "Storing GCP Workload Identity Federation variables"
98+
echo "$POOL_ID" > "${SHARED_DIR}"/gcp-filestore-pool-id
99+
echo "$PROVIDER_ID" > "${SHARED_DIR}"/gcp-filestore-provider-id
100+
echo "$SERVICE_ACCOUNT_EMAIL" > "${SHARED_DIR}"/gcp-filestore-service-account-email
101+
printf '"%s"' "$PROJECT_NUMBER" > "${SHARED_DIR}"/gcp-filestore-project-number
102+
103+
logger "INFO" "GCP Workload Identity Federation variables stored successfully in ${SHARED_DIR}:"
104+
logger "INFO" " Pool ID: $(cat ${SHARED_DIR}/gcp-filestore-pool-id)"
105+
logger "INFO" " Provider ID: $(cat ${SHARED_DIR}/gcp-filestore-provider-id)"
106+
logger "INFO" " Service Account Email: $(cat ${SHARED_DIR}/gcp-filestore-service-account-email)"
107+
logger "INFO" " Project Number: $(cat ${SHARED_DIR}/gcp-filestore-project-number)"
108+
109+
logger "INFO" "GCP Filestore Workload Identity Federation configuration completed"
110+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"path": "ipi/conf/gcp/filestore-workload-identity-federation/ipi-conf-gcp-filestore-workload-identity-federation-ref.yaml",
3+
"owners": {
4+
"approvers": [
5+
"dobsonj",
6+
"gnufied",
7+
"jsafrane",
8+
"RomanBednar",
9+
"tsmetana",
10+
"mpatlasov"
11+
],
12+
"reviewers": [
13+
"dobsonj",
14+
"gnufied",
15+
"jsafrane",
16+
"RomanBednar",
17+
"tsmetana",
18+
"mpatlasov"
19+
]
20+
}
21+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
ref:
2+
as: ipi-conf-gcp-filestore-workload-identity-federation
3+
from_image:
4+
namespace: ocp
5+
name: "4.12"
6+
tag: upi-installer
7+
# inject oc binary
8+
cli: latest
9+
commands: ipi-conf-gcp-filestore-workload-identity-federation-commands.sh
10+
resources:
11+
requests:
12+
cpu: 10m
13+
memory: 100Mi
14+
documentation: |-
15+
The script automates the setup of GCP Workload Identity Federation for OpenShift's Filestore service by creating and
16+
configuring necessary service accounts, permissions, and identity bindings between GCP and OpenShift environments.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
approvers:
2+
- dobsonj
3+
- gnufied
4+
- jsafrane
5+
- RomanBednar
6+
- tsmetana
7+
- mpatlasov

0 commit comments

Comments
 (0)