Skip to content

Commit 2537d07

Browse files
dgrove-ossdgrove-oss
committed
port to AppWrapper controller using cert-controller
1 parent 302e369 commit 2537d07

23 files changed

+337
-25
lines changed

Diff for: ‎.github/workflows/e2e_tests.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ jobs:
5555
- name: Setup and start KinD cluster
5656
uses: ./common/github-actions/kind
5757

58+
- name: Deploy Kueue
59+
run: |
60+
make kueue-e2e
61+
5862
- name: Deploy CodeFlare stack
5963
id: deploy
6064
run: |

Diff for: ‎.github/workflows/olm_tests.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ jobs:
5454
- name: Setup and start KinD cluster
5555
uses: ./common/github-actions/kind
5656

57+
- name: Deploy Kueue
58+
run: |
59+
make kueue-e2e
60+
5761
- name: Install OLM
5862
run: |
5963
kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml

Diff for: ‎Makefile

+8-1
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,14 @@ VERSION ?= v0.0.0-dev
1212
BUNDLE_VERSION ?= $(VERSION:v%=%)
1313

1414
# APPWRAPPER_VERSION defines the default version of the AppWrapper controller
15-
APPWRAPPER_VERSION ?= v0.6.4
15+
APPWRAPPER_VERSION ?= v0.7.0
1616
APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper
1717
# Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version)
1818
APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION}
1919

20+
# KUEUE_VERSION defines the default version of Kueue (used for testing)
21+
KUEUE_VERSION ?= v0.6.1
22+
2023
# KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing)
2124
KUBERAY_VERSION ?= v1.0.0
2225

@@ -375,6 +378,10 @@ test-e2e: manifests fmt vet ## Run e2e tests.
375378
kind-e2e: ## Set up e2e KinD cluster
376379
test/e2e/kind.sh
377380

381+
.PHONY: kueue-e2e
382+
kueue-e2e: ## Deploy Kueue
383+
KUEUE_VERSION=$(KUEUE_VERSION) test/e2e/kueue.sh
384+
378385
.PHONY: setup-e2e
379386
setup-e2e: ## Set up e2e tests.
380387
KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh

Diff for: ‎README.md

+2
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ The e2e tests can be executed locally by running the following commands:
3333
```bash
3434
# Create a KinD cluster
3535
make kind-e2e
36+
# Deploy Kueue
37+
make kueue-e2e
3638
# Install the CRDs
3739
make install
3840
```

Diff for: ‎config/crd/appwrapper/kustomization.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
apiVersion: kustomize.config.k8s.io/v1beta1
22
kind: Kustomization
33
resources:
4-
- github.com/project-codeflare/appwrapper/config/crd?ref=v0.6.4
4+
- github.com/project-codeflare/appwrapper/config/crd?ref=v0.7.0

Diff for: ‎config/crd/crd-appwrapper.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ apiVersion: apiextensions.k8s.io/v1
22
kind: CustomResourceDefinition
33
metadata:
44
annotations:
5-
cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME
65
controller-gen.kubebuilder.io/version: v0.14.0
76
name: appwrappers.workload.codeflare.dev
87
spec:
@@ -212,6 +211,10 @@ spec:
212211
phase:
213212
description: Phase of the AppWrapper object
214213
type: string
214+
resettingCount:
215+
description: Retries counts the number of times the AppWrapper has entered the Resetting Phase
216+
format: int32
217+
type: integer
215218
type: object
216219
type: object
217220
served: true

Diff for: ‎config/crd/kustomization.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,15 @@ resources:
55
- crd-appwrapper.yml
66

77
#+kubebuilder:scaffold:crdkustomizeresource
8+
9+
patches:
10+
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
11+
# patches here are for enabling the conversion webhook for each CRD
12+
- path: patches/webhook_in_appwrappers.yaml
13+
#+kubebuilder:scaffold:crdkustomizewebhookpatch
14+
15+
# [WEBHOOK] To enable webhook, uncomment the following section
16+
# the following config is for teaching kustomize how to do kustomization for CRDs.
17+
18+
configurations:
19+
- kustomizeconfig.yaml

Diff for: ‎config/crd/kustomizeconfig.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# This file is for teaching kustomize how to substitute name and namespace reference in CRD
2+
nameReference:
3+
- kind: Service
4+
version: v1
5+
fieldSpecs:
6+
- kind: CustomResourceDefinition
7+
version: v1
8+
group: apiextensions.k8s.io
9+
path: spec/conversion/webhook/clientConfig/service/name
10+
11+
namespace:
12+
- kind: CustomResourceDefinition
13+
version: v1
14+
group: apiextensions.k8s.io
15+
path: spec/conversion/webhook/clientConfig/service/namespace
16+
create: false
17+
18+
varReference:
19+
- path: metadata/annotations

Diff for: ‎config/crd/patches/webhook_in_appwrappers.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# The following patch enables a conversion webhook for the CRD
2+
apiVersion: apiextensions.k8s.io/v1
3+
kind: CustomResourceDefinition
4+
metadata:
5+
name: appwrappers.workload.codeflare.dev
6+
spec:
7+
conversion:
8+
strategy: Webhook
9+
webhook:
10+
clientConfig:
11+
service:
12+
namespace: system
13+
name: webhook-service
14+
path: /convert
15+
conversionReviewVersions:
16+
- v1

Diff for: ‎config/default/kustomization.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,17 @@ bases:
1717
- ../crd
1818
- ../rbac
1919
- ../manager
20+
- ../internalcert
21+
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
22+
# crd/kustomization.yaml
23+
- ../webhook
2024
# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
2125
# - ../prometheus
2226

27+
patches:
28+
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
29+
# crd/kustomization.yaml
30+
- path: manager_webhook_patch.yaml
31+
2332
resources:
2433
- metrics_service.yaml

Diff for: ‎config/default/manager_webhook_patch.yaml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: manager
5+
namespace: system
6+
spec:
7+
template:
8+
spec:
9+
containers:
10+
- name: manager
11+
ports:
12+
- containerPort: 9443
13+
name: webhook-server
14+
protocol: TCP
15+
volumeMounts:
16+
- mountPath: /tmp/k8s-webhook-server/serving-certs
17+
name: cert
18+
readOnly: true
19+
volumes:
20+
- name: cert
21+
secret:
22+
defaultMode: 420
23+
secretName: webhook-server-cert

Diff for: ‎config/internalcert/kustomization.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
resources:
2+
- secret.yaml

Diff for: ‎config/internalcert/secret.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: v1
2+
kind: Secret
3+
metadata:
4+
name: webhook-server-cert
5+
namespace: system

Diff for: ‎config/manager/manager.yaml

-3
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,6 @@ spec:
4747
valueFrom:
4848
fieldRef:
4949
fieldPath: metadata.namespace
50-
# TODO: Disabling webhooks is a temporary hack. Webhooks are required for Kueue integration.
51-
- name: ENABLE_WEBHOOKS
52-
value: "false"
5350
ports:
5451
- containerPort: 8080
5552
protocol: TCP

Diff for: ‎config/rbac/role.yaml

+64-5
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,21 @@ rules:
1919
- update
2020
- watch
2121
- apiGroups:
22-
- batch
22+
- admissionregistration.k8s.io
2323
resources:
24-
- jobs
24+
- mutatingwebhookconfigurations
25+
- validatingwebhookconfigurations
2526
verbs:
26-
- create
27-
- delete
27+
- get
2828
- list
29-
- patch
3029
- update
3130
- watch
31+
- apiGroups:
32+
- apiextensions.k8s.io
33+
resources:
34+
- customresourcedefinitions
35+
verbs:
36+
- list
3237
- apiGroups:
3338
- apps
3439
resources:
@@ -55,6 +60,17 @@ rules:
5560
- subjectaccessreviews
5661
verbs:
5762
- create
63+
- apiGroups:
64+
- batch
65+
resources:
66+
- jobs
67+
verbs:
68+
- create
69+
- delete
70+
- list
71+
- patch
72+
- update
73+
- watch
5874
- apiGroups:
5975
- config.openshift.io
6076
resources:
@@ -157,6 +173,41 @@ rules:
157173
- create
158174
- patch
159175
- update
176+
- apiGroups:
177+
- kueue.x-k8s.io
178+
resources:
179+
- resourceflavors
180+
- workloadpriorityclasses
181+
verbs:
182+
- get
183+
- list
184+
- watch
185+
- apiGroups:
186+
- kueue.x-k8s.io
187+
resources:
188+
- workloads
189+
verbs:
190+
- create
191+
- delete
192+
- get
193+
- list
194+
- patch
195+
- update
196+
- watch
197+
- apiGroups:
198+
- kueue.x-k8s.io
199+
resources:
200+
- workloads/finalizers
201+
verbs:
202+
- update
203+
- apiGroups:
204+
- kueue.x-k8s.io
205+
resources:
206+
- workloads/status
207+
verbs:
208+
- get
209+
- patch
210+
- update
160211
- apiGroups:
161212
- machine.openshift.io
162213
resources:
@@ -182,6 +233,14 @@ rules:
182233
- patch
183234
- update
184235
- watch
236+
- apiGroups:
237+
- scheduling.k8s.io
238+
resources:
239+
- priorityclasses
240+
verbs:
241+
- get
242+
- list
243+
- watch
185244
- apiGroups:
186245
- storage.k8s.io
187246
resources:

Diff for: ‎config/webhook/kustomization.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
resources:
2+
- manifests.yaml
3+
- service.yaml
4+
5+
configurations:
6+
- kustomizeconfig.yaml

Diff for: ‎config/webhook/kustomizeconfig.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# the following config is for teaching kustomize where to look at when substituting nameReference.
2+
# It requires kustomize v2.1.0 or newer to work properly.
3+
nameReference:
4+
- kind: Service
5+
version: v1
6+
fieldSpecs:
7+
- kind: MutatingWebhookConfiguration
8+
group: admissionregistration.k8s.io
9+
path: webhooks/clientConfig/service/name
10+
- kind: ValidatingWebhookConfiguration
11+
group: admissionregistration.k8s.io
12+
path: webhooks/clientConfig/service/name
13+
14+
namespace:
15+
- kind: MutatingWebhookConfiguration
16+
group: admissionregistration.k8s.io
17+
path: webhooks/clientConfig/service/namespace
18+
create: true
19+
- kind: ValidatingWebhookConfiguration
20+
group: admissionregistration.k8s.io
21+
path: webhooks/clientConfig/service/namespace
22+
create: true

Diff for: ‎config/webhook/manifests.yaml

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
---
2+
apiVersion: admissionregistration.k8s.io/v1
3+
kind: MutatingWebhookConfiguration
4+
metadata:
5+
name: mutating-webhook-configuration
6+
webhooks:
7+
- admissionReviewVersions:
8+
- v1
9+
clientConfig:
10+
service:
11+
name: webhook-service
12+
namespace: system
13+
path: /mutate-workload-codeflare-dev-v1beta2-appwrapper
14+
failurePolicy: Fail
15+
name: mappwrapper.kb.io
16+
rules:
17+
- apiGroups:
18+
- workload.codeflare.dev
19+
apiVersions:
20+
- v1beta2
21+
operations:
22+
- CREATE
23+
resources:
24+
- appwrappers
25+
sideEffects: None
26+
---
27+
apiVersion: admissionregistration.k8s.io/v1
28+
kind: ValidatingWebhookConfiguration
29+
metadata:
30+
name: validating-webhook-configuration
31+
webhooks:
32+
- admissionReviewVersions:
33+
- v1
34+
clientConfig:
35+
service:
36+
name: webhook-service
37+
namespace: system
38+
path: /validate-workload-codeflare-dev-v1beta2-appwrapper
39+
failurePolicy: Fail
40+
name: vappwrapper.kb.io
41+
rules:
42+
- apiGroups:
43+
- workload.codeflare.dev
44+
apiVersions:
45+
- v1beta2
46+
operations:
47+
- CREATE
48+
- UPDATE
49+
resources:
50+
- appwrappers
51+
sideEffects: None

0 commit comments

Comments
 (0)