Skip to content

Commit 6849b92

Browse files
yitzhaktalyitzhaktal
committed
DEVOPS-2694 - Update the lightrun-k8s-operator deployment to mount Secrets as files via volumes instead of exposing them as environment variables in containers.
1 parent 2be38da commit 6849b92

File tree

4 files changed

+147
-94
lines changed

4 files changed

+147
-94
lines changed

‎internal/controller/lightrunjavaagent_controller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ func (r *LightrunJavaAgentReconciler) reconcileDeployment(ctx context.Context, l
252252

253253
// Create config map
254254
log.V(2).Info("Reconciling config map with agent configuration")
255-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
255+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
256256
if err != nil {
257257
log.Error(err, "unable to create configMap")
258258
return r.errorStatus(ctx, lightrunJavaAgent, err)
@@ -493,7 +493,7 @@ func (r *LightrunJavaAgentReconciler) reconcileStatefulSet(ctx context.Context,
493493

494494
// Create config map
495495
log.V(2).Info("Reconciling config map with agent configuration")
496-
configMap, err := r.createAgentConfig(lightrunJavaAgent)
496+
configMap, err := r.createAgentConfig(lightrunJavaAgent, secret)
497497
if err != nil {
498498
log.Error(err, "unable to create configMap")
499499
return r.errorStatus(ctx, lightrunJavaAgent, err)

‎internal/controller/patch_funcs.go

Lines changed: 72 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ const (
2727
annotationAgentName = "lightrun.com/lightrunjavaagent"
2828
)
2929

30-
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent) (corev1.ConfigMap, error) {
30+
func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) (corev1.ConfigMap, error) {
3131
populateTags(lightrunJavaAgent.Spec.AgentTags, lightrunJavaAgent.Spec.AgentName, &metadata)
3232
jsonString, err := json.Marshal(metadata)
3333
if err != nil {
@@ -52,26 +52,28 @@ func (r *LightrunJavaAgentReconciler) createAgentConfig(lightrunJavaAgent *agent
5252
}
5353

5454
func (r *LightrunJavaAgentReconciler) patchDeployment(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, cmDataHash uint64) error {
55-
5655
// init spec.template.spec
5756
deploymentApplyConfig.WithSpec(
5857
appsv1ac.DeploymentSpec().WithTemplate(
5958
corev1ac.PodTemplateSpec().WithSpec(
6059
corev1ac.PodSpec(),
6160
).WithAnnotations(map[string]string{
6261
annotationConfigMapHash: fmt.Sprint(cmDataHash),
63-
},
64-
),
62+
}),
6563
),
6664
).WithAnnotations(map[string]string{
6765
annotationAgentName: lightrunJavaAgent.Name,
6866
})
6967
r.addVolume(deploymentApplyConfig, lightrunJavaAgent)
7068
r.addInitContainer(deploymentApplyConfig, lightrunJavaAgent, secret)
71-
err = r.patchAppContainers(lightrunJavaAgent, origDeployment, deploymentApplyConfig)
69+
err := r.patchAppContainers(lightrunJavaAgent, origDeployment, deploymentApplyConfig)
7270
if err != nil {
7371
return err
7472
}
73+
deploymentApplyConfig.Spec.Template.Spec.WithSecurityContext(
74+
corev1ac.PodSecurityContext().
75+
WithFSGroup(1000),
76+
)
7577
return nil
7678
}
7779

@@ -99,54 +101,53 @@ func (r *LightrunJavaAgentReconciler) addVolume(deploymentApplyConfig *appsv1ac.
99101
}
100102

101103
func (r *LightrunJavaAgentReconciler) addInitContainer(deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration, lightrunJavaAgent *agentv1beta.LightrunJavaAgent, secret *corev1.Secret) {
102-
103104
deploymentApplyConfig.Spec.Template.Spec.WithInitContainers(
104105
corev1ac.Container().
105106
WithName(initContainerName).
106107
WithImage(lightrunJavaAgent.Spec.InitContainer.Image).
107108
WithVolumeMounts(
108109
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
109110
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
110-
).WithEnv(
111-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
112-
corev1ac.EnvVarSource().WithSecretKeyRef(
113-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
114-
),
115-
),
116-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
117-
corev1ac.EnvVarSource().WithSecretKeyRef(
118-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
119-
),
120-
),
121-
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
122-
).
111+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
112+
).
113+
WithEnv(
114+
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
115+
).
116+
WithSecurityContext(
117+
corev1ac.SecurityContext().
118+
WithReadOnlyRootFilesystem(true).
119+
WithAllowPrivilegeEscalation(false).
120+
WithRunAsNonRoot(true).
121+
WithRunAsUser(1000),
122+
).
123123
WithResources(
124124
corev1ac.ResourceRequirements().
125125
WithLimits(
126126
corev1.ResourceList{
127127
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
128-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 500 * 10^6 = 500M
128+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
129129
},
130-
).WithRequests(
131-
corev1.ResourceList{
132-
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
133-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
134-
},
135-
),
136-
).
137-
WithSecurityContext(
138-
corev1ac.SecurityContext().
139-
WithCapabilities(
140-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
141130
).
142-
WithAllowPrivilegeEscalation(false).
143-
WithRunAsNonRoot(true).
144-
WithSeccompProfile(
145-
corev1ac.SeccompProfile().
146-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
131+
WithRequests(
132+
corev1.ResourceList{
133+
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
134+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
135+
},
147136
),
148137
),
149138
)
139+
140+
// Add volume for secret with proper permissions
141+
deploymentApplyConfig.Spec.Template.Spec.WithVolumes(
142+
corev1ac.Volume().WithName("lightrun-secret").
143+
WithSecret(corev1ac.SecretVolumeSource().
144+
WithSecretName(secret.Name).
145+
WithItems(
146+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
147+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
148+
).
149+
WithDefaultMode(0440)),
150+
)
150151
}
151152

152153
func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origDeployment *appsv1.Deployment, deploymentApplyConfig *appsv1ac.DeploymentApplyConfiguration) error {
@@ -167,8 +168,7 @@ func (r *LightrunJavaAgentReconciler) patchAppContainers(lightrunJavaAgent *agen
167168
}
168169
}
169170
if !found {
170-
err = errors.New("unable to find matching container to patch")
171-
return err
171+
return errors.New("unable to find matching container to patch")
172172
}
173173
return nil
174174
}
@@ -230,21 +230,18 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSet(lightrunJavaAgent *agentv
230230
corev1ac.PodSpec(),
231231
).WithAnnotations(map[string]string{
232232
annotationConfigMapHash: fmt.Sprint(cmDataHash),
233-
},
234-
),
233+
}),
235234
),
236235
).WithAnnotations(map[string]string{
237236
annotationAgentName: lightrunJavaAgent.Name,
238237
})
239238

240239
// Add volumes to the StatefulSet
241240
r.addVolumeToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent)
242-
243241
// Add init container to the StatefulSet
244242
r.addInitContainerToStatefulSet(statefulSetApplyConfig, lightrunJavaAgent, secret)
245-
246243
// Patch app containers in the StatefulSet
247-
err = r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
244+
err := r.patchStatefulSetAppContainers(lightrunJavaAgent, origStatefulSet, statefulSetApplyConfig)
248245
if err != nil {
249246
return err
250247
}
@@ -271,6 +268,15 @@ func (r *LightrunJavaAgentReconciler) addVolumeToStatefulSet(statefulSetApplyCon
271268
corev1ac.KeyToPath().WithKey("metadata").WithPath("agent.metadata.json"),
272269
),
273270
),
271+
).WithVolumes(
272+
corev1ac.Volume().WithName("lightrun-secret").
273+
WithSecret(corev1ac.SecretVolumeSource().
274+
WithSecretName(secret.Name).
275+
WithItems(
276+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
277+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
278+
).
279+
WithDefaultMode(0440)),
274280
)
275281
}
276282

@@ -282,46 +288,43 @@ func (r *LightrunJavaAgentReconciler) addInitContainerToStatefulSet(statefulSetA
282288
WithVolumeMounts(
283289
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
284290
corev1ac.VolumeMount().WithName(cmVolumeName).WithMountPath("/tmp/cm/"),
291+
corev1ac.VolumeMount().WithName("lightrun-secret").WithMountPath("/etc/lightrun/secret").WithReadOnly(true),
285292
).WithEnv(
286-
corev1ac.EnvVar().WithName("LIGHTRUN_KEY").WithValueFrom(
287-
corev1ac.EnvVarSource().WithSecretKeyRef(
288-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("lightrun_key"),
289-
),
290-
),
291-
corev1ac.EnvVar().WithName("PINNED_CERT").WithValueFrom(
292-
corev1ac.EnvVarSource().WithSecretKeyRef(
293-
corev1ac.SecretKeySelector().WithName(secret.Name).WithKey("pinned_cert_hash"),
294-
),
295-
),
296293
corev1ac.EnvVar().WithName("LIGHTRUN_SERVER").WithValue(lightrunJavaAgent.Spec.ServerHostname),
297294
).
295+
WithSecurityContext(
296+
corev1ac.SecurityContext().
297+
WithReadOnlyRootFilesystem(true).
298+
WithAllowPrivilegeEscalation(false).
299+
WithRunAsNonRoot(true).
300+
WithRunAsUser(1000),
301+
).
298302
WithResources(
299303
corev1ac.ResourceRequirements().
300304
WithLimits(
301305
corev1.ResourceList{
302306
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
303-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 64M
307+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 500 * 10^6 = 500M
304308
},
305309
).WithRequests(
306310
corev1.ResourceList{
307311
corev1.ResourceCPU: *resource.NewMilliQuantity(int64(50), resource.BinarySI),
308-
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)),
312+
corev1.ResourceMemory: *resource.NewScaledQuantity(int64(64), resource.Scale(6)), // 64M
309313
},
310314
),
311-
).
312-
WithSecurityContext(
313-
corev1ac.SecurityContext().
314-
WithCapabilities(
315-
corev1ac.Capabilities().WithDrop(corev1.Capability("ALL")),
316-
).
317-
WithAllowPrivilegeEscalation(false).
318-
WithRunAsNonRoot(true).
319-
WithSeccompProfile(
320-
corev1ac.SeccompProfile().
321-
WithType(corev1.SeccompProfileTypeRuntimeDefault),
322-
),
323315
),
324316
)
317+
318+
statefulSetApplyConfig.Spec.Template.Spec.WithVolumes(
319+
corev1ac.Volume().WithName("lightrun-secret").
320+
WithSecret(corev1ac.SecretVolumeSource().
321+
WithSecretName(secret.Name).
322+
WithItems(
323+
corev1ac.KeyToPath().WithKey("lightrun_key").WithPath("lightrun_key"),
324+
corev1ac.KeyToPath().WithKey("pinned_cert_hash").WithPath("pinned_cert_hash"),
325+
).
326+
WithDefaultMode(0440)),
327+
)
325328
}
326329

327330
func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJavaAgent *agentv1beta.LightrunJavaAgent, origStatefulSet *appsv1.StatefulSet, statefulSetApplyConfig *appsv1ac.StatefulSetApplyConfiguration) error {
@@ -335,22 +338,20 @@ func (r *LightrunJavaAgentReconciler) patchStatefulSetAppContainers(lightrunJava
335338
WithName(container.Name).
336339
WithImage(container.Image).
337340
WithVolumeMounts(
338-
corev1ac.VolumeMount().WithMountPath(lightrunJavaAgent.Spec.InitContainer.SharedVolumeMountPath).WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName),
341+
corev1ac.VolumeMount().WithName(lightrunJavaAgent.Spec.InitContainer.SharedVolumeName).WithMountPath("/tmp/"),
339342
),
340343
)
341344
}
342345
}
343346
}
344347
if !found {
345-
err = errors.New("unable to find matching container to patch")
346-
return err
348+
return errors.New("unable to find matching container to patch")
347349
}
348350
return nil
349351
}
350352

351353
// configMapDataHash calculates a hash of the ConfigMap data to detect changes
352354
func configMapDataHash(cmData map[string]string) uint64 {
353-
// Combine all data values into a single string for hashing
354355
var hashString string
355356
for _, v := range cmData {
356357
hashString += v

‎lightrun-init-agent/Dockerfile

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
ARG base_image_tag=alpine-3.20.0-r1
22

33
FROM lightruncom/prod-base:${base_image_tag}
4-
ARG FILE
4+
ARG FILE
55

66
COPY lightrun-init-agent/$FILE /tmp/$FILE
77

@@ -12,9 +12,20 @@ RUN unzip -o /tmp/$FILE -d /agent ;\
1212
sed -i.bak "s|pinned_certs=.*|pinned_certs=|" /agent/agent.config && rm /agent/agent.config.bak && \
1313
# In openshift UID will be dynamic per project, hence procide permissions to root group (defualt in k8s)
1414
chgrp -R 0 /agent && \
15-
chmod -R g=u /agent
15+
chmod -R g=u /agent && \
16+
# Set proper permissions for the agent directory
17+
chown -R 1000:1000 /agent && \
18+
chmod -R 750 /agent && \
19+
# Create secret directory with proper permissions
20+
mkdir -p /etc/lightrun/secret && \
21+
chown -R 1000:1000 /etc/lightrun/secret && \
22+
chmod -R 700 /etc/lightrun/secret
23+
24+
# Copy and set permissions for update_config.sh before switching user
25+
COPY update_config.sh /update_config.sh
26+
RUN chmod 750 /update_config.sh && \
27+
chown 1000:1000 /update_config.sh
1628

1729
USER 1000
18-
COPY lightrun-init-agent/update_config.sh /update_config.sh
1930

2031
CMD [ "/bin/sh", "/update_config.sh" ]

0 commit comments

Comments
 (0)