deployment.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
 name: spark-beam-jobserver
spec:
 serviceName: spark-headless
 selector:
   matchLabels:
     app: spark-beam-jobserver
 template:
   metadata:
     labels:
       app: spark-beam-jobserver
       app.kubernetes.io/instance: custom_spark
       app.kubernetes.io/name: spark
   spec:
     containers:
     - name: spark-beam-jobserver
       image: apache/beam_spark_job_server:2.33.0
       imagePullPolicy: Always
       ports:
       - containerPort: 8099
         name: jobservice
       - containerPort: 8098
         name: artifact
       - containerPort: 8097
         name: expansion
       volumeMounts:
         - name: beam-artifact-staging
           mountPath: "/tmp/beam-artifact-staging" 
          
       command: [
           "/bin/bash", "-c", "./spark-job-server.sh --job-port=8099 --spark-master-url=spark://spark-primary:7077"
       ]
     volumes:
     - name: beam-artifact-staging
       persistentVolumeClaim:
         claimName: spark-beam-pvc
---
apiVersion: v1
kind: Service
metadata:
 name: spark-beam-jobserver
 labels:
   app: spark-beam-jobserver
spec:
 selector:
   app: spark-beam-jobserver
 type: NodePort
 ports:
 - port: 8099
   nodePort: 32090
   name: job-service
 - port: 8098
   nodePort: 32091
   name: artifacts
#  type: ClusterIP
#  ports:
#  - port: 8099
#    name: job-service
#  - port: 8098
#    name: artifacts
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: spark-primary
spec:
  serviceName: spark-headless
  replicas: 1
  selector:
    matchLabels:
      app: spark
  template:
    metadata:
      labels:
        app: spark
        component: primary
        app.kubernetes.io/instance: custom_spark
        app.kubernetes.io/name: spark
    spec:
      containers:
      - name: primary
        image: docker.io/secondcomet/spark-custom-2.4.6
        env:
        - name: SPARK_MODE
          value: "master"
        - name: SPARK_RPC_AUTHENTICATION_ENABLED
          value: "no"
        - name: SPARK_RPC_ENCRYPTION_ENABLED
          value: "no"
        - name: SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED
          value: "no"
        - name: SPARK_SSL_ENABLED
          value: "no"
        ports:
        - containerPort: 7077
          name: masterendpoint
        - containerPort: 8080
          name: ui
        - containerPort: 7078
          name: driver-rpc-port
        - containerPort: 7079
          name: blockmanager
        livenessProbe:
          httpGet:
            path: /
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10                  
        resources:
          limits:
            cpu: 1.0
            memory: 1Gi
          requests:
            cpu: 0.5
            memory: 0.5Gi
---
apiVersion: v1
kind: Service
metadata:
  name: spark-primary
  labels:
    app: spark
    component: primary
spec:
  type: ClusterIP
  ports:
  - name: masterendpoint
    port: 7077
    targetPort: 7077
  - name: rest
    port: 6066
    targetPort: 6066
  - name: ui
    port: 8080
    targetPort: 8080

  - name: driver-rpc-port
    protocol: TCP 
    port: 7078
    targetPort: 7078
  - name: blockmanager
    protocol: TCP 
    port: 7079
    targetPort: 7079

  selector:
    app: spark
    component: primary
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: spark-children
  labels:
    app: spark
spec:
  serviceName: spark-headless
  replicas: 1
  selector:
    matchLabels:
      app: spark
  template:
    metadata:
      labels:
        app: spark
        component: children
        app.kubernetes.io/instance: custom_spark
        app.kubernetes.io/name: spark
    spec:
      containers:
      - name: docker
        image: docker:19.03.5-dind
        securityContext:
          privileged: true
        volumeMounts:
          - name: dind-storage
            mountPath: /var/lib/docker
        env:
          - name: DOCKER_TLS_CERTDIR
            value: ""
        resources:
          limits:
            cpu: 1.0
            memory: 1Gi
          requests:
            cpu: 0.5
            memory: 100Mi
      - name: children
        image: docker.io/secondcomet/spark-custom-2.4.6
        env:
        - name: DOCKER_HOST
          value: "tcp://localhost:2375"
        - name: SPARK_MODE
          value: "worker"
        - name: SPARK_MASTER_URL
          value: "spark://spark-primary:7077"
        - name: SPARK_WORKER_MEMORY
          value: "1G"
        - name: SPARK_WORKER_CORES
          value: "1"
        - name: SPARK_RPC_AUTHENTICATION_ENABLED
          value: "no"
        - name: SPARK_RPC_ENCRYPTION_ENABLED
          value: "no"
        - name: SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED
          value: "no"
        - name: SPARK_SSL_ENABLED
          value: "no"
        ports:
          - containerPort: 8081
            name: ui
        volumeMounts:
          - name: beam-artifact-staging
            mountPath: "/tmp/beam-artifact-staging"
        resources:
          limits:
            cpu: 1
            memory: 2Gi
          requests:
            cpu: 0.5
            memory: 1Gi
      volumes:
      - name: dind-storage
        emptyDir:
      - name: beam-artifact-staging
        persistentVolumeClaim:
          claimName: spark-beam-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: spark-children
  labels:
    app: spark
    component: children
spec:
  type: ClusterIP
  ports:
  - name: ui
    port: 8081
    targetPort: 8081
  selector:
    app: spark
    component: children
---
apiVersion: v1
kind: Service
metadata:
  name: spark-headless
spec:
  clusterIP: None
  selector:
    app.kubernetes.io/instance: custom_spark
    app.kubernetes.io/name: spark
  type: ClusterIP