Deploy Istio to the utility cluster

kubernetes/apps/istio.yaml

@@ -0,0 +1,98 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: istio-base
+  annotations:
+    argocd.argoproj.io/sync-wave: "-3"
+spec:
+  destination:
+    name: gke-utility
+    namespace: istio-system
+  project: default
+  sources:
+    - chart: base
+      repoURL: https://istio-release.storage.googleapis.com/charts
+      targetRevision: 1.23.2
+    - path: config/crd/experimental
+      repoURL: https://github.com/kubernetes-sigs/gateway-api
+      targetRevision: v1.1.0
+  ignoreDifferences:
+    - group: admissionregistration.k8s.io
+      kind: ValidatingWebhookConfiguration
+      jsonPointers:
+      - /webhooks/0/failurePolicy
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: istio
+spec:
+  destination:
+    name: gke-utility
+    namespace: istio-system
+  project: default
+  sources:
+    - chart: istiod
+      repoURL: https://istio-release.storage.googleapis.com/charts
+      targetRevision: 1.23.2
+      helm:
+        values: |
+          global:
+            hub: gcr.io/istio-release
+            istiod:
+              enableAnalysis: true
+          meshConfig:
+            accessLogFile: /dev/stdout
+            enablePrometheusMerge: true
+            protocolDetectionTimeout: 5s
+            enableTracing: true
+            defaultConfig:
+              tracing:
+            extensionProviders:
+            - name: stackdriver
+              stackdriver:
+            - name: "oauth2-proxy"
+              envoyExtAuthzHttp:
+                service: "oauth2-proxy.oauth2-proxy.svc.cluster.local"
+                port: "80"
+                headersToDownstreamOnDeny:
+                  - content-type
+                  - set-cookie
+                headersToUpstreamOnAllow:
+                  - authorization
+                  - cookie
+                  - path
+                  - x-*
+                includeHeadersInCheck:
+                  - authorization
+                  - cookie
+                  - user-agent
+                includeAdditionalHeadersInCheck:
+                  X-Auth-Request-Redirect: https://%REQ(Host)%%REQ(:PATH)%
+            defaultConfig:
+              gatewayTopology:
+                numTrustedProxies: 2
+          telemetry:
+            enabled: true
+            v2:
+              prometheus:
+                enabled: true
+                wasmEnabled: false
+    - path: kubernetes/gke-utility/istio-system
+      repoURL: https://github.com/kubernetes/k8s.io
+      targetRevision: main
+  ignoreDifferences:
+    - group: admissionregistration.k8s.io
+      kind: ValidatingWebhookConfiguration
+      jsonPointers:
+      - /webhooks/0/failurePolicy
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

kubernetes/apps/kustomization.yaml

@@ -6,3 +6,5 @@ resources:
   - cert-manager.yaml
   # - ingress-nginx.yaml
   - prow.yaml
+  - istio.yaml
+  - oauth2-proxy.yaml

kubernetes/apps/oauth2-proxy.yaml

@@ -0,0 +1,39 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ApplicationSet
+metadata:
+  name: oauth2-proxy
+spec:
+  goTemplate: true
+  generators:
+    - clusters:
+        selector:
+          matchLabels:
+            clusterType: 'utility'
+  template:
+    metadata:
+      name: 'oauth2-proxy-{{ .name }}'
+    spec:
+      destination:
+        namespace: oauth2-proxy
+        server: "{{ .server }}"
+      project: default
+      sources:
+        - chart: oauth2-proxy
+          repoURL: 'https://oauth2-proxy.github.io/manifests'
+          targetRevision: 7.7.19
+          helm:
+            releaseName: oauth2-proxy
+            valueFiles:
+            - $values/kubernetes/{{ .name }}/helm/oauth2-proxy.yaml
+        - repoURL: 'https://github.com/kubernetes/k8s.io.git'
+          targetRevision: main
+          ref: values
+      syncPolicy:
+        automated:
+          prune: true
+          selfHeal: true
+        syncOptions:
+          - CreateNamespace=true
+        managedNamespaceMetadata:
+          labels:
+            istio-injection: enabled

kubernetes/gke-utility/argocd/argocd-cm-rbac.yaml

@@ -4,6 +4,3 @@ metadata:
   name: argocd-rbac-cm
 data:
   policy.default: role:readonly
-  policy.csv: |
-    g, kubernetes:sig-k8s-infra-leads, role:admin
-  scopes: '[groups, email]'

kubernetes/gke-utility/argocd/argocd-cm.yaml

@@ -12,17 +12,11 @@ data:
       ignoreDifferences: |
         jqPathExpressions:
         - '.webhooks[]?.clientConfig.caBundle'
+  resource.exclusions: |
+    - apiGroups:
+        - cilium.io
+      kinds:
+        - CiliumIdentity
+      clusters:
+        - "*"
   kustomize.buildOptions: --load-restrictor LoadRestrictionsNone --enable-alpha-plugins
-  dex.config: |
-    connectors:
-      - type: github
-        id: github
-        name: GitHub
-        config:
-          clientID: $dex.github.clientId
-          clientSecret: $dex.github.clientSecret
-          orgs:
-          - name: kubernetes
-          useLoginAsID: true
-          loadAllGroups: true
-          teamNameField: slug

kubernetes/gke-utility/argocd/argocd-cmd-params-cm.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argocd-cmd-params-cm
+data:
+  server.insecure: "true" # Safe as we are using Istio Mesh

kubernetes/gke-utility/argocd/extras.yaml

@@ -6,7 +6,7 @@ metadata:
   labels:
     argocd.argoproj.io/secret-type: repository
 stringData:
-  url: https://github.com/kubernetes
+  url: https://github.com/kubernetes/k8s.io
   name: kubernetes
   type: git
 ---
@@ -27,3 +27,41 @@ spec:
     automated:
       prune: false
       selfHeal: true
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: argocd
+spec:
+  hostnames:
+    - argo.k8s.io
+  parentRefs:
+    - name: istio-ingressgateway
+      namespace: istio-system
+      sectionName: https
+  rules:
+    - backendRefs:
+        - name: argocd-server
+          port: 80
+---
+apiVersion: security.istio.io/v1
+kind: AuthorizationPolicy
+metadata:
+  name: argocd
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: argocd-server
+  action: ALLOW
+  rules:
+    - from:
+        - source:
+            namespaces:
+              - istio-system
+      when:
+        - key: request.headers[X-Auth-Request-User]
+          values:
+            - dims
+            - upodroid
+            - ameukam
+            - BenTheElder

kubernetes/gke-utility/argocd/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
 - clusters.yaml
 
 patches:
+- path: argocd-cmd-params-cm.yaml
 - path: argocd-cm.yaml
 - path: argocd-cm-rbac.yaml
 - path: argocd-sa.yaml

kubernetes/gke-utility/helm/cert-manager.yaml

@@ -1,6 +1,8 @@
-installCRDs: true
+crds:
+  enabled: true
 extraObjects:
-  - apiVersion: cert-manager.io/v1
+  - |
+    apiVersion: cert-manager.io/v1
     kind: ClusterIssuer
     metadata:
       name: letsencrypt-prod
@@ -11,6 +13,5 @@ extraObjects:
         privateKeySecretRef:
           name: letsencrypt-prod
         solvers:
-          - http01:
-              ingress:
-                ingressClassName: nginx
+          - cloudDNS:
+              project: kubernetes-public

kubernetes/gke-utility/helm/oauth2-proxy.yaml

@@ -0,0 +1,54 @@
+config:
+  existingSecret: oauth2-proxy-creds
+
+extraArgs:
+  provider: github
+  github-org: kubernetes
+  redirect-url: https://oauth2-proxy.k8s.io/oauth2/callback
+  reverse-proxy: true
+  pass-access-token: true
+  pass-user-headers: true
+  pass-authorization-header: true
+  cookie-samesite: lax
+  cookie-domain: .k8s.io
+  set-xauthrequest: true
+  whitelist-domain: "*.k8s.io"
+  skip-provider-button: true
+  skip-jwt-bearer-tokens: true
+  upstream: static://200
+  silence-ping-logging: true
+  show-debug-on-error: true
+
+metrics:
+  serviceMonitor:
+    enabled: false #enable when observability stack is ready
+
+extraObjects:
+  - apiVersion: external-secrets.io/v1beta1
+    kind: ExternalSecret
+    metadata:
+      name: oauth2-proxy-creds
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      dataFrom:
+      - extract:
+          key: oauth2-proxy-creds
+      secretStoreRef:
+        kind: ClusterSecretStore
+        name: k8s-infra-prow
+  - apiVersion: gateway.networking.k8s.io/v1
+    kind: HTTPRoute
+    metadata:
+      name: oauth2-proxy
+      namespace: "{{ .Release.Namespace }}"
+    spec:
+      hostnames:
+        - oauth2-proxy.k8s.io
+      parentRefs:
+        - name: istio-ingressgateway
+          namespace: istio-system
+          sectionName: https
+      rules:
+        - backendRefs:
+            - name: oauth2-proxy
+              port: 80

kubernetes/gke-utility/istio-system/auth-policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: security.istio.io/v1
+kind: AuthorizationPolicy
+metadata:
+  name: oauth-policy
+spec:
+  targetRefs:
+    - name: istio-ingressgateway
+      kind: Gateway
+      group: gateway.networking.k8s.io
+  action: CUSTOM
+  provider:
+    name: oauth2-proxy
+  rules:
+  - to:
+    - operation:
+        hosts:
+        - argo.k8s.io

kubernetes/gke-utility/istio-system/gateway.yaml

@@ -0,0 +1,53 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: istio-ingressgateway
+  annotations:
+    cloud.google.com/l4-rbs: enabled
+    networking.gke.io/load-balancer-ip-addresses: utility-ingress-v4,utility-ingress-v6
+spec:
+  gatewayClassName: istio
+  listeners:
+  - name: http
+    port: 80
+    protocol: HTTP
+  - name: https
+    port: 443
+    protocol: HTTPS
+    allowedRoutes:
+      namespaces:
+        from: All
+    tls:
+      mode: Terminate
+      certificateRefs:
+      - name: k8s-io-wild-cert
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: https-gateway-redirect
+spec:
+  parentRefs:
+  - name: istio-ingressgateway
+    sectionName: http
+  hostnames:
+  - '*.k8s.io'
+  rules:
+  - filters:
+    - type: RequestRedirect
+      requestRedirect:
+        scheme: https
+        statusCode: 302
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: k8s-io-wild
+spec:
+  secretName: k8s-io-wild-cert
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  commonName: '*.k8s.io'
+  dnsNames:
+  - '*.k8s.io'

kubernetes/gke-utility/istio-system/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: istio-system
+resources:
+  - auth-policy.yaml
+  - gateway.yaml