MGMT-20032: handle duplicates in CA bundle (#7409)

In some scenarios in which the `user-ca-bundle`
includes superfluous certificates, the `assisted-trusted-ca-bundle`
ConfigMap could include duplicate certificates.
To mitigate this issue, added logic for removing such duplicates
when creating the CM.

Author: danielerez

internal/common/common.go

@@ -418,6 +418,42 @@ func VerifyCaBundle(pemCerts []byte) error {
 	return nil
 }
 
+// RemoveDuplicatesFromCaBundle removes duplicate certificates from a given CA bundle.
+func RemoveDuplicatesFromCaBundle(caBundle string) (string, int, error) {
+	// Parse certificates
+	certs, ok := ParsePemCerts([]byte(caBundle))
+	if !ok {
+		return "", 0, errors.New("failed to remove duplicate certificate")
+	}
+
+	// Remove duplicates by serial number
+	uniqueCerts := funk.UniqBy(certs, func(cert x509.Certificate) string {
+		return fmt.Sprintf("%x", cert.SerialNumber)
+	})
+
+	// Convert certs back to a string
+	certStrings := funk.Map(uniqueCerts, func(cert x509.Certificate) string {
+		// Encode certificate to PEM format
+		block := &pem.Block{
+			Type:  "CERTIFICATE",
+			Bytes: cert.Raw,
+		}
+		var sb strings.Builder
+		if err := pem.Encode(&sb, block); err != nil {
+			// Error encoding certificate
+			return ""
+		}
+		return sb.String()
+	})
+
+	numOfCerts := len(certs)
+	numOfUniqueCerts := len(uniqueCerts.([]x509.Certificate))
+	numOfDuplicates := numOfCerts - numOfUniqueCerts
+
+	// Join the PEM-encoded certificates into a single string
+	return strings.Join(certStrings.([]string), "\n"), numOfDuplicates, nil
+}
+
 func CanonizeStrings(slice []string) (ret []string) {
 	if len(slice) == 0 {
 		return

internal/common/common_test.go

@@ -135,6 +135,58 @@ fQEw+cWRxwFPJujSOTSKRHZDo1UwOIQbxqkbznSHlLCICEXxuvQ=
 -----END CERTIFICATE-----
 `
 
+const singleCAcert1 string = `-----BEGIN CERTIFICATE-----
+MIIENDCCAxygAwIBAgIJANunI0D662cnMA0GCSqGSIb3DQEBCwUAMIGlMQswCQYD
+VQQGEwJVUzEXMBUGA1UECAwOTm9ydGggQ2Fyb2xpbmExEDAOBgNVBAcMB1JhbGVp
+Z2gxFjAUBgNVBAoMDVJlZCBIYXQsIEluYy4xEzARBgNVBAsMClJlZCBIYXQgSVQx
+GzAZBgNVBAMMElJlZCBIYXQgSVQgUm9vdCBDQTEhMB8GCSqGSIb3DQEJARYSaW5m
+b3NlY0ByZWRoYXQuY29tMCAXDTE1MDcwNjE3MzgxMVoYDzIwNTUwNjI2MTczODEx
+WjCBpTELMAkGA1UEBhMCVVMxFzAVBgNVBAgMDk5vcnRoIENhcm9saW5hMRAwDgYD
+VQQHDAdSYWxlaWdoMRYwFAYDVQQKDA1SZWQgSGF0LCBJbmMuMRMwEQYDVQQLDApS
+ZWQgSGF0IElUMRswGQYDVQQDDBJSZWQgSGF0IElUIFJvb3QgQ0ExITAfBgkqhkiG
+9w0BCQEWEmluZm9zZWNAcmVkaGF0LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBALQt9OJQh6GC5LT1g80qNh0u50BQ4sZ/yZ8aETxt+5lnPVX6MHKz
+bfwI6nO1aMG6j9bSw+6UUyPBHP796+FT/pTS+K0wsDV7c9XvHoxJBJJU38cdLkI2
+c/i7lDqTfTcfLL2nyUBd2fQDk1B0fxrskhGIIZ3ifP1Ps4ltTkv8hRSob3VtNqSo
+GxkKfvD2PKjTPxDPWYyruy9irLZioMffi3i/gCut0ZWtAyO3MVH5qWF/enKwgPES
+X9po+TdCvRB/RUObBaM761EcrLSM1GqHNueSfqnho3AjLQ6dBnPWlo638Zm1VebK
+BELyhkLWMSFkKwDmne0jQ02Y4g075vCKvCsCAwEAAaNjMGEwHQYDVR0OBBYEFH7R
+4yC+UehIIPeuL8Zqw3PzbgcZMB8GA1UdIwQYMBaAFH7R4yC+UehIIPeuL8Zqw3Pz
+bgcZMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB
+CwUAA4IBAQBDNvD2Vm9sA5A9AlOJR8+en5Xz9hXcxJB5phxcZQ8jFoG04Vshvd0e
+LEnUrMcfFgIZ4njMKTQCM4ZFUPAieyLx4f52HuDopp3e5JyIMfW+KFcNIpKwCsak
+oSoKtIUOsUJK7qBVZxcrIyeQV2qcYOeZhtS5wBqIwOAhFwlCET7Ze58QHmS48slj
+S9K0JAcps2xdnGu0fkzhSQxY8GPQNFTlr6rYld5+ID/hHeS76gq0YG3q6RLWRkHf
+4eTkRjivAlExrFzKcljC4axKQlnOvVAzz+Gm32U0xPBF4ByePVxCJUHw1TsyTmel
+RxNEp7yHoXcwn+fXna+t5JWh1gxUZty3
+-----END CERTIFICATE-----
+`
+
+const singleCAcert2 string = `-----BEGIN CERTIFICATE-----
+MIID6DCCAtCgAwIBAgIBFDANBgkqhkiG9w0BAQsFADCBpTELMAkGA1UEBhMCVVMx
+FzAVBgNVBAgMDk5vcnRoIENhcm9saW5hMRAwDgYDVQQHDAdSYWxlaWdoMRYwFAYD
+VQQKDA1SZWQgSGF0LCBJbmMuMRMwEQYDVQQLDApSZWQgSGF0IElUMRswGQYDVQQD
+DBJSZWQgSGF0IElUIFJvb3QgQ0ExITAfBgkqhkiG9w0BCQEWEmluZm9zZWNAcmVk
+aGF0LmNvbTAeFw0xNTEwMTQxNzI5MDdaFw00NTEwMDYxNzI5MDdaME4xEDAOBgNV
+BAoMB1JlZCBIYXQxDTALBgNVBAsMBHByb2QxKzApBgNVBAMMIkludGVybWVkaWF0
+ZSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
+ggEKAoIBAQDYpVfg+jjQ3546GHF6sxwMOjIwpOmgAXiHS4pgaCmu+AQwBs4rwxvF
+S+SsDHDTVDvpxJYBwJ6h8S3LK9xk70yGsOAu30EqITj6T+ZPbJG6C/0I5ukEVIeA
+xkgPeCBYiiPwoNc/te6Ry2wlaeH9iTVX8fx32xroSkl65P59/dMttrQtSuQX8jLS
+5rBSjBfILSsaUywND319E/Gkqvh6lo3TEax9rhqbNh2s+26AfBJoukZstg3TWlI/
+pi8v/D3ZFDDEIOXrP0JEfe8ETmm87T1CPdPIZ9+/c4ADPHjdmeBAJddmT0IsH9e6
+Gea2R/fQaSrIQPVmm/0QX2wlY4JfxyLJAgMBAAGjeTB3MB0GA1UdDgQWBBQw3gRU
+oYYCnxH6UPkFcKcowMBP/DAfBgNVHSMEGDAWgBR+0eMgvlHoSCD3ri/GasNz824H
+GTASBgNVHRMBAf8ECDAGAQH/AgEBMA4GA1UdDwEB/wQEAwIBhjARBglghkgBhvhC
+AQEEBAMCAQYwDQYJKoZIhvcNAQELBQADggEBADwaXLIOqoyQoBVck8/52AjWw1Cv
+ath9NGUEFROYm15VbAaFmeY2oQ0EV3tQRm32C9qe9RxVU8DBDjBuNyYhLg3k6/1Z
+JXggtSMtffr5T83bxgfh+vNxF7o5oNxEgRUYTBi4aV7v9LiDd1b7YAsUwj4NPWYZ
+dbuypFSWCoV7ReNt+37muMEZwi+yGIU9ug8hLOrvriEdU3RXt5XNISMMuC8JULdE
+3GVzoNtkznqv5ySEj4M9WsdBiG6bm4aBYIOE0XKE6QYtlsjTMB9UTXxmlUvDE0wC
+z9YYKfC1vLxL2wAgMhOCdKZM+Qlu1stb0B/EF3oxc/iZrhDvJLjijbMpphw=
+-----END CERTIFICATE-----
+`
+
 const inventoryWithSingleNIC string = "{\"bmc_address\":\"0.0.0.0\",\"bmc_v6address\":\"::/0\",\"boot\":{\"current_boot_mode\":\"bios\"},\"cpu\":{\"architecture\":\"x86_64\",\"count\":16,\"flags\":[\"fpu\",\"vme\",\"de\",\"pse\",\"tsc\",\"msr\",\"pae\",\"mce\",\"cx8\",\"apic\",\"sep\",\"mtrr\",\"pge\",\"mca\",\"cmov\",\"pat\",\"pse36\",\"clflush\",\"mmx\",\"fxsr\",\"sse\",\"sse2\",\"ss\",\"syscall\",\"nx\",\"pdpe1gb\",\"rdtscp\",\"lm\",\"constant_tsc\",\"arch_perfmon\",\"nopl\",\"xtopology\",\"tsc_reliable\",\"nonstop_tsc\",\"cpuid\",\"pni\",\"pclmulqdq\",\"ssse3\",\"fma\",\"cx16\",\"pcid\",\"sse4_1\",\"sse4_2\",\"x2apic\",\"movbe\",\"popcnt\",\"tsc_deadline_timer\",\"aes\",\"xsave\",\"avx\",\"f16c\",\"rdrand\",\"hypervisor\",\"lahf_lm\",\"abm\",\"3dnowprefetch\",\"cpuid_fault\",\"invpcid_single\",\"pti\",\"ssbd\",\"ibrs\",\"ibpb\",\"stibp\",\"fsgsbase\",\"tsc_adjust\",\"bmi1\",\"avx2\",\"smep\",\"bmi2\",\"invpcid\",\"rdseed\",\"adx\",\"smap\",\"xsaveopt\",\"arat\",\"md_clear\",\"flush_l1d\",\"arch_capabilities\"],\"frequency\":2194.917,\"model_name\":\"Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz\"},\"disks\":[{\"by_id\":\"/dev/disk/by-id/wwn-0x6000c2911a3fb8af754385340083d09c\",\"by_path\":\"/dev/disk/by-path/pci-0000:03:00.0-scsi-0:0:0:0\",\"drive_type\":\"HDD\",\"has_uuid\":true,\"hctl\":\"0:0:0:0\",\"id\":\"/dev/disk/by-id/wwn-0x6000c2911a3fb8af754385340083d09c\",\"installation_eligibility\":{\"eligible\":true,\"not_eligible_reasons\":null},\"model\":\"Virtual_disk\",\"name\":\"sda\",\"path\":\"/dev/sda\",\"serial\":\"6000c2911a3fb8af754385340083d09c\",\"size_bytes\":128849018880,\"smart\":\"SMART support is:     Unavailable - device lacks SMART capability.\\n\",\"vendor\":\"VMware\",\"wwn\":\"0x6000c2911a3fb8af754385340083d09c\"},{\"by_path\":\"/dev/disk/by-path/pci-0000:00:07.1-ata-1\",\"drive_type\":\"ODD\",\"hctl\":\"1:0:0:0\",\"id\":\"/dev/disk/by-path/pci-0000:00:07.1-ata-1\",\"installation_eligibility\":{\"not_eligible_reasons\":[\"Disk is removable\",\"Disk is too small (disk only has 106 MB, but 100 GB are required)\",\"Drive type is ODD, it must be one of HDD, SSD, Multipath.\"]},\"is_installation_media\":true,\"model\":\"VMware_IDE_CDR00\",\"name\":\"sr0\",\"path\":\"/dev/sr0\",\"removable\":true,\"serial\":\"00000000000000000001\",\"size_bytes\":106516480,\"smart\":\"SMART support is:     Unavailable - device lacks SMART capability.\\n\",\"vendor\":\"NECVMWar\"}],\"gpus\":[{\"address\":\"0000:00:0f.0\"}],\"hostname\":\"master-2.qe1.e2e.bos.redhat.com\",\"interfaces\":[{\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"has_carrier\":true,\"ipv4_addresses\":[\"10.19.114.222/23\"],\"ipv6_addresses\":[\"2620:52:0:1372:b55f:731b:f1dc:d773/64\"],\"mac_address\":\"00:50:56:83:87:09\",\"mtu\":1500,\"name\":\"ens192\",\"product\":\"0x07b0\",\"speed_mbps\":10000,\"type\":\"physical\",\"vendor\":\"0x15ad\"}],\"memory\":{\"physical_bytes\":34359738368,\"physical_bytes_method\":\"dmidecode\",\"usable_bytes\":33711775744},\"routes\":[{\"destination\":\"0.0.0.0\",\"family\":2,\"gateway\":\"10.19.115.254\",\"interface\":\"ens192\"},{\"destination\":\"10.19.114.0\",\"family\":2,\"interface\":\"ens192\"},{\"destination\":\"10.88.0.0\",\"family\":2,\"interface\":\"cni-podman0\"},{\"destination\":\"::1\",\"family\":10,\"interface\":\"lo\"},{\"destination\":\"2620:52:0:1372::\",\"family\":10,\"interface\":\"ens192\"},{\"destination\":\"fe80::\",\"family\":10,\"interface\":\"ens192\"},{\"destination\":\"fe80::\",\"family\":10,\"interface\":\"cni-podman0\"},{\"destination\":\"::\",\"family\":10,\"gateway\":\"fe80::a81:f4ff:fea6:dc01\",\"interface\":\"ens192\"}],\"system_vendor\":{\"manufacturer\":\"VMware, Inc.\",\"product_name\":\"VMware Virtual Platform\",\"serial_number\":\"VMware-42 09 5f ea c8 4e 8c 88-f3 0c 06 65 5a 4d 32 fb\",\"virtual\":true},\"tpm_version\":\"none\"}"
 const inventoryWithMultipleNICs string = "\"hostname\":\"localhost\",\"interfaces\":[{\"biosdevname\":\"em2\",\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"ipv4_addresses\":[],\"ipv6_addresses\":[],\"mac_address\":\"b4:7a:f1:da:fe:85\",\"mtu\":1500,\"name\":\"eno2\",\"product\":\"0x37ce\",\"speed_mbps\":-1,\"vendor\":\"0x8086\"},{\"biosdevname\":\"em1\",\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"has_carrier\":true,\"ipv4_addresses\":[],\"ipv6_addresses\":[],\"mac_address\":\"b4:7a:f1:da:fe:84\",\"mtu\":1500,\"name\":\"eno1\",\"product\":\"0x1537\",\"speed_mbps\":1000,\"vendor\":\"0x8086\"},{\"biosdevname\":\"em3\",\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"ipv4_addresses\":[],\"ipv6_addresses\":[],\"mac_address\":\"b4:7a:f1:da:fe:86\",\"mtu\":1500,\"name\":\"eno3\",\"product\":\"0x37ce\",\"speed_mbps\":-1,\"vendor\":\"0x8086\"},{\"biosdevname\":\"em4\",\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"ipv4_addresses\":[],\"ipv6_addresses\":[],\"mac_address\":\"b4:7a:f1:da:fe:87\",\"mtu\":1500,\"name\":\"eno4\",\"product\":\"0x37ce\",\"speed_mbps\":-1,\"vendor\":\"0x8086\"},{\"biosdevname\":\"em5\",\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"ipv4_addresses\":[],\"ipv6_addresses\":[],\"mac_address\":\"b4:7a:f1:da:fe:88\",\"mtu\":1500,\"name\":\"eno5\",\"product\":\"0x37ce\",\"speed_mbps\":-1,\"vendor\":\"0x8086\"},{\"biosdevname\":\"p1p1\",\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"has_carrier\":true,\"ipv4_addresses\":[],\"ipv6_addresses\":[],\"mac_address\":\"d4:f5:ef:56:35:64\",\"mtu\":8000,\"name\":\"ens1f0\",\"product\":\"0x158b\",\"speed_mbps\":25000,\"vendor\":\"0x8086\"},{\"biosdevname\":\"p1p2\",\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"has_carrier\":true,\"ipv4_addresses\":[],\"ipv6_addresses\":[],\"mac_address\":\"d4:f5:ef:56:35:64\",\"mtu\":8000,\"name\":\"ens1f1\",\"product\":\"0x158b\",\"speed_mbps\":25000,\"vendor\":\"0x8086\"},{\"flags\":[\"up\",\"broadcast\",\"multicast\"],\"has_carrier\":true,\"ipv4_addresses\":[\"10.195.70.120/24\"],\"ipv6_addresses\":[],\"mac_address\":\"d4:f5:ef:56:35:64\",\"mtu\":1500,\"name\":\"bond0\",\"speed_mbps\":25000}],\"memory\":{\"physical_bytes\":412316860416,\"physical_bytes_method\":\"dmidecode\",\"usable_bytes\":405391712256},\"routes\":[{\"destination\":\"0.0.0.0\",\"family\":2,\"gateway\":\"10.195.70.1\",\"interface\":\"bond0\"},{\"destination\":\"10.88.0.0\",\"family\":2,\"interface\":\"cni-podman0\"},{\"destination\":\"10.195.70.0\",\"family\":2,\"interface\":\"bond0\"},{\"destination\":\"::1\",\"family\":10,\"interface\":\"lo\"},{\"destination\":\"fe80::\",\"family\":10,\"interface\":\"cni-podman0\"}],\"system_vendor\":{\"manufacturer\":\"HPE\",\"product_name\":\"ProLiant e910\",\"serial_number\":\"MXQ1291HP3\"},\"timestamp\":1657725466,\"tpm_version\":\"2.0\"}"
 
@@ -501,6 +553,45 @@ var _ = Describe("ShouldMastersBeSchedulable", func() {
 	})
 })
 
+var _ = Describe("RemoveDuplicatesFromCaBundle", func() {
+	It("should remove duplicate certificates", func() {
+		caBundle := singleCAcert1 + singleCAcert1
+		result, numOfDuplicates, err := RemoveDuplicatesFromCaBundle(caBundle)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(result).To(Equal(singleCAcert1))
+		Expect(numOfDuplicates).To(Equal(1))
+
+		err = VerifyCaBundle([]byte(result))
+		Expect(err).ShouldNot(HaveOccurred())
+	})
+	It("should handle a bundle without duplicates", func() {
+		caBundle := singleCAcert1 + "\n" + singleCAcert2
+		result, numOfDuplicates, err := RemoveDuplicatesFromCaBundle(caBundle)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(result).To(Equal(singleCAcert1 + "\n" + singleCAcert2))
+		Expect(numOfDuplicates).To(Equal(0))
+
+		err = VerifyCaBundle([]byte(result))
+		Expect(err).ShouldNot(HaveOccurred())
+	})
+	It("should handle a single cert", func() {
+		caBundle := singleCAcert1
+		result, numOfDuplicates, err := RemoveDuplicatesFromCaBundle(caBundle)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(result).To(Equal(singleCAcert1))
+		Expect(numOfDuplicates).To(Equal(0))
+
+		err = VerifyCaBundle([]byte(result))
+		Expect(err).ShouldNot(HaveOccurred())
+	})
+	It("should return an error for malformed certificates", func() {
+		caBundle := malformedPEM
+		result, _, err := RemoveDuplicatesFromCaBundle(caBundle)
+		Expect(err).To(HaveOccurred())
+		Expect(result).To(Equal(""))
+	})
+})
+
 func createHost(hostRole models.HostRole, state string) *models.Host {
 	hostId := strfmt.UUID(uuid.New().String())
 	clusterId := strfmt.UUID(uuid.New().String())

internal/controller/controllers/agentserviceconfig_controller.go

@@ -1092,6 +1092,15 @@ func newAssistedTrustedCACM(ctx context.Context, log logrus.FieldLogger, asc ASC
 		}
 	}
 
+	// Ensure no duplicate certificates in the CA bundle
+	caBundle, numOfDuplicates, err := common.RemoveDuplicatesFromCaBundle(b.String())
+	if err != nil {
+		return nil, nil, err
+	}
+	if numOfDuplicates > 0 {
+		log.Infof("Removed %d duplicate certificates from CA bundle", numOfDuplicates)
+	}
+
 	cm := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      assistedCAConfigMapName,
@@ -1104,7 +1113,7 @@ func newAssistedTrustedCACM(ctx context.Context, log logrus.FieldLogger, asc ASC
 			return err
 		}
 		cm.Data = map[string]string{
-			caBundleKey: b.String(),
+			caBundleKey: caBundle,
 		}
 		return nil
 	}