[Infiniband] Fix unbinding of physical functions when configuring Infiniband virtual functions

.github/workflows/test.yml

@@ -100,6 +100,16 @@ jobs:
         # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
         version: v1.55.2
 
+  shellcheck:
+    name: Shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+        with:
+          severity: error
+
   test-coverage:
     name: test-coverage
     runs-on: ubuntu-latest

Dockerfile

@@ -2,9 +2,11 @@ FROM golang:1.22 AS builder
 WORKDIR /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator
 COPY . .
 RUN make _build-manager BIN_PATH=build/_output/cmd
+RUN make _build-sriov-network-operator-config-cleanup BIN_PATH=build/_output/cmd
 
 FROM quay.io/centos/centos:stream9
 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/manager /usr/bin/sriov-network-operator
+COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/sriov-network-operator-config-cleanup /usr/bin/sriov-network-operator-config-cleanup
 COPY bindata /bindata
 ENV OPERATOR_NAME=sriov-network-operator
 CMD ["/usr/bin/sriov-network-operator"]

Makefile

@@ -11,7 +11,7 @@ export OPERATOR_EXEC?=oc
 
 BUILD_GOPATH=$(TARGET_DIR):$(TARGET_DIR)/vendor:$(CURPATH)/cmd
 IMAGE_BUILDER?=docker
-IMAGE_BUILD_OPTS?=
+IMAGE_BUILD_OPTS?=--platform linux/amd64
 DOCKERFILE?=Dockerfile
 DOCKERFILE_CONFIG_DAEMON?=Dockerfile.sriov-network-config-daemon
 DOCKERFILE_WEBHOOK?=Dockerfile.webhook
@@ -53,14 +53,14 @@ GOLANGCI_LINT = $(BIN_DIR)/golangci-lint
 # golangci-lint version should be updated periodically
 # we keep it fixed to avoid it from unexpectedly failing on the project
 # in case of a version bump
-GOLANGCI_LINT_VER = v1.55.2
+GOLANGCI_LINT_VER = v1.61.0
 
 
 .PHONY: all build clean gendeepcopy test test-e2e test-e2e-k8s run image fmt sync-manifests test-e2e-conformance manifests update-codegen
 
 all: generate lint build
 
-build: manager _build-sriov-network-config-daemon _build-webhook
+build: manager _build-sriov-network-config-daemon _build-webhook _build-sriov-network-operator-config-cleanup
 
 _build-%:
 	WHAT=$* hack/build-go.sh
@@ -226,7 +226,7 @@ test-e2e-k8s: export NAMESPACE=sriov-network-operator
 test-e2e-k8s: test-e2e
 
 test-bindata-scripts: fakechroot
-	fakechroot ./test/scripts/enable-kargs_test.sh
+	fakechroot ./test/scripts/kargs_test.sh
 
 test-%: generate manifests envtest
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir=/tmp -p path)" HOME="$(shell pwd)" go test ./$*/... -coverprofile cover-$*.out -coverpkg ./... -v

api/v1/helper.go

@@ -26,11 +26,12 @@ import (
 )
 
 const (
-	LASTNETWORKNAMESPACE    = "operator.sriovnetwork.openshift.io/last-network-namespace"
-	NETATTDEFFINALIZERNAME  = "netattdef.finalizers.sriovnetwork.openshift.io"
-	POOLCONFIGFINALIZERNAME = "poolconfig.finalizers.sriovnetwork.openshift.io"
-	ESwithModeLegacy        = "legacy"
-	ESwithModeSwitchDev     = "switchdev"
+	LASTNETWORKNAMESPACE        = "operator.sriovnetwork.openshift.io/last-network-namespace"
+	NETATTDEFFINALIZERNAME      = "netattdef.finalizers.sriovnetwork.openshift.io"
+	POOLCONFIGFINALIZERNAME     = "poolconfig.finalizers.sriovnetwork.openshift.io"
+	OPERATORCONFIGFINALIZERNAME = "operatorconfig.finalizers.sriovnetwork.openshift.io"
+	ESwithModeLegacy            = "legacy"
+	ESwithModeSwitchDev         = "switchdev"
 
 	SriovCniStateEnable  = "enable"
 	SriovCniStateDisable = "disable"
@@ -721,7 +722,12 @@ func (cr *SriovIBNetwork) RenderNetAttDef() (*uns.Unstructured, error) {
 		data.Data["CapabilitiesConfigured"] = true
 		data.Data["SriovCniCapabilities"] = cr.Spec.Capabilities
 	}
-
+	if cr.Spec.PKey == "" {
+		data.Data["pKeyConfigured"] = false
+	} else {
+		data.Data["pKeyConfigured"] = true
+		data.Data["pKey"] = cr.Spec.PKey
+	}
 	if cr.Spec.IPAM != "" {
 		data.Data["SriovCniIpam"] = SriovCniIpam + ":" + strings.Join(strings.Fields(cr.Spec.IPAM), "")
 	} else {
@@ -764,6 +770,7 @@ func (cr *SriovNetwork) RenderNetAttDef() (*uns.Unstructured, error) {
 	data := render.MakeRenderData()
 	data.Data["CniType"] = "sriov"
 	data.Data["SriovNetworkName"] = cr.Name
+	data.Data["pKeyConfigured"] = false
 	if cr.Spec.NetworkNamespace == "" {
 		data.Data["SriovNetworkNamespace"] = cr.Namespace
 	} else {
@@ -784,7 +791,6 @@ func (cr *SriovNetwork) RenderNetAttDef() (*uns.Unstructured, error) {
 		data.Data["VlanProtoConfigured"] = true
 		data.Data["SriovCniVlanProto"] = cr.Spec.VlanProto
 	}
-
 	if cr.Spec.Capabilities == "" {
 		data.Data["CapabilitiesConfigured"] = false
 	} else {
@@ -882,6 +888,7 @@ func (cr *OVSNetwork) RenderNetAttDef() (*uns.Unstructured, error) {
 	data := render.MakeRenderData()
 	data.Data["CniType"] = "ovs"
 	data.Data["NetworkName"] = cr.Name
+	data.Data["pKeyConfigured"] = false
 	if cr.Spec.NetworkNamespace == "" {
 		data.Data["NetworkNamespace"] = cr.Namespace
 	} else {

api/v1/sriovibnetwork_types.go

@@ -43,6 +43,7 @@ type SriovIBNetworkSpec struct {
 	// MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned
 	// by the operator.
 	MetaPluginsConfig string `json:"metaPlugins,omitempty"`
+	PKey              string `json:"pKey,omitempty"`
 }
 
 // SriovIBNetworkStatus defines the observed state of SriovIBNetwork

api/v1/sriovnetworknodestate_types.go

@@ -27,6 +27,7 @@ import (
 type SriovNetworkNodeStateSpec struct {
 	Interfaces Interfaces `json:"interfaces,omitempty"`
 	Bridges    Bridges    `json:"bridges,omitempty"`
+	System     System     `json:"system,omitempty"`
 }
 
 type Interfaces []Interface
@@ -114,10 +115,17 @@ type OVSUplinkConfigExt struct {
 	Interface OVSInterfaceConfig `json:"interface,omitempty"`
 }
 
+type System struct {
+	// +kubebuilder:validation:Enum=shared;exclusive
+	//RDMA subsystem. Allowed value "shared", "exclusive".
+	RdmaMode string `json:"rdmaMode,omitempty"`
+}
+
 // SriovNetworkNodeStateStatus defines the observed state of SriovNetworkNodeState
 type SriovNetworkNodeStateStatus struct {
 	Interfaces    InterfaceExts `json:"interfaces,omitempty"`
 	Bridges       Bridges       `json:"bridges,omitempty"`
+	System        System        `json:"system,omitempty"`
 	SyncStatus    string        `json:"syncStatus,omitempty"`
 	LastSyncError string        `json:"lastSyncError,omitempty"`
 }

api/v1/sriovnetworkpoolconfig_types.go

@@ -21,6 +21,10 @@ type SriovNetworkPoolConfigSpec struct {
 	// Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards,
 	// even if maxUnavailable is greater than one.
 	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+
+	// +kubebuilder:validation:Enum=shared;exclusive
+	// RDMA subsystem. Allowed value "shared", "exclusive".
+	RdmaMode string `json:"rdmaMode,omitempty"`
 }
 
 type OvsHardwareOffloadConfig struct {

bindata/manifests/cni-config/sriov/sriov-cni-config.yaml

@@ -38,6 +38,9 @@ spec:
 {{- if .CapabilitiesConfigured -}}
   "capabilities":{{.SriovCniCapabilities}},
 {{- end -}}
+{{- if .pKeyConfigured -}}
+  "pkey":"{{.pKey}}",
+{{- end -}}
 {{- if .StateConfigured -}}
   "link_state":"{{.SriovCniState}}",
 {{- end -}}

bindata/manifests/metrics-exporter/metrics-prometheus-rule.yaml

@@ -0,0 +1,38 @@
+---
+{{ if and .IsPrometheusOperatorInstalled .PrometheusOperatorDeployRules }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: sriov-vf-rules
+  namespace: {{.Namespace}}
+spec:
+  groups:
+  - name: sriov-network-metrics-operator.rules
+    interval: 30s
+    rules:
+    - expr: |
+        sriov_vf_tx_packets * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice
+      record: network:sriov_vf_tx_packets
+    - expr: |
+        sriov_vf_rx_packets * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice
+      record: network:sriov_vf_rx_packets
+    - expr: |
+        sriov_vf_tx_bytes * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice
+      record: network:sriov_vf_tx_bytes
+    - expr: |
+        sriov_vf_rx_bytes * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice
+      record: network:sriov_vf_rx_bytes
+    - expr: |
+        sriov_vf_tx_dropped * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice
+      record: network:sriov_vf_tx_dropped
+    - expr: |
+        sriov_vf_rx_dropped * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice
+      record: network:sriov_vf_rx_dropped
+    - expr: |
+        sriov_vf_rx_broadcast * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice
+      record: network:sriov_vf_rx_broadcast
+    - expr: |
+        sriov_vf_rx_multicast * on (pciAddr,node) group_left(pod,namespace,dev_type) sriov_kubepoddevice
+      record: network:sriov_vf_rx_multicast
+{{ end }}
+

bindata/manifests/metrics-exporter/metrics-prometheus.yaml

@@ -12,6 +12,17 @@ spec:
       bearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token"
       scheme: "https"
       honorLabels: true
+      relabelings:
+      - action: replace
+        sourceLabels:
+        - __meta_kubernetes_endpoint_node_name
+        targetLabel: node
+      - action: labeldrop
+        regex: pod
+      - action: labeldrop
+        regex: container
+      - action: labeldrop
+        regex: namespace
       tlsConfig:
         serverName: sriov-network-metrics-exporter-service.{{.Namespace}}.svc
         caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt

bindata/manifests/plugins/sriov-device-plugin.yaml

@@ -27,7 +27,7 @@ spec:
       hostNetwork: true
       nodeSelector:
         {{- range $key, $value := .NodeSelectorField }}
-          {{ $key }}: {{ $value }}
+          {{ $key }}: "{{ $value }}"
         {{- end }}
       tolerations:
       - operator: Exists

bindata/manifests/webhook/002-rbac.yaml

@@ -21,7 +21,7 @@ rules:
 - apiGroups:
     - ""
   resources:
-    - configmap
+    - configmaps
   verbs:
     - 'watch'
     - 'list'

bindata/scripts/kargs.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -x
+
+command=$1
+shift
+declare -a kargs=( "$@" )
+ret=0
+args=$(chroot /host/ cat /proc/cmdline)
+
+if chroot /host/ test -f /run/ostree-booted ; then
+    for t in "${kargs[@]}";do
+        if [[ $command == "add" ]];then
+          if [[ $args != *${t}* ]];then
+              if chroot /host/ rpm-ostree kargs | grep -vq ${t}; then
+                  chroot /host/ rpm-ostree kargs --append ${t} > /dev/null 2>&1
+              fi
+              let ret++
+          fi
+        fi
+        if [[ $command == "remove" ]];then
+          if [[ $args == *${t}* ]];then
+                if chroot /host/ rpm-ostree kargs | grep -q ${t}; then
+                    chroot /host/ rpm-ostree kargs --delete ${t} > /dev/null 2>&1
+                fi
+                let ret++
+            fi
+        fi
+    done
+else
+    chroot /host/ which grubby > /dev/null 2>&1
+    # if grubby is not there, let's tell it
+    if [ $? -ne 0 ]; then
+        exit 127
+    fi
+    for t in "${kargs[@]}";do
+      if [[ $command == "add" ]];then
+        if [[ $args != *${t}* ]];then
+            if chroot /host/ grubby --info=DEFAULT | grep args | grep -vq ${t}; then
+                chroot /host/ grubby --update-kernel=DEFAULT --args=${t} > /dev/null 2>&1
+            fi
+            let ret++
+        fi
+      fi
+      if [[ $command == "remove" ]];then
+          if [[ $args == *${t}* ]];then
+            if chroot /host/ grubby --info=DEFAULT | grep args | grep -q ${t}; then
+                chroot /host/ grubby --update-kernel=DEFAULT --remove-args=${t} > /dev/null 2>&1
+            fi
+            let ret++
+          fi
+      fi
+    done
+fi
+
+echo $ret