Merge pull request #1745 from bart0sh/PR155-fpga-support-CDI

FPGA: support CDI

Author: tkatila

README.md

@@ -66,7 +66,7 @@ The FPGA plugin comes as three parts.
 
 - the [device plugin](#device-plugin)
 - the [admission controller](#admission-controller)
-- the [CRIO-O prestart hook](#cri-o-prestart-hook)
+- the [OCI createRuntime hook](#OCI-createRuntime-hook)
 
 Refer to each individual sub-components documentation for more details.
 Brief overviews of the sub-components are below.
@@ -88,9 +88,9 @@ is responsible for performing mapping from user-friendly function IDs to the
 Interface ID and Bitstream ID that are required for FPGA programming. It also
 implements access control by namespacing FPGA configuration information.
 
-#### CRI-O Prestart Hook
+#### OCI createRuntime Hook
 
-The [FPGA prestart CRI-O hook](cmd/fpga_crihook/README.md) performs discovery
+The [FPGA OCI createRuntime hook](cmd/fpga_crihook/README.md) performs discovery
 of the requested FPGA function bitstream and programs FPGA devices based on the
 environment variables in the workload description.
 

build/docker/intel-fpga-initcontainer.Dockerfile

@@ -66,13 +66,6 @@ RUN install -D ${DIR}/LICENSE /install_root/licenses/intel-device-plugins-for-ku
     --save_path /install_root/licenses/$CMD/go-licenses ; \
     else mkdir -p /install_root/licenses/$CMD/go-licenses/ && cd licenses/$CMD && cp -r * /install_root/licenses/$CMD/go-licenses/ ; fi
 ###
-ARG SRC_DIR=/usr/local/fpga-sw
-ARG DST_DIR=/opt/intel/fpga-sw
-RUN echo "{\n\
-    \"hook\" : \"$DST_DIR/$CRI_HOOK\",\n\
-    \"stage\" : [ \"prestart\" ],\n\
-    \"annotation\": [ \"fpga.intel.com/region\" ]\n\
-}\n">>/install_root/$SRC_DIR/$CRI_HOOK.json
 ARG TOYBOX_VERSION="0.8.11"
 ARG TOYBOX_SHA256="83a3a88cbe1fa30f099c2f58295baef4637aaf988085aaea56e03aa29168175d"
 ARG ROOT=/install_root
@@ -93,7 +86,7 @@ LABEL vendor='Intel®'
 LABEL version='devel'
 LABEL release='1'
 LABEL name='intel-fpga-initcontainer'
-LABEL summary='Intel® FPGA programming CRI hook for Kubernetes'
-LABEL description='The FPGA prestart CRI-O hook performs discovery of the requested FPGA function bitstream and programs FPGA devices based on the environment variables in the workload description'
+LABEL summary='Intel® FPGA programming CDI hook for Kubernetes'
+LABEL description='The FPGA OCI createRuntime hook performs discovery of the requested FPGA function bitstream and programs FPGA devices based on the environment variables in the workload description'
 COPY --from=builder /install_root /
-ENTRYPOINT [ "/usr/bin/sh", "-c", "cp -a /usr/local/fpga-sw/* /opt/intel/fpga-sw/ && ln -sf /opt/intel/fpga-sw/intel-fpga-crihook.json /etc/containers/oci/hooks.d/" ]
+ENTRYPOINT [ "/usr/bin/sh", "-c", "cp -a /usr/local/fpga-sw/* /opt/intel/fpga-sw/" ]

build/docker/templates/intel-fpga-initcontainer.Dockerfile.in

@@ -14,25 +14,16 @@ ARG CMD=fpga_tool
 ARG EP=/usr/local/fpga-sw/$CMD
 #include "default_build.docker"
 
-ARG SRC_DIR=/usr/local/fpga-sw
-ARG DST_DIR=/opt/intel/fpga-sw
-
-RUN echo "{\n\N
-    \"hook\" : \"$DST_DIR/$CRI_HOOK\",\n\N
-    \"stage\" : [ \"prestart\" ],\n\N
-    \"annotation\": [ \"fpga.intel.com/region\" ]\n\N
-}\n">>/install_root/$SRC_DIR/$CRI_HOOK.json
-
 #include "toybox_build.docker"
 
 FROM ${FINAL_BASE}
 
 #include "default_labels.docker"
 
 LABEL name='intel-fpga-initcontainer'
-LABEL summary='Intel® FPGA programming CRI hook for Kubernetes'
-LABEL description='The FPGA prestart CRI-O hook performs discovery of the requested FPGA function bitstream and programs FPGA devices based on the environment variables in the workload description'
+LABEL summary='Intel® FPGA programming CDI hook for Kubernetes'
+LABEL description='The FPGA OCI createRuntime hook performs discovery of the requested FPGA function bitstream and programs FPGA devices based on the environment variables in the workload description'
 
 COPY --from=builder /install_root /
 
-ENTRYPOINT [ "/usr/bin/sh", "-c", "cp -a /usr/local/fpga-sw/* /opt/intel/fpga-sw/ && ln -sf /opt/intel/fpga-sw/intel-fpga-crihook.json /etc/containers/oci/hooks.d/" ]
+ENTRYPOINT [ "/usr/bin/sh", "-c", "cp -a /usr/local/fpga-sw/* /opt/intel/fpga-sw/" ]

cmd/dlb_plugin/dlb_plugin.go

@@ -88,7 +88,7 @@ func (dp *DevicePlugin) scan() dpapi.DeviceTree {
 			ContainerPath: file,
 			Permissions:   "rw",
 		}}
-		deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devs, nil, nil, nil)
+		deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, devs, nil, nil, nil, nil)
 
 		sysfsDev := filepath.Join(dp.sysfsDir, filepath.Base(file))
 		sriovNumVFs := pluginutils.GetSriovNumVFs(sysfsDev)

cmd/fpga_admissionwebhook/README.md

@@ -23,23 +23,23 @@ devices to Kubernetes.
 
 The FPGA admission controller webhook is responsible for performing mapping from user-friendly
 function IDs to the Interface ID and Bitstream ID that are required for FPGA programming by
-the [FPGA CRI-O hook](../fpga_crihook/README.md).
+the [FPGA OCI createRuntime hook](../fpga_crihook/README.md).
 
 Mappings are stored in namespaced custom resource definition (CRD) objects, therefore the admission
 controller also performs access control, determining which bitstream can be used for which namespace.
 More details can be found in the [Mappings](#mappings) section.
 
 The admission controller also keeps the user from bypassing namespaced mapping restrictions,
 by denying admission of any pods that are trying to use internal knowledge of InterfaceID or
-Bitstream ID environment variables used by the prestart hook.
+Bitstream ID environment variables used by the createRuntime hook.
 
 ## Dependencies
 
 This component is one of a set of components that work together. You may also want to
 install the following:
 
 -   [FPGA device plugin](../fpga_plugin/README.md)
--   [FPGA prestart CRI-O hook](../fpga_crihook/README.md)
+-   [FPGA OCI createRuntime hook](../fpga_crihook/README.md)
 
 All components have the same basic dependencies as the
 [generic plugin framework dependencies](../../README.md#about)
@@ -129,7 +129,7 @@ The same mapping, but with its mode field set to `region`, would translate
 and the corresponding AF IDs are set in environment variables for the container.
 Though in this case the cluster administrator would probably want to rename
 the mapping `arria10.dcp1.2-nlb0-preprogrammed` to something like `arria10.dcp1.2-nlb0-orchestrated`
-to reflect its mode. The [FPGA CRI-O hook](../fpga_crihook/README.md) then loads the requested
+to reflect its mode. The [FPGA OCI createRuntime hook](../fpga_crihook/README.md) then loads the requested
 bitstream to a region before the container is started.
 
 Mappings of resource names are configured with objects of `AcceleratorFunction` and
@@ -183,4 +183,4 @@ and they are applicable to pods created in the corresponding namespaces.
 
 ## Next steps
 
-Continue with [FPGA prestart CRI-O hook](../fpga_crihook/README.md).
+Continue with [FPGA OCI createRuntime hook](../fpga_crihook/README.md).

cmd/fpga_crihook/README.md

@@ -1,27 +1,26 @@
-# Intel FPGA prestart CRI-O webhook for Kubernetes
+# Intel FPGA OCI createRuntime hook for Kubernetes
 
 Table of Contents
 
 * [Introduction](#introduction)
 * [Dependencies](#dependencies)
-* [Configuring CRI-O](#configuring-cri-o)
+* [Configuring CRI runtimes](#configuring-cri-runtimes)
 
 ## Introduction
 
-The FPGA CRI-O webhook is one of the components used to add support for Intel FPGA
+The FPGA CDI hook is one of the components used to add support for Intel FPGA
 devices to Kubernetes.
 
-The FPGA prestart CRI-O hook is triggered by container annotations, such as set by the
-[FPGA device plugin](../fpga_plugin/README.md).  It performs discovery of the requested FPGA
-function bitstream and then programs FPGA devices based on the environment variables
-in the workload description.
+The FPGA OCI createRuntime hook is passed by the [FPGA device plugin](../fpga_plugin/README.md) as
+a CDI device attribute to the Kubelet and then to the CRI runtime.
+It performs discovery of the requested FPGA function bitstream and then programs FPGA devices
+based on the environment variables in the workload description.
 
-The CRI-O prestart hook is only *required* when the
-[FPGA admission webhook](../fpga_admissionwebhook/README.md) is configured for orchestration
-programmed mode, and is benign (un-used) otherwise.
+The hook is only *required* when the [FPGA admission webhook](../fpga_admissionwebhook/README.md)
+is configured for orchestration programmed mode, and is benign (un-used) otherwise.
 
-> **Note:** The fpga CRI-O webhook is usually installed by the same DaemonSet as the
-> FPGA device plugin. If building and installing the CRI-O webhook by hand, it is
+> **Note:** The fpga CDI hook is usually installed by the same DaemonSet as the
+> FPGA device plugin. If building and installing the hook by hand, it is
 > recommended you reference the
 > [fpga plugin DaemonSet YAML](/deployments/fpga_plugin/base/intel-fpga-plugin-daemonset.yaml ) for
 > more details.
@@ -39,11 +38,8 @@ All components have the same basic dependencies as the
 
 See [the development guide](../../DEVEL.md) for details if you want to deploy a customized version of the CRI hook.
 
-## Configuring CRI-O
+## Configuring CRI runtimes
 
-Recent versions of [CRI-O](https://github.com/cri-o/cri-o) are shipped with default configuration
-file that prevents CRI-O to discover and configure hooks automatically.
-For FPGA orchestration programmed mode, the OCI hooks are the key component.
-Please ensure that your `/etc/crio/crio.conf` parameter `hooks_dir` is either unset
-(to enable default search paths for OCI hooks configuration) or contains the directory
-`/etc/containers/oci/hooks.d`.
+CDI should be enabled for the CRI runtime to call the hook. CRI-O has it enabled by
+default and for Containerd it should be enabled explicitly in its configuration file as
+explained in the [CDI documentation](https://github.com/cncf-tags/container-device-interface?tab=readme-ov-file#how-to-configure-cdi)

cmd/fpga_crihook/main.go

@@ -32,9 +32,6 @@ const (
 	configJSON             = "config.json"
 	fpgaRegionEnvPrefix    = "FPGA_REGION_"
 	fpgaAfuEnvPrefix       = "FPGA_AFU_"
-
-	annotationName  = "com.intel.fpga.mode"
-	annotationValue = "fpga.intel.com/region"
 )
 
 // Stdin defines structure for standard JSONed input of the OCI platform hook.
@@ -220,16 +217,6 @@ func getStdin(reader io.Reader) (*Stdin, error) {
 		return nil, err
 	}
 
-	// Check if device plugin annotation is set
-	if stdinJ.Annotations.ComIntelFpgaMode == "" {
-		return nil, errors.Errorf("annotation %s is not set", annotationName)
-	}
-
-	// Check if device plugin annotation is set
-	if stdinJ.Annotations.ComIntelFpgaMode != annotationValue {
-		return nil, errors.Errorf("annotation %s has incorrect value '%s'", annotationName, stdinJ.Annotations.ComIntelFpgaMode)
-	}
-
 	if stdinJ.Bundle == "" {
 		return nil, errors.New("'bundle' field is not set in the stdin JSON")
 	}

cmd/fpga_crihook/main_test.go

@@ -73,16 +73,6 @@ func TestGetStdin(t *testing.T) {
 			stdinJSON:   "stdin-incorrect-JSON.json",
 			expectedErr: true,
 		},
-		{
-			name:        "no annotations",
-			stdinJSON:   "stdin-no-annotations.json",
-			expectedErr: true,
-		},
-		{
-			name:        "annotation is not set",
-			stdinJSON:   "stdin-incorrect-intel-annotation.json",
-			expectedErr: true,
-		},
 	}
 	for _, tc := range tcases {
 		t.Run(tc.name, func(t *testing.T) {

cmd/fpga_crihook/testdata/config-correct.json

@@ -52,14 +52,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-no-FPGA-devices.json

@@ -52,14 +52,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-no-afu.json

@@ -51,14 +51,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-no-devices.json

@@ -52,14 +52,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-no-env.json

@@ -42,14 +42,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-no-linux.json

@@ -52,14 +52,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-no-process.json

@@ -31,14 +31,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-no-region.json

@@ -51,14 +51,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-non-existing-bitstream.json

@@ -52,14 +52,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]

cmd/fpga_crihook/testdata/config-region-afu-dont-match.json

@@ -53,14 +53,14 @@
 		}
 	],
 	"hooks": {
-		"prestart": [
+		"createRuntime": [
 			{
 				"path": "/usr/local/bin/fpga_crihook",
 				"args": [
 					"/usr/local/bin/fpga_crihook"
 				],
 				"env": [
-					"stage=prestart"
+					"stage=createRuntime"
 				]
 			}
 		]