move nvidia-ctk hook command into own binary

Signed-off-by: Avi Deitcher <[email protected]>

Author: deitch

cmd/nvidia-cdi-hook/README.md

@@ -0,0 +1,31 @@
+# NVIDIA CDI Hook
+
+The CLI `nvidia-cdi-hook` provides container device runtime hook capabilities when
+called by a container runtime, as specific in a
+[Container Device Interface](https://github.com/cncf-tags/container-device-interface/blob/main/SPEC.md)
+file.
+
+## Generating a CDI
+
+The CDI itself is created for an Nvidia-capable device using the
+[`nvidia-ctk cdi`](../nvidia-ctk/) command.
+
+When `nvidia-ctk cdi generate` is run, the CDI specification is generated as a yaml file.
+The CDI file provides instructions for a container runtime to set up devices, files and
+other resources for the container prior to starting it. Those instructions
+may include executing command-line tools to prepare the filesystem. The execution
+of such command-line tools is called a hook.
+
+`nvidia-cdi-hook` is the CLI tool that is expected to be called by the container runtime,
+when specified by the CDI file.
+
+See the [`nvidia-ctk` documentation](../nvidia-ctk/README.md) for more information
+on generating a CDI file.
+
+## Functionality
+
+The `nvidia-cdi-hook` CLI provides the following functionality:
+
+* `chmod` - Change the permissions of a file or directory inside the directory path to be mounted into a container.
+* `create-symlinks` - Create symlinks inside the directory path to be mounted into a container.
+* `update-ldcache` - Update the dynamic linker cache inside the directory path to be mounted into a container.

cmd/nvidia-ctk/hook/chmod/chmod.go renamed to cmd/nvidia-cdi-hook/chmod/chmod.go

@@ -0,0 +1,99 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"os"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
+
+	cli "github.com/urfave/cli/v2"
+
+	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
+	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
+)
+
+// options defines the options that can be set for the CLI through config files,
+// environment variables, or command line flags
+type options struct {
+	// Debug indicates whether the CLI is started in "debug" mode
+	Debug bool
+	// Quiet indicates whether the CLI is started in "quiet" mode
+	Quiet bool
+}
+
+func main() {
+	logger := logrus.New()
+
+	// Create a options struct to hold the parsed environment variables or command line flags
+	opts := options{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "NVIDIA CDI Hook"
+	c.UseShortOptionHandling = true
+	c.EnableBashCompletion = true
+	c.Usage = "Command to structure files for usage inside a container, called as hooks from a container runtime, defined in a CDI yaml file"
+	c.Version = info.GetVersionString()
+
+	// Setup the flags for this command
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "debug",
+			Aliases:     []string{"d"},
+			Usage:       "Enable debug-level logging",
+			Destination: &opts.Debug,
+			EnvVars:     []string{"NVIDIA_CDI_DEBUG"},
+		},
+		&cli.BoolFlag{
+			Name:        "quiet",
+			Usage:       "Suppress all output except for errors; overrides --debug",
+			Destination: &opts.Quiet,
+			EnvVars:     []string{"NVIDIA_CDI_QUIET"},
+		},
+	}
+
+	// Set log-level for all subcommands
+	c.Before = func(c *cli.Context) error {
+		logLevel := logrus.InfoLevel
+		if opts.Debug {
+			logLevel = logrus.DebugLevel
+		}
+		if opts.Quiet {
+			logLevel = logrus.ErrorLevel
+		}
+		logger.SetLevel(logLevel)
+		return nil
+	}
+
+	// Define the subcommands
+	c.Commands = []*cli.Command{
+		ldcache.NewCommand(logger),
+		symlinks.NewCommand(logger),
+		chmod.NewCommand(logger),
+	}
+
+	// Run the CLI
+	err := c.Run(os.Args)
+	if err != nil {
+		logger.Errorf("%v", err)
+		os.Exit(1)
+	}
+}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go renamed to cmd/nvidia-cdi-hook/create-symlinks/create-symlinks.go

@@ -47,7 +47,7 @@ type options struct {
 	deviceNameStrategies cli.StringSlice
 	driverRoot           string
 	devRoot              string
-	nvidiaCTKPath        string
+	nvidiaCDIHookPath    string
 	ldconfigPath         string
 	mode                 string
 	vendor               string
@@ -131,10 +131,15 @@ func (m command) build() *cli.Command {
 			Usage:       "Specify the path to search for libraries when discovering the entities that should be included in the CDI specification.\n\tNote: This option only applies to CSV mode.",
 			Destination: &opts.librarySearchPaths,
 		},
+		&cli.StringFlag{
+			Name:        "nvidia-cdi-hook-path",
+			Usage:       "Specify the path to use for the nvidia-cdi-hook in the generated CDI specification. If this is left empty, the path will be searched.",
+			Destination: &opts.nvidiaCDIHookPath,
+		},
 		&cli.StringFlag{
 			Name:        "nvidia-ctk-path",
-			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
-			Destination: &opts.nvidiaCTKPath,
+			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched. DEPRECATED. Use flag --nvidia-cdi-hook-path instead.",
+			Destination: &opts.nvidiaCDIHookPath,
 		},
 		&cli.StringFlag{
 			Name:        "ldconfig-path",
@@ -198,7 +203,7 @@ func (m command) validateFlags(c *cli.Context, opts *options) error {
 		}
 	}
 
-	opts.nvidiaCTKPath = config.ResolveNVIDIACTKPath(m.logger, opts.nvidiaCTKPath)
+	opts.nvidiaCDIHookPath = config.ResolveNVIDIACDIHookPath(m.logger, opts.nvidiaCDIHookPath)
 
 	if outputFileFormat := formatFromFilename(opts.output); outputFileFormat != "" {
 		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
@@ -262,7 +267,7 @@ func (m command) generateSpec(opts *options) (spec.Interface, error) {
 		nvcdi.WithLogger(m.logger),
 		nvcdi.WithDriverRoot(opts.driverRoot),
 		nvcdi.WithDevRoot(opts.devRoot),
-		nvcdi.WithNVIDIACTKPath(opts.nvidiaCTKPath),
+		nvcdi.WithNVIDIACDIHookPath(opts.nvidiaCDIHookPath),
 		nvcdi.WithLdconfigPath(opts.ldconfigPath),
 		nvcdi.WithDeviceNamers(deviceNamers...),
 		nvcdi.WithMode(opts.mode),

cmd/nvidia-cdi-hook/main.go

@@ -17,13 +17,13 @@
 package hook
 
 import (
-	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
+	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
 
 	"github.com/urfave/cli/v2"
 
-	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
-	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
+	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
+	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
 )
 
 type hookCommand struct {

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go renamed to cmd/nvidia-cdi-hook/update-ldcache/update-ldcache.go

@@ -33,8 +33,9 @@ const (
 	configOverride = "XDG_CONFIG_HOME"
 	configFilePath = "nvidia-container-runtime/config.toml"
 
-	nvidiaCTKExecutable      = "nvidia-ctk"
-	nvidiaCTKDefaultFilePath = "/usr/bin/nvidia-ctk"
+	nvidiaCTKExecutable          = "nvidia-ctk"
+	nvidiaCTKDefaultFilePath     = "/usr/bin/nvidia-ctk"
+	nvidiaCDIHookDefaultFilePath = "/usr/bin/nvidia-cdi-hook"
 
 	nvidiaContainerRuntimeHookExecutable  = "nvidia-container-runtime-hook"
 	nvidiaContainerRuntimeHookDefaultPath = "/usr/bin/nvidia-container-runtime-hook"
@@ -186,6 +187,22 @@ func ResolveNVIDIACTKPath(logger logger.Interface, nvidiaCTKPath string) string
 	)
 }
 
+// ResolveNVIDIACDIHookPath resolves the path to the nvidia-cdi-hook binary.
+// This executable is used in hooks and needs to be an absolute path.
+// If the path is specified as an absolute path, it is used directly
+// without checking for existence of an executable at that path.
+func ResolveNVIDIACDIHookPath(logger logger.Interface, nvidiaCDIHookPath string) string {
+	if filepath.Base(nvidiaCDIHookPath) == "nvidia-ctk" {
+		return ResolveNVIDIACTKPath(logger, nvidiaCDIHookPath)
+	}
+	return resolveWithDefault(
+		logger,
+		"NVIDIA Container Toolkit CLI",
+		nvidiaCDIHookPath,
+		nvidiaCDIHookDefaultFilePath,
+	)
+}
+
 // ResolveNVIDIAContainerRuntimeHookPath resolves the path the nvidia-container-runtime-hook binary.
 func ResolveNVIDIAContainerRuntimeHookPath(logger logger.Interface, nvidiaContainerRuntimeHookPath string) string {
 	return resolveWithDefault(

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -36,20 +36,20 @@ import (
 // TODO: The logic for creating DRM devices should be consolidated between this
 // and the logic for generating CDI specs for a single device. This is only used
 // when applying OCI spec modifications to an incoming spec in "legacy" mode.
-func NewDRMNodesDiscoverer(logger logger.Interface, devices image.VisibleDevices, devRoot string, nvidiaCTKPath string) (Discover, error) {
+func NewDRMNodesDiscoverer(logger logger.Interface, devices image.VisibleDevices, devRoot string, nvidiaCDIHookPath string) (Discover, error) {
 	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, devices, devRoot)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
 	}
 
-	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, devRoot, nvidiaCTKPath)
+	drmByPathSymlinks := newCreateDRMByPathSymlinks(logger, drmDeviceNodes, devRoot, nvidiaCDIHookPath)
 
 	discover := Merge(drmDeviceNodes, drmByPathSymlinks)
 	return discover, nil
 }
 
 // NewGraphicsMountsDiscoverer creates a discoverer for the mounts required by graphics tools such as vulkan.
-func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string) (Discover, error) {
+func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCDIHookPath string) (Discover, error) {
 	libraries := NewMounts(
 		logger,
 		driver.Libraries(),
@@ -74,7 +74,7 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, n
 		},
 	)
 
-	xorg := optionalXorgDiscoverer(logger, driver, nvidiaCTKPath)
+	xorg := optionalXorgDiscoverer(logger, driver, nvidiaCDIHookPath)
 
 	discover := Merge(
 		libraries,
@@ -87,19 +87,19 @@ func NewGraphicsMountsDiscoverer(logger logger.Interface, driver *root.Driver, n
 
 type drmDevicesByPath struct {
 	None
-	logger        logger.Interface
-	nvidiaCTKPath string
-	devRoot       string
-	devicesFrom   Discover
+	logger            logger.Interface
+	nvidiaCDIHookPath string
+	devRoot           string
+	devicesFrom       Discover
 }
 
 // newCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
-func newCreateDRMByPathSymlinks(logger logger.Interface, devices Discover, devRoot string, nvidiaCTKPath string) Discover {
+func newCreateDRMByPathSymlinks(logger logger.Interface, devices Discover, devRoot string, nvidiaCDIHookPath string) Discover {
 	d := drmDevicesByPath{
-		logger:        logger,
-		nvidiaCTKPath: nvidiaCTKPath,
-		devRoot:       devRoot,
-		devicesFrom:   devices,
+		logger:            logger,
+		nvidiaCDIHookPath: nvidiaCDIHookPath,
+		devRoot:           devRoot,
+		devicesFrom:       devices,
 	}
 
 	return &d
@@ -127,8 +127,8 @@ func (d drmDevicesByPath) Hooks() ([]Hook, error) {
 		args = append(args, "--link", l)
 	}
 
-	hook := CreateNvidiaCTKHook(
-		d.nvidiaCTKPath,
+	hook := CreateNvidiaCDIHook(
+		d.nvidiaCDIHookPath,
 		"create-symlinks",
 		args...,
 	)
@@ -233,25 +233,25 @@ func newDRMDeviceFilter(devices image.VisibleDevices, devRoot string) (Filter, e
 }
 
 type xorgHooks struct {
-	libraries     Discover
-	driverVersion string
-	nvidiaCTKPath string
+	libraries         Discover
+	driverVersion     string
+	nvidiaCDIHookPath string
 }
 
 var _ Discover = (*xorgHooks)(nil)
 
 // optionalXorgDiscoverer creates a discoverer for Xorg libraries.
 // If the creation of the discoverer fails, a None discoverer is returned.
-func optionalXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string) Discover {
-	xorg, err := newXorgDiscoverer(logger, driver, nvidiaCTKPath)
+func optionalXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCDIHookPath string) Discover {
+	xorg, err := newXorgDiscoverer(logger, driver, nvidiaCDIHookPath)
 	if err != nil {
 		logger.Warningf("Failed to create Xorg discoverer: %v; skipping xorg libraries", err)
 		return None{}
 	}
 	return xorg
 }
 
-func newXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPath string) (Discover, error) {
+func newXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCDIHookPath string) (Discover, error) {
 	libCudaPaths, err := cuda.New(
 		driver.Libraries(),
 	).Locate(".*.*")
@@ -281,9 +281,9 @@ func newXorgDiscoverer(logger logger.Interface, driver *root.Driver, nvidiaCTKPa
 		},
 	)
 	xorgHooks := xorgHooks{
-		libraries:     xorgLibs,
-		driverVersion: version,
-		nvidiaCTKPath: nvidiaCTKPath,
+		libraries:         xorgLibs,
+		driverVersion:     version,
+		nvidiaCDIHookPath: nvidiaCDIHookPath,
 	}
 
 	xorgConfig := NewMounts(
@@ -332,7 +332,7 @@ func (m xorgHooks) Hooks() ([]Hook, error) {
 	link := strings.TrimSuffix(target, "."+m.driverVersion)
 	links := []string{fmt.Sprintf("%s::%s", filepath.Base(target), link)}
 	symlinkHook := CreateCreateSymlinkHook(
-		m.nvidiaCTKPath,
+		m.nvidiaCDIHookPath,
 		links,
 	)