Skip to content

Commit 2d7b236

Browse files
elezarelezar
authored
Merge pull request #497 from elezar/systemdcgroup
Add option to set additional containerd configs per runtime
2 parents a61dc14 + b435b79 commit 2d7b236

File tree

11 files changed

+313
-11
lines changed

11 files changed

+313
-11
lines changed

‎cmd/nvidia-ctk/runtime/configure/configure.go

+35
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package configure
1818

1919
import (
20+
"encoding/json"
2021
"fmt"
2122
"path/filepath"
2223

@@ -66,6 +67,8 @@ type config struct {
6667
mode string
6768
hookFilePath string
6869

70+
runtimeConfigOverrideJSON string
71+
6972
nvidiaRuntime struct {
7073
name string
7174
path string
@@ -153,6 +156,13 @@ func (m command) build() *cli.Command {
153156
Usage: "Enable CDI in the configured runtime",
154157
Destination: &config.cdi.enabled,
155158
},
159+
&cli.StringFlag{
160+
Name: "runtime-config-override",
161+
Destination: &config.runtimeConfigOverrideJSON,
162+
Usage: "specify additional runtime options as a JSON string. The paths are relative to the runtime config.",
163+
Value: "{}",
164+
EnvVars: []string{"RUNTIME_CONFIG_OVERRIDE"},
165+
},
156166
}
157167

158168
return &configure
@@ -194,6 +204,11 @@ func (m command) validateFlags(c *cli.Context, config *config) error {
194204
config.cdi.enabled = false
195205
}
196206

207+
if config.runtimeConfigOverrideJSON != "" && config.runtime != "containerd" {
208+
m.logger.Warningf("Ignoring runtime-config-override flag for %v", config.runtime)
209+
config.runtimeConfigOverrideJSON = ""
210+
}
211+
197212
return nil
198213
}
199214

@@ -237,10 +252,16 @@ func (m command) configureConfigFile(c *cli.Context, config *config) error {
237252
return fmt.Errorf("unable to load config for runtime %v: %v", config.runtime, err)
238253
}
239254

255+
runtimeConfigOverride, err := config.runtimeConfigOverride()
256+
if err != nil {
257+
return fmt.Errorf("unable to parse config overrides: %w", err)
258+
}
259+
240260
err = cfg.AddRuntime(
241261
config.nvidiaRuntime.name,
242262
config.nvidiaRuntime.path,
243263
config.nvidiaRuntime.setAsDefault,
264+
runtimeConfigOverride,
244265
)
245266
if err != nil {
246267
return fmt.Errorf("unable to update config: %v", err)
@@ -293,6 +314,20 @@ func (c *config) getOuputConfigPath() string {
293314
return c.resolveConfigFilePath()
294315
}
295316

317+
// runtimeConfigOverride converts the specified runtimeConfigOverride JSON string to a map.
318+
func (o *config) runtimeConfigOverride() (map[string]interface{}, error) {
319+
if o.runtimeConfigOverrideJSON == "" {
320+
return nil, nil
321+
}
322+
323+
runtimeOptions := make(map[string]interface{})
324+
if err := json.Unmarshal([]byte(o.runtimeConfigOverrideJSON), &runtimeOptions); err != nil {
325+
return nil, fmt.Errorf("failed to read %v as JSON: %w", o.runtimeConfigOverrideJSON, err)
326+
}
327+
328+
return runtimeOptions, nil
329+
}
330+
296331
// configureOCIHook creates and configures the OCI hook for the NVIDIA runtime
297332
func (m *command) configureOCIHook(c *cli.Context, config *config) error {
298333
err := ocihook.CreateHook(config.hookFilePath, config.nvidiaRuntime.hookPath)

‎pkg/config/engine/api.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ package engine
1919
// Interface defines the API for a runtime config updater.
2020
type Interface interface {
2121
DefaultRuntime() string
22-
AddRuntime(string, string, bool) error
22+
AddRuntime(string, string, bool, ...map[string]interface{}) error
2323
Set(string, interface{})
2424
RemoveRuntime(string) error
2525
Save(string) (int64, error)

‎pkg/config/engine/containerd/config_v1.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ type ConfigV1 Config
3030
var _ engine.Interface = (*ConfigV1)(nil)
3131

3232
// AddRuntime adds a runtime to the containerd config
33-
func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error {
33+
func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool, configOverrides ...map[string]interface{}) error {
3434
if c == nil || c.Tree == nil {
3535
return fmt.Errorf("config is nil")
3636
}
@@ -75,6 +75,16 @@ func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error
7575
}
7676
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
7777
config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
78+
79+
defaultRuntimeSubtree := subtreeAtPath(config, "plugins", "cri", "containerd", "default_runtime")
80+
if err := defaultRuntimeSubtree.applyOverrides(configOverrides...); err != nil {
81+
return fmt.Errorf("failed to apply config overrides to default_runtime: %w", err)
82+
}
83+
}
84+
85+
runtimeSubtree := subtreeAtPath(config, "plugins", "cri", "containerd", "runtimes", name)
86+
if err := runtimeSubtree.applyOverrides(configOverrides...); err != nil {
87+
return fmt.Errorf("failed to apply config overrides: %w", err)
7888
}
7989

8090
*c.Tree = config

‎pkg/config/engine/containerd/config_v2.go

+6-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import (
2525
)
2626

2727
// AddRuntime adds a runtime to the containerd config
28-
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
28+
func (c *Config) AddRuntime(name string, path string, setAsDefault bool, configOverrides ...map[string]interface{}) error {
2929
if c == nil || c.Tree == nil {
3030
return fmt.Errorf("config is nil")
3131
}
@@ -60,6 +60,11 @@ func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
6060
config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
6161
}
6262

63+
runtimeSubtree := subtreeAtPath(config, "plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name)
64+
if err := runtimeSubtree.applyOverrides(configOverrides...); err != nil {
65+
return fmt.Errorf("failed to apply config overrides: %w", err)
66+
}
67+
6368
*c.Tree = config
6469
return nil
6570
}

‎pkg/config/engine/containerd/config_v2_test.go

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/**
2+
# Copyright 2024 NVIDIA CORPORATION
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package containerd
18+
19+
import (
20+
"testing"
21+
22+
"github.com/pelletier/go-toml"
23+
"github.com/stretchr/testify/require"
24+
)
25+
26+
func TestAddRuntime(t *testing.T) {
27+
testCases := []struct {
28+
description string
29+
config string
30+
setAsDefault bool
31+
configOverrides []map[string]interface{}
32+
expectedConfig string
33+
expectedError error
34+
}{
35+
{
36+
description: "empty config not default runtime",
37+
expectedConfig: `
38+
version = 2
39+
[plugins]
40+
[plugins."io.containerd.grpc.v1.cri"]
41+
[plugins."io.containerd.grpc.v1.cri".containerd]
42+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
43+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
44+
privileged_without_host_devices = false
45+
runtime_engine = ""
46+
runtime_root = ""
47+
runtime_type = ""
48+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
49+
BinaryName = "/usr/bin/test"
50+
`,
51+
expectedError: nil,
52+
},
53+
{
54+
description: "empty config not default runtime with overrides",
55+
configOverrides: []map[string]interface{}{
56+
{
57+
"options": map[string]interface{}{
58+
"SystemdCgroup": true,
59+
},
60+
},
61+
},
62+
expectedConfig: `
63+
version = 2
64+
[plugins]
65+
[plugins."io.containerd.grpc.v1.cri"]
66+
[plugins."io.containerd.grpc.v1.cri".containerd]
67+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
68+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test]
69+
privileged_without_host_devices = false
70+
runtime_engine = ""
71+
runtime_root = ""
72+
runtime_type = ""
73+
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test.options]
74+
BinaryName = "/usr/bin/test"
75+
SystemdCgroup = true
76+
`,
77+
},
78+
}
79+
80+
for _, tc := range testCases {
81+
t.Run(tc.description, func(t *testing.T) {
82+
config, err := toml.Load(tc.config)
83+
require.NoError(t, err)
84+
expectedConfig, err := toml.Load(tc.expectedConfig)
85+
require.NoError(t, err)
86+
87+
c := &Config{
88+
Tree: config,
89+
}
90+
91+
err = c.AddRuntime("test", "/usr/bin/test", tc.setAsDefault, tc.configOverrides...)
92+
require.NoError(t, err)
93+
94+
require.EqualValues(t, expectedConfig.String(), config.String())
95+
})
96+
}
97+
}

‎pkg/config/engine/containerd/toml.go

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/**
2+
# Copyright 2024 NVIDIA CORPORATION
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package containerd
18+
19+
import (
20+
"fmt"
21+
22+
"github.com/pelletier/go-toml"
23+
)
24+
25+
// tomlTree is an alias for toml.Tree that allows for extensions.
26+
type tomlTree toml.Tree
27+
28+
func subtreeAtPath(c toml.Tree, path ...string) *tomlTree {
29+
tree := c.GetPath(path).(*toml.Tree)
30+
return (*tomlTree)(tree)
31+
}
32+
33+
func (t *tomlTree) insert(other map[string]interface{}) error {
34+
35+
for key, value := range other {
36+
if insertsubtree, ok := value.(map[string]interface{}); ok {
37+
subtree := (*toml.Tree)(t).Get(key).(*toml.Tree)
38+
return (*tomlTree)(subtree).insert(insertsubtree)
39+
}
40+
(*toml.Tree)(t).Set(key, value)
41+
}
42+
return nil
43+
}
44+
45+
func (t *tomlTree) applyOverrides(overrides ...map[string]interface{}) error {
46+
for _, override := range overrides {
47+
subconfig, err := toml.TreeFromMap(override)
48+
if err != nil {
49+
return fmt.Errorf("invalid toml config: %w", err)
50+
}
51+
if err := t.insert(subconfig.ToMap()); err != nil {
52+
return err
53+
}
54+
}
55+
return nil
56+
}

‎pkg/config/engine/crio/crio.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func New(opts ...Option) (engine.Interface, error) {
4040
}
4141

4242
// AddRuntime adds a new runtime to the crio config
43-
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
43+
func (c *Config) AddRuntime(name string, path string, setAsDefault bool, _ ...map[string]interface{}) error {
4444
if c == nil {
4545
return fmt.Errorf("config is nil")
4646
}

‎pkg/config/engine/docker/docker.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func New(opts ...Option) (engine.Interface, error) {
4949
}
5050

5151
// AddRuntime adds a new runtime to the docker config
52-
func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
52+
func (c *Config) AddRuntime(name string, path string, setAsDefault bool, _ ...map[string]interface{}) error {
5353
if c == nil {
5454
return fmt.Errorf("config is nil")
5555
}

‎tools/container/container.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ func ParseArgs(c *cli.Context, o *Options) error {
6767
}
6868

6969
// Configure applies the options to the specified config
70-
func (o Options) Configure(cfg engine.Interface) error {
71-
err := o.UpdateConfig(cfg)
70+
func (o Options) Configure(cfg engine.Interface, configOverrides ...map[string]interface{}) error {
71+
err := o.UpdateConfig(cfg, configOverrides...)
7272
if err != nil {
7373
return fmt.Errorf("unable to update config: %v", err)
7474
}
@@ -98,14 +98,14 @@ func (o Options) flush(cfg engine.Interface) error {
9898
}
9999

100100
// UpdateConfig updates the specified config to include the nvidia runtimes
101-
func (o Options) UpdateConfig(cfg engine.Interface) error {
101+
func (o Options) UpdateConfig(cfg engine.Interface, configOverrides ...map[string]interface{}) error {
102102
runtimes := operator.GetRuntimes(
103103
operator.WithNvidiaRuntimeName(o.RuntimeName),
104104
operator.WithSetAsDefault(o.SetAsDefault),
105105
operator.WithRoot(o.RuntimeDir),
106106
)
107107
for name, runtime := range runtimes {
108-
err := cfg.AddRuntime(name, runtime.Path, runtime.SetAsDefault)
108+
err := cfg.AddRuntime(name, runtime.Path, runtime.SetAsDefault, configOverrides...)
109109
if err != nil {
110110
return fmt.Errorf("failed to update runtime %q: %v", name, err)
111111
}

0 commit comments

Comments
 (0)