Skip to content

Commit 5c28118

Browse files
committed
Support cpulimit for cgroup v2
1 parent 7ac5d9d commit 5c28118

File tree

5 files changed

+172
-17
lines changed

5 files changed

+172
-17
lines changed

components/ws-daemon/pkg/cpulimit/cfs.go

+11-11
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@ import (
1616
"golang.org/x/xerrors"
1717
)
1818

19-
// CgroupCFSController controls a cgroup's CFS settings
20-
type CgroupCFSController string
19+
// CgroupV1CFSController controls a cgroup's CFS settings
20+
type CgroupV1CFSController string
2121

2222
// Usage returns the cpuacct.usage value of the cgroup
23-
func (basePath CgroupCFSController) Usage() (usage CPUTime, err error) {
23+
func (basePath CgroupV1CFSController) Usage() (usage CPUTime, err error) {
2424
cputime, err := basePath.readCpuUsage()
2525
if err != nil {
2626
return 0, xerrors.Errorf("cannot read cpuacct.usage: %w", err)
@@ -30,7 +30,7 @@ func (basePath CgroupCFSController) Usage() (usage CPUTime, err error) {
3030
}
3131

3232
// SetQuota sets a new CFS quota on the cgroup
33-
func (basePath CgroupCFSController) SetLimit(limit Bandwidth) (changed bool, err error) {
33+
func (basePath CgroupV1CFSController) SetLimit(limit Bandwidth) (changed bool, err error) {
3434
period, err := basePath.readCfsPeriod()
3535
if err != nil {
3636
err = xerrors.Errorf("cannot parse CFS period: %w", err)
@@ -55,8 +55,8 @@ func (basePath CgroupCFSController) SetLimit(limit Bandwidth) (changed bool, err
5555
return true, nil
5656
}
5757

58-
func (basePath CgroupCFSController) readParentQuota() time.Duration {
59-
parent := CgroupCFSController(filepath.Dir(string(basePath)))
58+
func (basePath CgroupV1CFSController) readParentQuota() time.Duration {
59+
parent := CgroupV1CFSController(filepath.Dir(string(basePath)))
6060
pq, err := parent.readCfsQuota()
6161
if err != nil {
6262
return time.Duration(0)
@@ -65,7 +65,7 @@ func (basePath CgroupCFSController) readParentQuota() time.Duration {
6565
return time.Duration(pq) * time.Microsecond
6666
}
6767

68-
func (basePath CgroupCFSController) readString(path string) (string, error) {
68+
func (basePath CgroupV1CFSController) readString(path string) (string, error) {
6969
fn := filepath.Join(string(basePath), path)
7070
fc, err := os.ReadFile(fn)
7171
if err != nil {
@@ -76,7 +76,7 @@ func (basePath CgroupCFSController) readString(path string) (string, error) {
7676
return s, nil
7777
}
7878

79-
func (basePath CgroupCFSController) readCfsPeriod() (time.Duration, error) {
79+
func (basePath CgroupV1CFSController) readCfsPeriod() (time.Duration, error) {
8080
s, err := basePath.readString("cpu.cfs_period_us")
8181
if err != nil {
8282
return 0, err
@@ -89,7 +89,7 @@ func (basePath CgroupCFSController) readCfsPeriod() (time.Duration, error) {
8989
return time.Duration(uint64(p)) * time.Microsecond, nil
9090
}
9191

92-
func (basePath CgroupCFSController) readCfsQuota() (time.Duration, error) {
92+
func (basePath CgroupV1CFSController) readCfsQuota() (time.Duration, error) {
9393
s, err := basePath.readString("cpu.cfs_quota_us")
9494
if err != nil {
9595
return 0, err
@@ -106,7 +106,7 @@ func (basePath CgroupCFSController) readCfsQuota() (time.Duration, error) {
106106
return time.Duration(p) * time.Microsecond, nil
107107
}
108108

109-
func (basePath CgroupCFSController) readCpuUsage() (time.Duration, error) {
109+
func (basePath CgroupV1CFSController) readCpuUsage() (time.Duration, error) {
110110
s, err := basePath.readString("cpuacct.usage")
111111
if err != nil {
112112
return 0, err
@@ -120,7 +120,7 @@ func (basePath CgroupCFSController) readCpuUsage() (time.Duration, error) {
120120
}
121121

122122
// NrThrottled returns the number of CFS periods the cgroup was throttled in
123-
func (basePath CgroupCFSController) NrThrottled() (uint64, error) {
123+
func (basePath CgroupV1CFSController) NrThrottled() (uint64, error) {
124124
f, err := os.Open(filepath.Join(string(basePath), "cpu.stat"))
125125
if err != nil {
126126
return 0, xerrors.Errorf("cannot read cpu.stat: %w", err)

components/ws-daemon/pkg/cpulimit/cfs_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func TestCfsSetLimit(t *testing.T) {
6969
t.Fatal(err)
7070
}
7171

72-
cfs := CgroupCFSController(tempdir)
72+
cfs := CgroupV1CFSController(tempdir)
7373
changed, err := cfs.SetLimit(tc.bandWidth)
7474
if err != nil {
7575
t.Fatal(err)
@@ -110,7 +110,7 @@ func TestReadCfsQuota(t *testing.T) {
110110
t.Fatal(err)
111111
}
112112

113-
cfs := CgroupCFSController(tempdir)
113+
cfs := CgroupV1CFSController(tempdir)
114114
v, err := cfs.readCfsQuota()
115115
if err != nil {
116116
t.Fatal(err)
@@ -132,7 +132,7 @@ func TestReadCfsPeriod(t *testing.T) {
132132
t.Fatal(err)
133133
}
134134

135-
cfs := CgroupCFSController(tempdir)
135+
cfs := CgroupV1CFSController(tempdir)
136136
v, err := cfs.readCfsPeriod()
137137
if err != nil {
138138
t.Fatal(err)
@@ -155,7 +155,7 @@ func TestReadCpuUsage(t *testing.T) {
155155
t.Fatal(err)
156156
}
157157

158-
cfs := CgroupCFSController(tempdir)
158+
cfs := CgroupV1CFSController(tempdir)
159159
v, err := cfs.readCpuUsage()
160160
if err != nil {
161161
t.Fatal(err)
+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License-AGPL.txt in the project root for license information.
4+
5+
package cpulimit
6+
7+
import (
8+
"bufio"
9+
"math"
10+
"os"
11+
"path/filepath"
12+
"strconv"
13+
"strings"
14+
"time"
15+
16+
"golang.org/x/xerrors"
17+
)
18+
19+
type CgroupV2CFSController string
20+
21+
func (basePath CgroupV2CFSController) Usage() (CPUTime, error) {
22+
usage, err := basePath.getFlatKeyedValue("usage_usec")
23+
if err != nil {
24+
return 0, err
25+
}
26+
27+
return CPUTime(time.Duration(usage) * time.Microsecond), nil
28+
}
29+
30+
func (basePath CgroupV2CFSController) SetLimit(limit Bandwidth) (changed bool, err error) {
31+
quota, period, err := basePath.readCpuMax()
32+
if err != nil {
33+
return false, xerrors.Errorf("failed to read cpu max from %s: %w", basePath, err)
34+
}
35+
36+
target := limit.Quota(period)
37+
if quota == target {
38+
return false, nil
39+
}
40+
41+
err = basePath.writeQuota(target)
42+
if err != nil {
43+
return false, xerrors.Errorf("cannot set CFS quota of %d (period is %d, parent quota is %d): %w",
44+
target.Microseconds(), period.Microseconds(), basePath.readParentQuota().Microseconds(), err)
45+
}
46+
47+
return true, nil
48+
}
49+
50+
func (basePath CgroupV2CFSController) NrThrottled() (uint64, error) {
51+
throttled, err := basePath.getFlatKeyedValue("nr_throttled")
52+
if err != nil {
53+
return 0, err
54+
}
55+
56+
return uint64(throttled), nil
57+
}
58+
59+
func (basePath CgroupV2CFSController) readCpuMax() (time.Duration, time.Duration, error) {
60+
cpuMaxPath := filepath.Join(string(basePath), "cpu.max")
61+
cpuMax, err := os.ReadFile(cpuMaxPath)
62+
if err != nil {
63+
return 0, 0, xerrors.Errorf("unable to read cpu.max: %w", err)
64+
}
65+
66+
parts := strings.Fields(string(cpuMax))
67+
if len(parts) < 2 {
68+
return 0, 0, xerrors.Errorf("cpu.max did not have expected number of fields: %s", parts)
69+
}
70+
71+
var quota int64
72+
if parts[0] == "max" {
73+
quota = math.MaxInt64
74+
} else {
75+
quota, err = strconv.ParseInt(parts[0], 10, 64)
76+
if err != nil {
77+
return 0, 0, xerrors.Errorf("could not parse quota of %s: %w", parts[0], err)
78+
}
79+
}
80+
81+
period, err := strconv.ParseInt(parts[1], 10, 64)
82+
if err != nil {
83+
return 0, 0, xerrors.Errorf("could not parse period of %s: %w", parts[1], err)
84+
}
85+
86+
return time.Duration(quota) * time.Microsecond, time.Duration(period) * time.Microsecond, nil
87+
}
88+
89+
func (basePath CgroupV2CFSController) writeQuota(quota time.Duration) error {
90+
cpuMaxPath := filepath.Join(string(basePath), "cpu.max")
91+
return os.WriteFile(cpuMaxPath, []byte(strconv.FormatInt(quota.Microseconds(), 10)), 0644)
92+
}
93+
94+
func (basePath CgroupV2CFSController) readParentQuota() time.Duration {
95+
parent := CgroupV2CFSController(filepath.Dir(string(basePath)))
96+
quota, _, err := parent.readCpuMax()
97+
if err != nil {
98+
return time.Duration(0)
99+
}
100+
101+
return time.Duration(quota)
102+
}
103+
104+
func (basePath CgroupV2CFSController) getFlatKeyedValue(key string) (int64, error) {
105+
stats, err := os.Open(filepath.Join(string(basePath), "cpu.stat"))
106+
if err != nil {
107+
return 0, xerrors.Errorf("cannot read cpu.stat: %w", err)
108+
}
109+
defer stats.Close()
110+
111+
scanner := bufio.NewScanner(stats)
112+
for scanner.Scan() {
113+
entry := scanner.Text()
114+
if !strings.HasPrefix(entry, key) {
115+
continue
116+
}
117+
118+
r, err := strconv.ParseInt(strings.TrimSpace(strings.TrimPrefix(entry, key)), 10, 64)
119+
if err != nil {
120+
return 0, xerrors.Errorf("cannot parse cpu.stat: %s: %w", entry, err)
121+
}
122+
return int64(r), nil
123+
}
124+
return 0, xerrors.Errorf("cpu.stat did not contain nr_throttled")
125+
}

components/ws-daemon/pkg/cpulimit/cpulimit.go

+9
Original file line numberDiff line numberDiff line change
@@ -294,3 +294,12 @@ func (bl *ClampingBucketLimiter) Limit(budgetSpent CPUTime) (newLimit Bandwidth)
294294
// empty bucket list
295295
return 0
296296
}
297+
298+
type CFSController interface {
299+
// Usage returns the cpuacct.usage value of the cgroup
300+
Usage() (usage CPUTime, err error)
301+
// SetQuota sets a new CFS quota on the cgroup
302+
SetLimit(limit Bandwidth) (changed bool, err error)
303+
// NrThrottled returns the number of CFS periods the cgroup was throttled
304+
NrThrottled() (uint64, error)
305+
}

components/ws-daemon/pkg/cpulimit/dispatch.go

+23-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package cpulimit
66

77
import (
88
"context"
9+
"os"
910
"path/filepath"
1011
"sync"
1112
"time"
@@ -97,7 +98,7 @@ type DispatchListener struct {
9798
}
9899

99100
type workspace struct {
100-
CFS CgroupCFSController
101+
CFS CFSController
101102
OWI logrus.Fields
102103
HardLimit ResourceLimiter
103104

@@ -175,8 +176,13 @@ func (d *DispatchListener) WorkspaceAdded(ctx context.Context, ws *dispatch.Work
175176
return xerrors.Errorf("cannot start governer: %w", err)
176177
}
177178

179+
controller, err := newCFSController(d.Config.CGroupBasePath, cgroupPath)
180+
if err != nil {
181+
return err
182+
}
183+
178184
d.workspaces[ws.InstanceID] = &workspace{
179-
CFS: CgroupCFSController(filepath.Join(d.Config.CGroupBasePath, "cpu", cgroupPath)),
185+
CFS: controller,
180186
OWI: ws.OWI(),
181187
}
182188
go func() {
@@ -214,3 +220,18 @@ func (d *DispatchListener) WorkspaceUpdated(ctx context.Context, ws *dispatch.Wo
214220

215221
return nil
216222
}
223+
224+
func newCFSController(basePath, cgroupPath string) (CFSController, error) {
225+
controllers := filepath.Join(basePath, "cgroup.controllers")
226+
_, err := os.Stat(controllers)
227+
228+
if os.IsNotExist(err) {
229+
return CgroupV1CFSController(filepath.Join(basePath, "cpu", cgroupPath)), nil
230+
}
231+
232+
if err == nil {
233+
return CgroupV2CFSController(filepath.Join(basePath, cgroupPath)), nil
234+
}
235+
236+
return nil, err
237+
}

0 commit comments

Comments
 (0)