diff --git a/components/ws-daemon/nsinsider/main.go b/components/ws-daemon/nsinsider/main.go index 1abefa1dd5559a..b8d02bcbff395e 100644 --- a/components/ws-daemon/nsinsider/main.go +++ b/components/ws-daemon/nsinsider/main.go @@ -490,6 +490,10 @@ func main() { Name: "bucketsize", Required: false, }, + &cli.BoolFlag{ + Name: "enforce", + Required: false, + }, }, Action: func(c *cli.Context) error { const drop_stats = "ws-connection-drop-stats" @@ -500,6 +504,7 @@ func main() { if bucketSize == 0 { bucketSize = 1000 } + enforce := c.Bool("enforce") // nft add table ip gitpod gitpodTable := nftcon.AddTable(&nftables.Table{ @@ -534,6 +539,11 @@ func main() { return err } + verdict := expr.VerdictAccept + if enforce { + verdict = expr.VerdictDrop + } + // nft add rule ip gitpod ratelimit ip protocol tcp ct state new meter ws-connections // '{ ip daddr & 0.0.0.0 timeout 1m limit rate over 3000/minute burst 1000 packets }' counter name ws-connection-drop-stats drop nftcon.AddRule(&nftables.Rule{ @@ -613,7 +623,7 @@ func main() { }, // drop &expr.Verdict{ - Kind: expr.VerdictAccept, + Kind: verdict, }, }, }) diff --git a/components/ws-daemon/pkg/daemon/daemon.go b/components/ws-daemon/pkg/daemon/daemon.go index 5fbea844cab42a..20a4ec323e74dd 100644 --- a/components/ws-daemon/pkg/daemon/daemon.go +++ b/components/ws-daemon/pkg/daemon/daemon.go @@ -97,24 +97,28 @@ func NewDaemon(config Config, reg prometheus.Registerer) (*Daemon, error) { return nil, xerrors.Errorf("cannot register cgroup plugin metrics: %w", err) } - var configReloader CompositeConfigReloader - configReloader = append(configReloader, ConfigReloaderFunc(func(ctx context.Context, config *Config) error { - cgroupV1IOLimiter.Update(config.IOLimit.WriteBWPerSecond.Value(), config.IOLimit.ReadBWPerSecond.Value(), config.IOLimit.WriteIOPS, config.IOLimit.ReadIOPS) - cgroupV2IOLimiter.Update(config.IOLimit.WriteBWPerSecond.Value(), config.IOLimit.ReadBWPerSecond.Value(), config.IOLimit.WriteIOPS, config.IOLimit.ReadIOPS) - procV2Plugin.Update(config.ProcLimit) - return nil - })) - listener := []dispatch.Listener{ cpulimit.NewDispatchListener(&config.CPULimit, reg), markUnmountFallback, cgroupPlugins, } + netlimiter := netlimit.NewConnLimiter(config.NetLimit, reg) if config.NetLimit.Enabled { - listener = append(listener, netlimit.NewConnLimiter(config.NetLimit, reg)) + listener = append(listener, netlimiter) } + var configReloader CompositeConfigReloader + configReloader = append(configReloader, ConfigReloaderFunc(func(ctx context.Context, config *Config) error { + cgroupV1IOLimiter.Update(config.IOLimit.WriteBWPerSecond.Value(), config.IOLimit.ReadBWPerSecond.Value(), config.IOLimit.WriteIOPS, config.IOLimit.ReadIOPS) + cgroupV2IOLimiter.Update(config.IOLimit.WriteBWPerSecond.Value(), config.IOLimit.ReadBWPerSecond.Value(), config.IOLimit.WriteIOPS, config.IOLimit.ReadIOPS) + procV2Plugin.Update(config.ProcLimit) + if config.NetLimit.Enabled { + netlimiter.Update(config.NetLimit) + } + return nil + })) + dsptch, err := dispatch.NewDispatch(containerRuntime, clientset, config.Runtime.KubernetesNamespace, nodename, listener...) if err != nil { return nil, err diff --git a/components/ws-daemon/pkg/netlimit/config.go b/components/ws-daemon/pkg/netlimit/config.go index e898e8469a3f44..a010e895db7b0d 100644 --- a/components/ws-daemon/pkg/netlimit/config.go +++ b/components/ws-daemon/pkg/netlimit/config.go @@ -6,6 +6,7 @@ package netlimit type Config struct { Enabled bool `json:"enabled"` + Enforce bool `json:"enforce"` ConnectionsPerMinute int64 `json:"connectionsPerMinute"` BucketSize int64 `json:"bucketSize"` } diff --git a/components/ws-daemon/pkg/netlimit/netlimit.go b/components/ws-daemon/pkg/netlimit/netlimit.go index 4619fd086cf365..d18cfedbdb3cf1 100644 --- a/components/ws-daemon/pkg/netlimit/netlimit.go +++ b/components/ws-daemon/pkg/netlimit/netlimit.go @@ -48,10 +48,12 @@ func NewConnLimiter(config Config, prom prometheus.Registerer) *ConnLimiter { s.config = config - prom.MustRegister( - s.droppedBytes, - s.droppedPackets, - ) + if config.Enabled { + prom.MustRegister( + s.droppedBytes, + s.droppedPackets, + ) + } return s } @@ -136,6 +138,9 @@ func (c *ConnLimiter) limitWorkspace(ctx context.Context, ws *dispatch.Workspace err = nsinsider.Nsinsider(ws.InstanceID, int(pid), func(cmd *exec.Cmd) { cmd.Args = append(cmd.Args, "setup-connection-limit", "--limit", strconv.Itoa(int(c.config.ConnectionsPerMinute)), "--bucketsize", strconv.Itoa(int(c.config.BucketSize))) + if c.config.Enforce { + cmd.Args = append(cmd.Args, "--enforce") + } }, nsinsider.EnterMountNS(false), nsinsider.EnterNetNS(true)) if err != nil { log.WithError(err).WithFields(ws.OWI()).Error("cannot enable connection limiting") @@ -171,3 +176,11 @@ func (c *ConnLimiter) limitWorkspace(ctx context.Context, ws *dispatch.Workspace return nil } + +func (c *ConnLimiter) Update(config Config) { + c.mu.Lock() + defer c.mu.Unlock() + + c.config = config + log.WithField("config", config).Info("updating network connection limits") +} diff --git a/install/installer/cmd/testdata/render/aws-setup/output.golden b/install/installer/cmd/testdata/render/aws-setup/output.golden index 0d2371b630ab87..3955b50ccb5799 100644 --- a/install/installer/cmd/testdata/render/aws-setup/output.golden +++ b/install/installer/cmd/testdata/render/aws-setup/output.golden @@ -4708,6 +4708,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -6868,7 +6869,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: de8ed99a057f28c772db0287eeb93b12fe89401a2645f94d0b533eb0e5640654 + gitpod.io/checksum_config: b16f1014837e91b50bf4b9d18e29a1c41adf08d502dee1bcc035987eb0810c5a seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/azure-setup/output.golden b/install/installer/cmd/testdata/render/azure-setup/output.golden index ed8d8a85dd248a..4c828c6385302b 100644 --- a/install/installer/cmd/testdata/render/azure-setup/output.golden +++ b/install/installer/cmd/testdata/render/azure-setup/output.golden @@ -4569,6 +4569,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -6713,7 +6714,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f39e45d788bb170ba728cfc3c69c5ca00a2b88a73a2d7288bcc1caee8e638405 + gitpod.io/checksum_config: c6d2a80d1de7a1f91bb2bea177c784c3bc7d4cd50a3e761795fad1188e41f831 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/customization/output.golden b/install/installer/cmd/testdata/render/customization/output.golden index ffeece7b004006..6659dfbe8c7f09 100644 --- a/install/installer/cmd/testdata/render/customization/output.golden +++ b/install/installer/cmd/testdata/render/customization/output.golden @@ -5526,6 +5526,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -7946,7 +7947,7 @@ spec: metadata: annotations: gitpod.io: hello - gitpod.io/checksum_config: eed464cffbb7a8f5698bc1163b1e22cbc4593e154e2662c43dd2f643863c4548 + gitpod.io/checksum_config: 2285dea7c1cdbfb36111baedc49012c61ac5f537bc3083118721ddf55e21ae07 hello: world seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null diff --git a/install/installer/cmd/testdata/render/external-registry/output.golden b/install/installer/cmd/testdata/render/external-registry/output.golden index defae8853ccef4..f78821f4f01de7 100644 --- a/install/installer/cmd/testdata/render/external-registry/output.golden +++ b/install/installer/cmd/testdata/render/external-registry/output.golden @@ -4756,6 +4756,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -6995,7 +6996,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f39e45d788bb170ba728cfc3c69c5ca00a2b88a73a2d7288bcc1caee8e638405 + gitpod.io/checksum_config: c6d2a80d1de7a1f91bb2bea177c784c3bc7d4cd50a3e761795fad1188e41f831 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/gcp-setup/output.golden b/install/installer/cmd/testdata/render/gcp-setup/output.golden index 0f146bf651c9ad..830e87ea4fad93 100644 --- a/install/installer/cmd/testdata/render/gcp-setup/output.golden +++ b/install/installer/cmd/testdata/render/gcp-setup/output.golden @@ -4529,6 +4529,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -6687,7 +6688,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f985a365ca2934d925e1510c4398f0cf27ba9a85413f34a1f119317ad098f028 + gitpod.io/checksum_config: f24b5c0e027f2d12934fd5a2f5217d05b812d8dd6713d7600a9044e49346ad73 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/http-proxy/output.golden b/install/installer/cmd/testdata/render/http-proxy/output.golden index 7c47ec60362fbb..de3b99b755e0b0 100644 --- a/install/installer/cmd/testdata/render/http-proxy/output.golden +++ b/install/installer/cmd/testdata/render/http-proxy/output.golden @@ -4979,6 +4979,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -7519,7 +7520,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f39e45d788bb170ba728cfc3c69c5ca00a2b88a73a2d7288bcc1caee8e638405 + gitpod.io/checksum_config: c6d2a80d1de7a1f91bb2bea177c784c3bc7d4cd50a3e761795fad1188e41f831 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/insecure-s3-setup/output.golden b/install/installer/cmd/testdata/render/insecure-s3-setup/output.golden index 784db997d69493..b427da4f4e1d07 100644 --- a/install/installer/cmd/testdata/render/insecure-s3-setup/output.golden +++ b/install/installer/cmd/testdata/render/insecure-s3-setup/output.golden @@ -4891,6 +4891,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -7145,7 +7146,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: 707e266a9ce1d118c63137dfeee9740d7577326c2c6fa509b2f79b6a2a02aa47 + gitpod.io/checksum_config: 363bfc94ab8fd9188de7d28d665d6befe9ce521d4cc93ea6169e4b7985574108 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/minimal/output.golden b/install/installer/cmd/testdata/render/minimal/output.golden index a02ad5fbd5ab28..d5de3f062c5035 100644 --- a/install/installer/cmd/testdata/render/minimal/output.golden +++ b/install/installer/cmd/testdata/render/minimal/output.golden @@ -4976,6 +4976,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -7275,7 +7276,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f39e45d788bb170ba728cfc3c69c5ca00a2b88a73a2d7288bcc1caee8e638405 + gitpod.io/checksum_config: c6d2a80d1de7a1f91bb2bea177c784c3bc7d4cd50a3e761795fad1188e41f831 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/statefulset-customization/output.golden b/install/installer/cmd/testdata/render/statefulset-customization/output.golden index dfe277a056544b..22d0f08b232d31 100644 --- a/install/installer/cmd/testdata/render/statefulset-customization/output.golden +++ b/install/installer/cmd/testdata/render/statefulset-customization/output.golden @@ -4988,6 +4988,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -7287,7 +7288,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f39e45d788bb170ba728cfc3c69c5ca00a2b88a73a2d7288bcc1caee8e638405 + gitpod.io/checksum_config: c6d2a80d1de7a1f91bb2bea177c784c3bc7d4cd50a3e761795fad1188e41f831 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden b/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden index 1a93cfb87ed26e..59e44868d63bba 100644 --- a/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden +++ b/install/installer/cmd/testdata/render/use-pod-security-policies/output.golden @@ -5309,6 +5309,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -7719,7 +7720,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f39e45d788bb170ba728cfc3c69c5ca00a2b88a73a2d7288bcc1caee8e638405 + gitpod.io/checksum_config: c6d2a80d1de7a1f91bb2bea177c784c3bc7d4cd50a3e761795fad1188e41f831 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden b/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden index b80bc1e66252d4..e3faa72a9701d3 100644 --- a/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden +++ b/install/installer/cmd/testdata/render/workspace-requests-limits/output.golden @@ -4979,6 +4979,7 @@ data: "procLimit": 0, "netlimit": { "enabled": false, + "enforce": false, "connectionsPerMinute": 3000, "bucketSize": 1000 }, @@ -7278,7 +7279,7 @@ spec: template: metadata: annotations: - gitpod.io/checksum_config: f39e45d788bb170ba728cfc3c69c5ca00a2b88a73a2d7288bcc1caee8e638405 + gitpod.io/checksum_config: c6d2a80d1de7a1f91bb2bea177c784c3bc7d4cd50a3e761795fad1188e41f831 seccomp.security.alpha.kubernetes.io/shiftfs-module-loader: unconfined creationTimestamp: null labels: diff --git a/install/installer/pkg/components/ws-daemon/configmap.go b/install/installer/pkg/components/ws-daemon/configmap.go index b9f7d54fe5405c..7c82b6d276319d 100644 --- a/install/installer/pkg/components/ws-daemon/configmap.go +++ b/install/installer/pkg/components/ws-daemon/configmap.go @@ -50,6 +50,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { var procLimit int64 networkLimitConfig := netlimit.Config{ Enabled: false, + Enforce: false, ConnectionsPerMinute: 3000, BucketSize: 1000, } @@ -74,6 +75,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) { ioLimitConfig.ReadIOPS = ucfg.Workspace.IOLimits.ReadIOPS networkLimitConfig.Enabled = ucfg.Workspace.NetworkLimits.Enabled + networkLimitConfig.Enforce = ucfg.Workspace.NetworkLimits.Enforce networkLimitConfig.ConnectionsPerMinute = ucfg.Workspace.NetworkLimits.ConnectionsPerMinute networkLimitConfig.BucketSize = ucfg.Workspace.NetworkLimits.BucketSize diff --git a/install/installer/pkg/config/v1/experimental/experimental.go b/install/installer/pkg/config/v1/experimental/experimental.go index 351f2736287e6d..ac5db245a39930 100644 --- a/install/installer/pkg/config/v1/experimental/experimental.go +++ b/install/installer/pkg/config/v1/experimental/experimental.go @@ -77,6 +77,7 @@ type WorkspaceConfig struct { } `json:"ioLimits"` NetworkLimits struct { Enabled bool `json:"enabled"` + Enforce bool `json:"enforce"` ConnectionsPerMinute int64 `json:"connectionsPerMinute"` BucketSize int64 `json:"bucketSize"` } `json:"networkLimits"`