Skip to content

Commit da8dc43

Browse files
committed
Add flagz implementation and enablement in apiserver
1 parent 60651eb commit da8dc43

File tree

16 files changed

+445
-9
lines changed

16 files changed

+445
-9
lines changed

cmd/kube-apiserver/app/options/completion.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,14 @@ import (
2626
_ "k8s.io/component-base/metrics/prometheus/workqueue"
2727
netutils "k8s.io/utils/net"
2828

29-
controlplane "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
29+
cp "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
3030
"k8s.io/kubernetes/pkg/kubeapiserver"
3131
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
3232
)
3333

3434
// completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
3535
type completedOptions struct {
36-
controlplane.CompletedOptions
36+
cp.CompletedOptions
3737
CloudProvider *kubeoptions.CloudProviderOptions
3838

3939
Extra
@@ -57,7 +57,7 @@ func (s *ServerRunOptions) Complete(ctx context.Context) (CompletedOptions, erro
5757
if err != nil {
5858
return CompletedOptions{}, err
5959
}
60-
controlplane, err := s.Options.Complete(ctx, []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
60+
controlplane, err := s.Options.Complete(ctx, s.Flags(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP})
6161
if err != nil {
6262
return CompletedOptions{}, err
6363
}
@@ -107,7 +107,7 @@ func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, ne
107107
// nothing provided by user, use default range (only applies to the Primary)
108108
if len(serviceClusterIPRangeList) == 0 {
109109
var primaryServiceClusterCIDR net.IPNet
110-
primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(primaryServiceClusterCIDR)
110+
primaryServiceIPRange, apiServerServiceIP, err = cp.ServiceIPRange(primaryServiceClusterCIDR)
111111
if err != nil {
112112
return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges: %v", err)
113113
}
@@ -119,7 +119,7 @@ func getServiceIPAndRanges(serviceClusterIPRanges string) (net.IP, net.IPNet, ne
119119
return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("service-cluster-ip-range[0] is not a valid cidr")
120120
}
121121

122-
primaryServiceIPRange, apiServerServiceIP, err = controlplane.ServiceIPRange(*primaryServiceClusterCIDR)
122+
primaryServiceIPRange, apiServerServiceIP, err = cp.ServiceIPRange(*primaryServiceClusterCIDR)
123123
if err != nil {
124124
return net.IP{}, net.IPNet{}, net.IPNet{}, fmt.Errorf("error determining service IP ranges for primary service cidr: %v", err)
125125
}

pkg/controlplane/apiserver/config.go

+1
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ func BuildGenericConfig(
119119
lastErr error,
120120
) {
121121
genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
122+
genericConfig.Flagz = s.Flagz
122123
genericConfig.MergedResourceConfig = resourceConfig
123124

124125
if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {

pkg/controlplane/apiserver/config_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"k8s.io/apimachinery/pkg/runtime"
2626
"k8s.io/apimachinery/pkg/runtime/schema"
2727
apiserveroptions "k8s.io/apiserver/pkg/server/options"
28+
cliflag "k8s.io/component-base/cli/flag"
2829
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
2930
"k8s.io/kubernetes/pkg/api/legacyscheme"
3031
"k8s.io/kubernetes/pkg/controlplane/apiserver/options"
@@ -46,7 +47,7 @@ func TestBuildGenericConfig(t *testing.T) {
4647
s.BindPort = ln.Addr().(*net.TCPAddr).Port
4748
opts.SecureServing = s
4849

49-
completedOptions, err := opts.Complete(context.TODO(), nil, nil)
50+
completedOptions, err := opts.Complete(context.TODO(), cliflag.NamedFlagSets{}, nil, nil)
5051
if err != nil {
5152
t.Fatalf("Failed to complete apiserver options: %v", err)
5253
}

pkg/controlplane/apiserver/options/options.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"k8s.io/component-base/logs"
3535
logsapi "k8s.io/component-base/logs/api/v1"
3636
"k8s.io/component-base/metrics"
37+
"k8s.io/component-base/zpages/flagz"
3738
"k8s.io/klog/v2"
3839
netutil "k8s.io/utils/net"
3940

@@ -47,6 +48,7 @@ import (
4748
// Options define the flags and validation for a generic controlplane. If the
4849
// structs are nil, the options are not added to the command line and not validated.
4950
type Options struct {
51+
Flagz flagz.Reader
5052
GenericServerRunOptions *genericoptions.ServerRunOptions
5153
Etcd *genericoptions.EtcdOptions
5254
SecureServing *genericoptions.SecureServingOptionsWithLoopback
@@ -201,7 +203,7 @@ func (s *Options) AddFlags(fss *cliflag.NamedFlagSets) {
201203
"Path to socket where a external JWT signer is listening. This flag is mutually exclusive with --service-account-signing-key-file and --service-account-key-file. Requires enabling feature gate (ExternalServiceAccountTokenSigner)")
202204
}
203205

204-
func (o *Options) Complete(ctx context.Context, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
206+
func (o *Options) Complete(ctx context.Context, fss cliflag.NamedFlagSets, alternateDNS []string, alternateIPs []net.IP) (CompletedOptions, error) {
205207
if o == nil {
206208
return CompletedOptions{completedOptions: &completedOptions{}}, nil
207209
}
@@ -257,6 +259,8 @@ func (o *Options) Complete(ctx context.Context, alternateDNS []string, alternate
257259
}
258260
}
259261

262+
completed.Flagz = flagz.NamedFlagSetsReader{FlagSets: fss}
263+
260264
return CompletedOptions{
261265
completedOptions: &completed,
262266
}, nil

pkg/controlplane/apiserver/samples/generic/server/server.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"path/filepath"
2525

2626
"github.com/spf13/cobra"
27+
"github.com/spf13/pflag"
2728
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
2829
utilerrors "k8s.io/apimachinery/pkg/util/errors"
2930
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -85,7 +86,7 @@ APIs.`,
8586

8687
ctx := genericapiserver.SetupSignalContext()
8788

88-
completedOptions, err := s.Complete(ctx, []string{}, []net.IP{})
89+
completedOptions, err := s.Complete(ctx, cliflag.NamedFlagSets{FlagSets: map[string]*pflag.FlagSet{"sample_generic_controlplane": fs}}, []string{}, []net.IP{})
8990
if err != nil {
9091
return err
9192
}

pkg/controlplane/apiserver/samples/generic/server/testing/testserver.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ func StartTestServer(t ktesting.TB, instanceOptions *TestServerInstanceOptions,
164164
o.Authentication.ServiceAccounts.Issuers = []string{"https://foo.bar.example.com"}
165165
o.Authentication.ServiceAccounts.KeyFiles = []string{saSigningKeyFile.Name()}
166166

167-
completedOptions, err := o.Complete(tCtx, nil, nil)
167+
completedOptions, err := o.Complete(tCtx, fss, nil, nil)
168168
if err != nil {
169169
return result, fmt.Errorf("failed to set default ServerRunOptions: %w", err)
170170
}

pkg/controlplane/apiserver/server.go

+7
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737
clientgoinformers "k8s.io/client-go/informers"
3838
"k8s.io/client-go/kubernetes"
3939
zpagesfeatures "k8s.io/component-base/zpages/features"
40+
"k8s.io/component-base/zpages/flagz"
4041
"k8s.io/component-base/zpages/statusz"
4142
"k8s.io/component-helpers/apimachinery/lease"
4243
"k8s.io/klog/v2"
@@ -153,6 +154,12 @@ func (c completedConfig) New(name string, delegationTarget genericapiserver.Dele
153154
return nil, fmt.Errorf("failed to get listener address: %w", err)
154155
}
155156

157+
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
158+
if c.Generic.Flagz != nil {
159+
flagz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, c.Generic.Flagz)
160+
}
161+
}
162+
156163
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
157164
statusz.Install(s.GenericAPIServer.Handler.NonGoRestfulMux, name, statusz.NewRegistry())
158165
}

pkg/features/versioned_kube_features.go

+4
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
803803
{Version: version.MustParse("1.26"), Default: true, PreRelease: featuregate.Alpha},
804804
},
805805

806+
zpagesfeatures.ComponentFlagz: {
807+
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
808+
},
809+
806810
zpagesfeatures.ComponentStatusz: {
807811
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
808812
},

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

+4
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,10 @@ func ClusterRoles() []rbacv1.ClusterRole {
203203
).RuleOrDie(),
204204
}
205205

206+
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentFlagz) {
207+
monitoringRules = append(monitoringRules, rbacv1helpers.NewRule("get").URLs("/flagz").RuleOrDie())
208+
}
209+
206210
if utilfeature.DefaultFeatureGate.Enabled(zpagesfeatures.ComponentStatusz) {
207211
monitoringRules = append(monitoringRules, rbacv1helpers.NewRule("get").URLs("/statusz").RuleOrDie())
208212
}

staging/src/k8s.io/apiserver/pkg/server/config.go

+2
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ import (
7979
"k8s.io/component-base/metrics/prometheus/slis"
8080
"k8s.io/component-base/tracing"
8181
utilversion "k8s.io/component-base/version"
82+
"k8s.io/component-base/zpages/flagz"
8283
"k8s.io/klog/v2"
8384
openapicommon "k8s.io/kube-openapi/pkg/common"
8485
"k8s.io/kube-openapi/pkg/spec3"
@@ -189,6 +190,7 @@ type Config struct {
189190
LivezChecks []healthz.HealthChecker
190191
// The default set of readyz-only checks. There might be more added via AddReadyzChecks dynamically.
191192
ReadyzChecks []healthz.HealthChecker
193+
Flagz flagz.Reader
192194
// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
193195
// to InstallLegacyAPIGroup. New API servers don't generally have legacy groups at all.
194196
LegacyAPIGroupPrefixes sets.String

staging/src/k8s.io/component-base/zpages/features/kube_features.go

+7
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ import (
2222
)
2323

2424
const (
25+
// owner: @richabanker
26+
// kep: https://kep.k8s.io/4828
27+
ComponentFlagz featuregate.Feature = "ComponentFlagz"
28+
2529
// owner: @richabanker
2630
// kep: https://kep.k8s.io/4827
2731
// alpha: v1.32
@@ -33,6 +37,9 @@ const (
3337

3438
func featureGates() map[featuregate.Feature]featuregate.VersionedSpecs {
3539
return map[featuregate.Feature]featuregate.VersionedSpecs{
40+
ComponentFlagz: {
41+
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
42+
},
3643
ComponentStatusz: {
3744
{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
3845
},
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package flagz
18+
19+
import (
20+
"github.com/spf13/pflag"
21+
cliflag "k8s.io/component-base/cli/flag"
22+
)
23+
24+
type Reader interface {
25+
GetFlagz() map[string]string
26+
}
27+
28+
// NamedFlagSetsGetter implements Reader for cliflag.NamedFlagSets
29+
type NamedFlagSetsReader struct {
30+
FlagSets cliflag.NamedFlagSets
31+
}
32+
33+
func (n NamedFlagSetsReader) GetFlagz() map[string]string {
34+
return convertNamedFlagSetToFlags(&n.FlagSets)
35+
}
36+
37+
func convertNamedFlagSetToFlags(flagSets *cliflag.NamedFlagSets) map[string]string {
38+
flags := make(map[string]string)
39+
for _, fs := range flagSets.FlagSets {
40+
fs.VisitAll(func(flag *pflag.Flag) {
41+
if flag.Value != nil {
42+
value := flag.Value.String()
43+
if set, ok := flag.Annotations["classified"]; ok && len(set) > 0 {
44+
value = "CLASSIFIED"
45+
}
46+
flags[flag.Name] = value
47+
}
48+
})
49+
}
50+
51+
return flags
52+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package flagz
18+
19+
import (
20+
"reflect"
21+
"testing"
22+
23+
"github.com/spf13/pflag"
24+
25+
"k8s.io/component-base/cli/flag"
26+
)
27+
28+
func TestConvertNamedFlagSetToFlags(t *testing.T) {
29+
tests := []struct {
30+
name string
31+
flagSets *flag.NamedFlagSets
32+
want map[string]string
33+
}{
34+
{
35+
name: "basic flags",
36+
flagSets: &flag.NamedFlagSets{
37+
FlagSets: map[string]*pflag.FlagSet{
38+
"test": flagSet(t, map[string]flagValue{
39+
"flag1": {value: "value1", sensitive: false},
40+
"flag2": {value: "value2", sensitive: false},
41+
}),
42+
},
43+
},
44+
want: map[string]string{
45+
"flag1": "value1",
46+
"flag2": "value2",
47+
},
48+
},
49+
{
50+
name: "classified flags",
51+
flagSets: &flag.NamedFlagSets{
52+
FlagSets: map[string]*pflag.FlagSet{
53+
"test": flagSet(t, map[string]flagValue{
54+
"secret1": {value: "value1", sensitive: true},
55+
"flag2": {value: "value2", sensitive: false},
56+
}),
57+
},
58+
},
59+
want: map[string]string{
60+
"flag2": "value2",
61+
"secret1": "CLASSIFIED",
62+
},
63+
},
64+
}
65+
66+
for _, tt := range tests {
67+
t.Run(tt.name, func(t *testing.T) {
68+
got := convertNamedFlagSetToFlags(tt.flagSets)
69+
if !reflect.DeepEqual(got, tt.want) {
70+
t.Errorf("ConvertNamedFlagSetToFlags() = %v, want %v", got, tt.want)
71+
}
72+
})
73+
}
74+
}
75+
76+
type flagValue struct {
77+
value string
78+
sensitive bool
79+
}
80+
81+
func flagSet(t *testing.T, flags map[string]flagValue) *pflag.FlagSet {
82+
fs := pflag.NewFlagSet("test-set", pflag.ContinueOnError)
83+
for flagName, flagVal := range flags {
84+
flagValue := ""
85+
fs.StringVar(&flagValue, flagName, flagVal.value, "test-usage")
86+
if flagVal.sensitive {
87+
err := fs.SetAnnotation(flagName, "classified", []string{"true"})
88+
if err != nil {
89+
t.Fatalf("unexpected error when setting flag annotation: %v", err)
90+
}
91+
}
92+
}
93+
94+
return fs
95+
}

0 commit comments

Comments
 (0)