Skip to content

Commit 0d65762

Browse files
author
Matthew Wong
committed
Add volume operation metrics to operation executor and PV controller
1 parent bc1a58a commit 0d65762

File tree

10 files changed

+198
-94
lines changed

10 files changed

+198
-94
lines changed

pkg/controller/volume/persistentvolume/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ go_library(
2424
"//pkg/util/io:go_default_library",
2525
"//pkg/util/mount:go_default_library",
2626
"//pkg/volume:go_default_library",
27+
"//pkg/volume/util:go_default_library",
2728
"//vendor/github.com/golang/glog:go_default_library",
2829
"//vendor/k8s.io/api/core/v1:go_default_library",
2930
"//vendor/k8s.io/api/storage/v1:go_default_library",

pkg/controller/volume/persistentvolume/pv_controller.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
"k8s.io/kubernetes/pkg/util/goroutinemap"
4040
"k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff"
4141
vol "k8s.io/kubernetes/pkg/volume"
42+
"k8s.io/kubernetes/pkg/volume/util"
4243

4344
"github.com/golang/glog"
4445
)
@@ -1216,7 +1217,10 @@ func (ctrl *PersistentVolumeController) doDeleteVolume(volume *v1.PersistentVolu
12161217
return false, fmt.Errorf("Failed to create deleter for volume %q: %v", volume.Name, err)
12171218
}
12181219

1219-
if err = deleter.Delete(); err != nil {
1220+
opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_delete")
1221+
err = deleter.Delete()
1222+
opComplete(err)
1223+
if err != nil {
12201224
// Deleter failed
12211225
return false, err
12221226
}
@@ -1326,7 +1330,9 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
13261330
return
13271331
}
13281332

1333+
opComplete := util.OperationCompleteHook(plugin.GetPluginName(), "volume_provision")
13291334
volume, err = provisioner.Provision()
1335+
opComplete(err)
13301336
if err != nil {
13311337
strerr := fmt.Sprintf("Failed to provision volume with StorageClass %q: %v", storageClass.Name, err)
13321338
glog.V(2).Infof("failed to provision volume for claim %q with StorageClass %q: %v", claimToClaimKey(claim), storageClass.Name, err)

pkg/controller/volume/persistentvolume/pv_controller_base.go

+3
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import (
4242
"k8s.io/kubernetes/pkg/controller"
4343
"k8s.io/kubernetes/pkg/util/goroutinemap"
4444
vol "k8s.io/kubernetes/pkg/volume"
45+
"k8s.io/kubernetes/pkg/volume/util"
4546

4647
"github.com/golang/glog"
4748
)
@@ -74,6 +75,8 @@ func NewController(p ControllerParameters) (*PersistentVolumeController, error)
7475
eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "persistentvolume-controller"})
7576
}
7677

78+
util.RegisterMetrics()
79+
7780
controller := &PersistentVolumeController{
7881
volumes: newPersistentVolumeOrderedIndex(),
7982
claims: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc),

pkg/volume/util/BUILD

+2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ go_library(
1515
"doc.go",
1616
"fs_unsupported.go",
1717
"io_util.go",
18+
"metrics.go",
1819
"util.go",
1920
] + select({
2021
"@io_bazel_rules_go//go/platform:darwin_amd64": [
@@ -31,6 +32,7 @@ go_library(
3132
"//pkg/api/v1/helper:go_default_library",
3233
"//pkg/util/mount:go_default_library",
3334
"//vendor/github.com/golang/glog:go_default_library",
35+
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
3436
"//vendor/k8s.io/api/core/v1:go_default_library",
3537
"//vendor/k8s.io/api/storage/v1:go_default_library",
3638
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",

pkg/volume/util/metrics.go

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
Copyright 2017 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package util
18+
19+
import (
20+
"sync"
21+
"time"
22+
23+
"github.com/prometheus/client_golang/prometheus"
24+
)
25+
26+
var StorageOperationMetric = prometheus.NewHistogramVec(
27+
prometheus.HistogramOpts{
28+
Name: "storage_operation_duration_seconds",
29+
Help: "Storage operation duration",
30+
},
31+
[]string{"volume_plugin", "operation_name"},
32+
)
33+
34+
var StorageOperationErrorMetric = prometheus.NewCounterVec(
35+
prometheus.CounterOpts{
36+
Name: "storage_operation_errors_total",
37+
Help: "Storage operation errors",
38+
},
39+
[]string{"volume_plugin", "operation_name"},
40+
)
41+
42+
var registerMetrics sync.Once
43+
44+
func RegisterMetrics() {
45+
registerMetrics.Do(func() {
46+
prometheus.MustRegister(StorageOperationMetric)
47+
prometheus.MustRegister(StorageOperationErrorMetric)
48+
})
49+
}
50+
51+
// OperationCompleteHook returns a hook to call when an operation is completed
52+
func OperationCompleteHook(plugin, operationName string) func(error) {
53+
requestTime := time.Now()
54+
opComplete := func(err error) {
55+
timeTaken := time.Since(requestTime).Seconds()
56+
// Create metric with operation name and plugin name
57+
if err != nil {
58+
StorageOperationErrorMetric.WithLabelValues(plugin, operationName).Inc()
59+
} else {
60+
StorageOperationMetric.WithLabelValues(plugin, operationName).Observe(timeTaken)
61+
}
62+
}
63+
return opComplete
64+
}

pkg/volume/util/nestedpendingoperations/nestedpendingoperations.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ type NestedPendingOperations interface {
5555
// concatenation of volumeName and podName is removed from the list of
5656
// executing operations allowing a new operation to be started with the
5757
// volumeName without error.
58-
Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error) error
58+
Run(volumeName v1.UniqueVolumeName, podName types.UniquePodName, operationFunc func() error, operationCompleteFunc func(error)) error
5959

6060
// Wait blocks until all operations are completed. This is typically
6161
// necessary during tests - the test should wait until all operations finish
@@ -94,7 +94,8 @@ type operation struct {
9494
func (grm *nestedPendingOperations) Run(
9595
volumeName v1.UniqueVolumeName,
9696
podName types.UniquePodName,
97-
operationFunc func() error) error {
97+
operationFunc func() error,
98+
operationCompleteFunc func(error)) error {
9899
grm.lock.Lock()
99100
defer grm.lock.Unlock()
100101
opExists, previousOpIndex := grm.isOperationExists(volumeName, podName)
@@ -132,6 +133,7 @@ func (grm *nestedPendingOperations) Run(
132133
defer k8sRuntime.HandleCrash()
133134
// Handle completion of and error, if any, from operationFunc()
134135
defer grm.operationComplete(volumeName, podName, &err)
136+
defer operationCompleteFunc(err)
135137
// Handle panic, if any, from operationFunc()
136138
defer k8sRuntime.RecoverFromPanic(&err)
137139
return operationFunc()

0 commit comments

Comments
 (0)