Implement API usage metrics for gce storage

This PR implements cloudprovider storage metrics for gce. A sample of metrics look like: gce_instance_list_bucket{kube_namespace="hekumar-gce-metrics2-master",le="100"} 0 gce_instance_list_bucket{kube_namespace="hekumar-gce-metrics2-master",le="200"} 1 gce_instance_list_bucket{kube_namespace="hekumar-gce-metrics2-master",le="25600"} 1 gce_instance_list_bucket{kube_namespace="hekumar-gce-metrics2-master",le="51200"} 1 gce_instance_list_sum{kube_namespace="hekumar-gce-metrics2-master"} 122 gce_instance_list_count{kube_namespace="hekumar-gce-metrics2-master"} 1
kubernetes · Mar 21, 2017 · 76392ee · 76392ee
1 parent 4974a05
commit 76392ee
Show file tree

Hide file tree

Showing 3 changed files with 143 additions and 8 deletions.
diff --git a/pkg/cloudprovider/providers/gce/BUILD b/pkg/cloudprovider/providers/gce/BUILD
@@ -13,6 +13,7 @@ go_library(
     srcs = [
         "doc.go",
         "gce.go",
+        "gce_metrics.go",
         "token_source.go",
     ],
     tags = ["automanaged"],
@@ -25,10 +26,12 @@ go_library(
         "//vendor:cloud.google.com/go/compute/metadata",
         "//vendor:github.com/golang/glog",
         "//vendor:github.com/prometheus/client_golang/prometheus",
+        "//vendor:golang.org/x/net/context",
         "//vendor:golang.org/x/oauth2",
         "//vendor:golang.org/x/oauth2/google",
         "//vendor:google.golang.org/api/compute/v1",
         "//vendor:google.golang.org/api/container/v1",
+        "//vendor:google.golang.org/api/gensupport",
         "//vendor:google.golang.org/api/googleapi",
         "//vendor:gopkg.in/gcfg.v1",
         "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",

diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go
@@ -30,6 +30,8 @@ import (
 
 	"gopkg.in/gcfg.v1"
 
+	"golang.org/x/net/context"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -48,6 +50,7 @@ import (
 	"golang.org/x/oauth2/google"
 	compute "google.golang.org/api/compute/v1"
 	container "google.golang.org/api/container/v1"
+	"google.golang.org/api/gensupport"
 	"google.golang.org/api/googleapi"
 )
 
@@ -113,6 +116,12 @@ type Config struct {
 
 type DiskType string
 
+// ApiWithNamespace stores api and namespace in context
+type apiWithNamespace struct {
+	namespace string
+	apiCall   string
+}
+
 const (
 	DiskTypeSSD      = "pd-ssd"
 	DiskTypeStandard = "pd-standard"
@@ -153,7 +162,27 @@ type Disks interface {
 }
 
 func init() {
+	registerMetrics()
 	cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { return newGCECloud(config) })
+	gensupport.RegisterHook(trackAPILatency)
+}
+
+func trackAPILatency(ctx context.Context, req *http.Request) func(resp *http.Response) {
+	requestTime := time.Now()
+	t := ctx.Value("kube-api-namespace")
+	apiNamespace, ok := t.(apiWithNamespace)
+
+	if !ok {
+		return nil
+	}
+
+	apiResponseReceived := func(resp *http.Response) {
+		timeTaken := SinceInMilliSeconds(requestTime)
+		if mi, ok := gceMetricMap[apiNamespace.apiCall]; ok {
+			mi.WithLabelValues(apiNamespace.namespace).Observe(timeTaken)
+		}
+	}
+	return apiResponseReceived
 }
 
 // Raw access to the underlying GCE service, probably should only be used for e2e tests
@@ -530,7 +559,8 @@ func (gce *GCECloud) waitForZoneOp(op *compute.Operation, zone string) error {
 // GetLoadBalancer is an implementation of LoadBalancer.GetLoadBalancer
 func (gce *GCECloud) GetLoadBalancer(clusterName string, service *v1.Service) (*v1.LoadBalancerStatus, bool, error) {
 	loadBalancerName := cloudprovider.GetLoadBalancerName(service)
-	fwd, err := gce.service.ForwardingRules.Get(gce.projectID, gce.region, loadBalancerName).Do()
+	fwd, err := gce.service.ForwardingRules.
+		Get(gce.projectID, gce.region, loadBalancerName).Do()
 	if err == nil {
 		status := &v1.LoadBalancerStatus{}
 		status.Ingress = []v1.LoadBalancerIngress{{IP: fwd.IPAddress}}
@@ -548,6 +578,15 @@ func isHTTPErrorCode(err error, code int) bool {
 	return ok && apiErr.Code == code
 }
 
+func contextWithNamespace(apiCall string) context.Context {
+	rootContext := context.Background()
+	apiNamespace := apiWithNamespace{
+		namespace: "",
+		apiCall:   apiCall,
+	}
+	return context.WithValue(rootContext, "kube-api-namespace", apiNamespace)
+}
+
 func nodeNames(nodes []*v1.Node) []string {
 	ret := make([]string, len(nodes))
 	for i, node := range nodes {
@@ -900,7 +939,8 @@ func (gce *GCECloud) ensureHttpHealthCheck(name, path string, port int32) (hc *c
 // Returns whether the forwarding rule exists, whether it needs to be updated,
 // what its IP address is (if it exists), and any error we encountered.
 func (gce *GCECloud) forwardingRuleNeedsUpdate(name, region string, loadBalancerIP string, ports []v1.ServicePort) (exists bool, needsUpdate bool, ipAddress string, err error) {
-	fwd, err := gce.service.ForwardingRules.Get(gce.projectID, region, name).Do()
+	fwd, err := gce.service.ForwardingRules.
+		Get(gce.projectID, region, name).Do()
 	if err != nil {
 		if isHTTPErrorCode(err, http.StatusNotFound) {
 			return false, true, "", nil
@@ -2479,7 +2519,9 @@ func (gce *GCECloud) CreateDisk(name string, diskType string, zone string, sizeG
 		Type:        diskTypeUri,
 	}
 
-	createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Do()
+	dc := contextWithNamespace("gce_disk_insert")
+
+	createOp, err := gce.service.Disks.Insert(gce.projectID, zone, diskToCreate).Context(dc).Do()
 	if err != nil {
 		return err
 	}
@@ -2498,7 +2540,9 @@ func (gce *GCECloud) doDeleteDisk(diskToDelete string) error {
 		return err
 	}
 
-	deleteOp, err := gce.service.Disks.Delete(gce.projectID, disk.Zone, disk.Name).Do()
+	dc := contextWithNamespace("gce_disk_delete")
+	deleteOp, err := gce.service.Disks.
+		Delete(gce.projectID, disk.Zone, disk.Name).Context(dc).Do()
 	if err != nil {
 		return err
 	}
@@ -2597,7 +2641,8 @@ func (gce *GCECloud) AttachDisk(diskName string, nodeName types.NodeName, readOn
 	}
 	attachedDisk := gce.convertDiskToAttachedDisk(disk, readWrite)
 
-	attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instance.Name, attachedDisk).Do()
+	dc := contextWithNamespace("gce_attach_disk")
+	attachOp, err := gce.service.Instances.AttachDisk(gce.projectID, disk.Zone, instance.Name, attachedDisk).Context(dc).Do()
 	if err != nil {
 		return err
 	}
@@ -2621,7 +2666,10 @@ func (gce *GCECloud) DetachDisk(devicePath string, nodeName types.NodeName) erro
 		return fmt.Errorf("error getting instance %q", instanceName)
 	}
 
-	detachOp, err := gce.service.Instances.DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Do()
+	dc := contextWithNamespace("gce_detach_disk")
+
+	detachOp, err := gce.service.Instances.
+		DetachDisk(gce.projectID, inst.Zone, inst.Name, devicePath).Context(dc).Do()
 	if err != nil {
 		return err
 	}
@@ -2690,7 +2738,9 @@ func (gce *GCECloud) DisksAreAttached(diskNames []string, nodeName types.NodeNam
 // Returns a gceDisk for the disk, if it is found in the specified zone.
 // If not found, returns (nil, nil)
 func (gce *GCECloud) findDiskByName(diskName string, zone string) (*gceDisk, error) {
-	disk, err := gce.service.Disks.Get(gce.projectID, zone, diskName).Do()
+	dc := contextWithNamespace("gce_list_disk")
+	disk, err := gce.service.Disks.
+		Get(gce.projectID, zone, diskName).Context(dc).Do()
 	if err == nil {
 		d := &gceDisk{
 			Zone: lastComponent(disk.Zone),
@@ -2902,7 +2952,8 @@ func (gce *GCECloud) getInstanceByName(name string) (*gceInstance, error) {
 	// Avoid changing behaviour when not managing multiple zones
 	for _, zone := range gce.managedZones {
 		name = canonicalizeInstanceName(name)
-		res, err := gce.service.Instances.Get(gce.projectID, zone, name).Do()
+		dc := contextWithNamespace("gce_instance_list")
+		res, err := gce.service.Instances.Get(gce.projectID, zone, name).Context(dc).Do()
 		if err != nil {
 			glog.Errorf("getInstanceByName: failed to get instance %s; err: %v", name, err)
 			if isHTTPErrorCode(err, http.StatusNotFound) {
@@ -2931,3 +2982,8 @@ func lastComponent(s string) string {
 	}
 	return s
 }
+
+// Gets the time since the specified start in Milliseconds
+func SinceInMilliSeconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds() / time.Millisecond.Nanoseconds())
+}
diff --git a/pkg/cloudprovider/providers/gce/gce_metrics.go b/pkg/cloudprovider/providers/gce/gce_metrics.go
@@ -0,0 +1,76 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var gceMetricMap = map[string]*prometheus.HistogramVec{
+	"gce_instance_list": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "gce_instance_list",
+			Help:    "Latency of instance listing calls in ms",
+			Buckets: prometheus.ExponentialBuckets(100, 2, 10),
+		},
+		[]string{"kube_namespace"},
+	),
+	"gce_disk_insert": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "gce_disk_insert",
+			Help:    "Latency of disk insert calls in ms",
+			Buckets: prometheus.ExponentialBuckets(100, 2, 10),
+		},
+		[]string{"kube_namespace"},
+	),
+	"gce_disk_delete": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "gce_disk_delete",
+			Help:    "Latency of disk delete calls in ms",
+			Buckets: prometheus.ExponentialBuckets(100, 2, 10),
+		},
+		[]string{"kube_namespace"},
+	),
+	"gce_attach_disk": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "gce_attach_disk",
+			Help:    "Latency of attach disk calls in ms",
+			Buckets: prometheus.ExponentialBuckets(100, 2, 10),
+		},
+		[]string{"kube_namespace"},
+	),
+	"gce_detach_disk": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "gce_detach_disk",
+			Help:    "Latency of detach disk calls in ms",
+			Buckets: prometheus.ExponentialBuckets(100, 2, 10),
+		},
+		[]string{"kube_namespace"},
+	),
+	"gce_list_disk": prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "gce_list_disk",
+			Help:    "Latency of list disk calls in ms",
+			Buckets: prometheus.ExponentialBuckets(100, 2, 10),
+		},
+		[]string{"kube_namespace"},
+	),
+}
+
+func registerMetrics() {
+	for _, metric := range gceMetricMap {
+		prometheus.MustRegister(metric)
+	}
+}