initial commit

Author: dashpole

Diff for: LICENSE

@@ -0,0 +1,43 @@
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GO := go
+pkgs  = $(shell $(GO) list ./... | grep -v vendor)
+
+all: presubmit build test
+
+test:
+	@echo ">> running tests"
+	@$(GO) test -short -race $(pkgs)
+
+format:
+	@echo ">> formatting code"
+	@$(GO) fmt $(pkgs)
+
+vet:
+	@echo ">> vetting code"
+	@$(GO) vet $(pkgs)
+
+build:
+	@echo ">> building binaries"
+	@./build/build.sh
+
+docker:
+	@docker build -t cadvisor:$(shell git rev-parse --short HEAD) -f deploy/Dockerfile .
+
+presubmit: vet
+	@echo ">> checking go formatting"
+	@./build/check_gofmt.sh
+
+.PHONY: all build docker format release test vet presubmit

Diff for: Makefile

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+GO_FLAGS=${GO_FLAGS:-"-tags netgo"}    # Extra go flags to use in the build.
+BUILD_USER=${BUILD_USER:-"${USER}@${HOSTNAME}"}
+BUILD_DATE=${BUILD_DATE:-$( date +%Y%m%d-%H:%M:%S )}
+VERBOSE=${VERBOSE:-}
+
+repo_path="github.com/dashpole/example-gpu-monitor/cmd"
+
+version=$( git describe --tags --dirty --abbrev=14 | sed -E 's/-([0-9]+)-g/.\1+/' )
+revision=$( git rev-parse --short HEAD 2> /dev/null || echo 'unknown' )
+branch=$( git rev-parse --abbrev-ref HEAD 2> /dev/null || echo 'unknown' )
+go_version=$( go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/' )
+
+ldflags="
+  -X ${repo_path}/version.Version${ldseparator}=${version}
+  -X ${repo_path}/version.Revision${ldseparator}=${revision}
+  -X ${repo_path}/version.Branch${ldseparator}=${branch}
+  -X ${repo_path}/version.BuildUser${ldseparator}=${BUILD_USER}
+  -X ${repo_path}/version.BuildDate${ldseparator}=${BUILD_DATE}
+  -X ${repo_path}/version.GoVersion${ldseparator}=${go_version}"
+
+echo ">> building monitor"
+
+if [ -n "$VERBOSE" ]; then
+  echo "Building with -ldflags $ldflags"
+fi
+
+GOBIN=$PWD go build -o monitor ${GO_FLAGS} -ldflags "${ldflags}" "${repo_path}"
+
+exit 0

Diff for: build/build.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Check formatting on non Godep'd code.
+GOFMT_PATHS=$(find . -not -wholename "*.git*" -not -wholename "*Godeps*" -not -wholename "*vendor*" -not -name "." -type d)
+
+# Find any files with gofmt problems
+BAD_FILES=$(gofmt -s -l $GOFMT_PATHS)
+
+if [ -n "$BAD_FILES" ]; then
+  echo "The following files are not properly formatted:"
+  echo $BAD_FILES
+  exit 1
+fi

Diff for: build/check_gofmt.sh

@@ -0,0 +1,40 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"github.com/golang/glog"
+
+	"github.com/dashpole/example-gpu-monitor/pkg/gpustats"
+	"github.com/dashpole/example-gpu-monitor/pkg/kubeletdevices"
+	"github.com/dashpole/example-gpu-monitor/pkg/metrics"
+)
+
+var (
+	socket             = flag.String("socket", "", "location of the kubelet's podresources service")
+	port               = flag.Int("port", 8080, "port on which to listen")
+	prometheusEndpoint = flag.String("prometheus_endpoint", "/metrics", "Endpoint to expose Prometheus metrics on")
+)
+
+func main() {
+	defer glog.Flush()
+	flag.Parse()
+
+	glog.V(1).Infof("Starting example-gpu-monitor")
+
+	metrics.Register(gpustats.NewGPUStatsProvider(), kubeletdevices.NewDeviceProvider())
+
+}

Diff for: cmd/main.go

@@ -0,0 +1,136 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gpustats
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/mindprince/gonvml"
+)
+
+type GPUStatsProvider interface {
+	GetStats(deviceID string) (*Stats, error)
+	Stop()
+}
+
+type Stats struct {
+	Model       string
+	ID          string
+	MemoryTotal uint64
+	MemoryUsed  uint64
+	DutyCycle   uint64
+}
+
+func NewGPUStatsProvider() GPUStatsProvider {
+	m := monitorImpl{
+		devices: make(map[int]gonvml.Device),
+	}
+	if err := gonvml.Initialize(); err != nil {
+		// This is under a logging level because otherwise we may cause
+		// log spam if the drivers/nvml is not installed on the system.
+		glog.V(4).Infof("Could not initialize NVML: %v", err)
+		return &m
+	}
+	m.nvmlInitialized = true
+	numDevices, err := gonvml.DeviceCount()
+	if err != nil {
+		glog.Warningf("GPU metrics would not be available. Failed to get the number of nvidia devices: %v", err)
+		return &m
+	}
+	glog.V(1).Infof("NVML initialized. Number of nvidia devices: %v", numDevices)
+	m.devices = make(map[int]gonvml.Device, numDevices)
+	for i := 0; i < int(numDevices); i++ {
+		device, err := gonvml.DeviceHandleByIndex(uint(i))
+		if err != nil {
+			glog.Warningf("Failed to get nvidia device handle %d: %v", i, err)
+			continue
+		}
+		minorNumber, err := device.MinorNumber()
+		if err != nil {
+			glog.Warningf("Failed to get nvidia device minor number: %v", err)
+			continue
+		}
+		m.devices[int(minorNumber)] = device
+	}
+	return &m
+}
+
+type monitorImpl struct {
+	nvmlInitialized bool
+
+	// map from device minor number to Device
+	devices map[int]gonvml.Device
+}
+
+func (m *monitorImpl) Stop() {
+	if m.nvmlInitialized {
+		gonvml.Shutdown()
+	}
+}
+
+// GetStats assumes the device id is nvidia[minor number]
+func (m *monitorImpl) GetStats(deviceId string) (*Stats, error) {
+	i, err := getMinorNumber(deviceId)
+	if err != nil {
+		return nil, fmt.Errorf("error getting device minor number from path %s: %v", deviceId, err)
+	}
+	device, found := m.devices[i]
+	if !found {
+		return nil, fmt.Errorf("device with minor number %d was not found", i)
+	}
+	model, err := device.Name()
+	if err != nil {
+		return nil, fmt.Errorf("error while getting gpu name: %v", err)
+	}
+	uuid, err := device.UUID()
+	if err != nil {
+		return nil, fmt.Errorf("error while getting gpu uuid: %v", err)
+	}
+	memoryTotal, memoryUsed, err := device.MemoryInfo()
+	if err != nil {
+		return nil, fmt.Errorf("error while getting gpu memory info: %v", err)
+	}
+	utilizationGPU, err := device.AverageGPUUtilization(10 * time.Second)
+	if err != nil {
+		return nil, fmt.Errorf("error while getting gpu utilization: %v", err)
+	}
+
+	return &Stats{
+		Model:       model,
+		ID:          uuid,
+		MemoryTotal: memoryTotal,
+		MemoryUsed:  memoryUsed,
+		DutyCycle:   uint64(utilizationGPU),
+	}, nil
+}
+
+var deviceExpr = regexp.MustCompile(`^nvidia([0-9]+)$`)
+
+func getMinorNumber(deviceId string) (int, error) {
+	matches := deviceExpr.FindStringSubmatch(deviceId)
+	if len(matches) != 2 {
+		return 0, fmt.Errorf("%s does not match nvidia[0-9]+", deviceId)
+	}
+	minorString := matches[1]
+	i, err := strconv.ParseInt(minorString, 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse %s, to an int: %v", minorString, err)
+	}
+	return int(i), nil
+}

Diff for: deploy/Dockerfile

@@ -0,0 +1,62 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubeletdevices
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
+	podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
+)
+
+const (
+	// defaultPodResourcesSocket is the path to the socket serving the podresources API.
+	// defaultPodResourcesSocket  = "unix:///var/lib/kubelet/pod-resources/kubelet.sock"
+	defaultPodResourcesSocket  = "unix:///var/lib/kubelet/kubelet.sock"
+	defaultPodResourcesTimeout = 10 * time.Second
+	defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb
+)
+
+type DeviceProvider interface {
+	GetDevices() (*podresourcesapi.ListPodResourcesResponse, error)
+}
+
+type deviceProvider struct {
+	client podresourcesapi.PodResourcesListerClient
+}
+
+func NewDeviceProvider() DeviceProvider {
+	client, _, err := podresources.GetClient(defaultPodResourcesSocket, defaultPodResourcesTimeout, defaultPodResourcesMaxSize)
+	if err != nil {
+		glog.Fatalf("Failed to get grpc client: %v", err)
+	}
+	return &deviceProvider{
+		client: client,
+	}
+}
+
+func (d *deviceProvider) GetDevices() (*podresourcesapi.ListPodResourcesResponse, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	resp, err := d.client.List(ctx, &podresourcesapi.ListPodResourcesRequest{})
+	if err != nil {
+		return nil, fmt.Errorf("%v.Get(_) = _, %v", d.client, err)
+	}
+	return resp, nil
+}