Merge pull request #9793 from smarterclayton/protobuf

Merged by openshift-bot

Author: OpenShift Bot

Godeps/Godeps.json

@@ -52,6 +52,7 @@ rsync -av \
 /test/fixtures/***
 /test/integration/***
 /third_party/golang/***
+/third_party/protobuf/***
 /README.md
 EOF
 

hack/copy-kube-artifacts.sh

@@ -19,7 +19,7 @@ if [[ ! "$clientgen" ]]; then
     echo "It looks as if you don't have a compiled client-gen binary"
     echo
     echo "If you are running from a clone of the git repo, please run"
-    echo "'./hack/build-go.sh Godeps/_workspace/src/k8s.io/kubernetes/cmd/libs/go2idl/client-gen'."
+    echo "'./hack/build-go.sh vendor/k8s.io/kubernetes/cmd/libs/go2idl/client-gen'."
   } >&2
   exit 1
 fi

hack/update-generated-clientsets.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+OS_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${OS_ROOT}/hack/lib/init.sh"
+
+# Go to the top of the tree.
+cd "${OS_ROOT}"
+
+if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
+  echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
+  echo "install the platform appropriate Protobuf package for your OS: "
+  echo
+  echo "  https://github.com/google/protobuf/releases"
+  echo
+  exit 1
+fi
+
+os::build::setup_env
+
+hack/build-go.sh tools/genprotobuf vendor/k8s.io/kubernetes/cmd/libs/go2idl/go-to-protobuf/protoc-gen-gogo
+genprotobuf="$( os::build::find-binary genprotobuf )"
+
+if [[ -z "${genprotobuf}" ]]; then
+	echo "It looks as if you don't have a compiled genprotobuf binary."
+	echo
+	echo "If you are running from a clone of the git repo, please run"
+	echo "'./hack/build-go.sh tools/genprotobuf'."
+	exit 1
+fi
+
+${genprotobuf} --output-base="${GOPATH}/src" "$@"

hack/update-generated-protobuf.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+OS_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${OS_ROOT}/hack/lib/init.sh"
+
+APIROOTS=${APIROOTS:-pkg}
+_tmp="${OS_ROOT}/_output/diff"
+
+cleanup() {
+  echo rm -rf "${_tmp}"
+}
+
+trap "cleanup" EXIT SIGINT
+
+cleanup
+for APIROOT in ${APIROOTS}; do
+  mkdir -p "${_tmp}/${APIROOT%/*}"
+  cp -rf "${OS_ROOT}/${APIROOT}" "${_tmp}/"
+done
+
+"${OS_ROOT}/hack/update-generated-protobuf.sh"
+for APIROOT in ${APIROOTS}; do
+  TMP_APIROOT="${_tmp}/${APIROOT}"
+  echo "diffing ${APIROOT} against freshly generated protobuf"
+  ret=0
+  diff -Naupr -I 'Auto generated by' "${OS_ROOT}/${APIROOT}" "${TMP_APIROOT}" || ret=$?
+  # cp -rf "${TMP_APIROOT}" "${OS_ROOT}/${APIROOT%/*}"
+  if [[ $ret -eq 0 ]]; then
+    echo "${APIROOT} up to date."
+  else
+    echo "${APIROOT} is out of date. Please run hack/update-generated-protobuf.sh"
+    exit 1
+  fi
+done

hack/verify-generated-protobuf.sh

@@ -50,6 +50,7 @@ DIR_BLACKLIST='./hack
 ./pkg/project
 ./pkg/quota
 ./pkg/router
+./pkg/sdn
 ./pkg/security
 ./pkg/serviceaccounts
 ./pkg/template

hack/verify-govet.sh

@@ -70,8 +70,16 @@ func DecodeNestedRawExtensionOrUnknown(d runtime.Decoder, ext *runtime.RawExtens
 	}
 	obj, gvk, err := d.Decode(ext.Raw, nil, nil)
 	if err != nil {
-		// TODO: record mime-type with the object
 		unk := &runtime.Unknown{Raw: ext.Raw}
+		if runtime.IsNotRegisteredError(err) {
+			if _, gvk, err := d.Decode(ext.Raw, nil, unk); err == nil {
+				unk.APIVersion = gvk.GroupVersion().String()
+				unk.Kind = gvk.Kind
+				ext.Object = unk
+				return
+			}
+		}
+		// TODO: record mime-type with the object
 		if gvk != nil {
 			unk.APIVersion = gvk.GroupVersion().String()
 			unk.Kind = gvk.Kind

pkg/api/extension/extension.go

@@ -1,6 +1,8 @@
 package api_test
 
 import (
+	"bytes"
+	"encoding/hex"
 	"fmt"
 	"math/rand"
 	"reflect"
@@ -11,10 +13,12 @@ import (
 	"github.com/google/gofuzz"
 	kapi "k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/meta"
+	"k8s.io/kubernetes/pkg/api/testapi"
 	apitesting "k8s.io/kubernetes/pkg/api/testing"
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/api/validation"
 	"k8s.io/kubernetes/pkg/runtime"
+	"k8s.io/kubernetes/pkg/runtime/serializer/protobuf"
 	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util/diff"
 	"k8s.io/kubernetes/pkg/util/intstr"
@@ -42,6 +46,16 @@ import (
 	_ "k8s.io/kubernetes/pkg/api/install"
 )
 
+var codecsToTest = []func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error){
+	func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error) {
+		return kapi.Codecs.LegacyCodec(version), nil
+	},
+	func(version unversioned.GroupVersion, item runtime.Object) (runtime.Codec, error) {
+		s := protobuf.NewSerializer(kapi.Scheme, kapi.Scheme, "application/arbitrary.content.type")
+		return kapi.Codecs.CodecForVersions(s, s, testapi.ExternalGroupVersions(), nil), nil
+	},
+}
+
 func fuzzInternalObject(t *testing.T, forVersion unversioned.GroupVersion, item runtime.Object, seed int64) runtime.Object {
 	f := apitesting.FuzzerFor(t, forVersion, rand.NewSource(seed))
 	f.Funcs(
@@ -130,7 +144,7 @@ func fuzzInternalObject(t *testing.T, forVersion unversioned.GroupVersion, item
 			case 0:
 				j.AttributeRestrictions = &authorizationapi.IsPersonalSubjectAccessReview{}
 			case 1:
-				j.AttributeRestrictions = &runtime.Unknown{TypeMeta: runtime.TypeMeta{Kind: "Type", APIVersion: "other"}, Raw: []byte(`{"apiVersion":"other","kind":"Type"}`)}
+				j.AttributeRestrictions = &runtime.Unknown{TypeMeta: runtime.TypeMeta{Kind: "Type", APIVersion: "other"}, ContentType: "application/json", Raw: []byte(`{"apiVersion":"other","kind":"Type"}`)}
 			default:
 				j.AttributeRestrictions = nil
 			}
@@ -439,6 +453,21 @@ func defaultHookContainerName(hook *deploy.LifecycleHook, containerName string)
 	}
 }
 
+func roundTripWithAllCodecs(t *testing.T, version unversioned.GroupVersion, item runtime.Object) {
+	var codecs []runtime.Codec
+	for _, fn := range codecsToTest {
+		codec, err := fn(version, item)
+		if err != nil {
+			t.Errorf("unable to get codec: %v", err)
+			return
+		}
+		codecs = append(codecs, codec)
+	}
+	for _, codec := range codecs {
+		roundTrip(t, codec, item)
+	}
+}
+
 func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
 	// Make a copy of the originalItem to give to conversion functions
 	// This lets us know if conversion messed with the input object
@@ -453,7 +482,8 @@ func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
 	data, err := runtime.Encode(codec, item)
 	if err != nil {
 		if runtime.IsNotRegisteredError(err) {
-			t.Logf("%v is not registered", name)
+			t.Logf("%v skipped: not registered: %v", name, err)
+			return
 		}
 		t.Errorf("%v: %v (%#v)", name, err, item)
 		return
@@ -474,7 +504,7 @@ func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
 	}
 
 	if !kapi.Semantic.DeepEqual(originalItem, obj2) {
-		t.Errorf("1: %v: diff: %v\nCodec: %v\nData: %s", name, diff.ObjectReflectDiff(originalItem, obj2), codec, string(data))
+		t.Errorf("1: %v: diff: %v\nCodec: %v\nData: %s", name, diff.ObjectReflectDiff(originalItem, obj2), codec, dataToString(data))
 		return
 	}
 
@@ -489,6 +519,13 @@ func roundTrip(t *testing.T, codec runtime.Codec, originalItem runtime.Object) {
 	}
 }
 
+func dataToString(s []byte) string {
+	if bytes.HasPrefix(s, []byte("k8s")) {
+		return "\n" + hex.Dump(s)
+	}
+	return string(s)
+}
+
 // skipStandardVersions is a map of Kind to a list of API versions to test with.
 var skipStandardVersions = map[string][]unversioned.GroupVersion{
 	// The API versions here are to test our object that serializes from/into
@@ -503,17 +540,20 @@ func TestSpecificKind(t *testing.T) {
 	kapi.Scheme.Log(t)
 	defer kapi.Scheme.Log(nil)
 
-	kind := "DeploymentConfig"
+	kind := "ClusterRole"
 	item, err := kapi.Scheme.New(osapi.SchemeGroupVersion.WithKind(kind))
 	if err != nil {
-		t.Errorf("Couldn't make a %v? %v", kind, err)
-		return
+		t.Fatalf("Couldn't make a %v? %v", kind, err)
+	}
+	codec, err := codecsToTest[1](v1.SchemeGroupVersion, nil)
+	if err != nil {
+		t.Fatal(err)
 	}
-	seed := int64(2703387474910584091) //rand.Int63()
+	seed := int64(2703387474910584091)
 	for i := 0; i < fuzzIters; i++ {
 		//t.Logf(`About to test %v with "v1"`, kind)
 		fuzzInternalObject(t, v1.SchemeGroupVersion, item, seed)
-		roundTrip(t, kapi.Codecs.LegacyCodec(v1.SchemeGroupVersion), item)
+		roundTrip(t, codec, item)
 	}
 }
 
@@ -558,7 +598,7 @@ func TestTypes(t *testing.T) {
 						continue
 					}
 					fuzzInternalObject(t, externalVersion, item, seed)
-					roundTrip(t, kapi.Codecs.LegacyCodec(externalVersion), item)
+					roundTripWithAllCodecs(t, externalVersion, item)
 				}
 			}
 		}