Update boilerplate checks and generated files

Makefile

@@ -101,15 +101,15 @@ clientset: ## Generate a typed clientset
 	cd ./vendor/k8s.io/code-generator/cmd && go install ./client-gen ./lister-gen ./informer-gen
 	$$GOPATH/bin/client-gen --clientset-name clientset --input-base sigs.k8s.io/cluster-api/pkg/apis \
 		--input cluster/v1alpha1 --output-package sigs.k8s.io/cluster-api/pkg/client/clientset_generated \
-		--go-header-file=./hack/boilerplate.go.txt
+		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
 	$$GOPATH/bin/lister-gen --input-dirs sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1 \
 		--output-package sigs.k8s.io/cluster-api/pkg/client/listers_generated \
-		--go-header-file=./hack/boilerplate.go.txt
+		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
 	$$GOPATH/bin/informer-gen --input-dirs sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1 \
 		--versioned-clientset-package sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset \
 		--listers-package sigs.k8s.io/cluster-api/pkg/client/listers_generated \
 		--output-package sigs.k8s.io/cluster-api/pkg/client/informers_generated \
-		--go-header-file=./hack/boilerplate.go.txt
+		--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
 	$(MAKE) gazelle
 
 .PHONY: clean
@@ -162,6 +162,6 @@ docker-push-ci: docker-build-ci  ## Build the docker image for ci
 
 .PHONY: verify
 verify:
-	./hack/verify_boilerplate.py
+	./hack/verify-boilerplate.sh
 	./hack/verify_clientset.sh
 	./hack/verify-bazel.sh

cmd/clusterctl/clusterdeployer/bootstrap/kind/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/bootstrap/kind",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/util:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],

cmd/clusterctl/clusterdeployer/bootstrap/minikube/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/bootstrap/minikube",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/util:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],

hack/boilerplate/BUILD

@@ -0,0 +1,31 @@
+package(default_visibility = ["//visibility:public"])
+
+exports_files(glob(["*.txt"]))
+
+py_test(
+    name = "boilerplate_test",
+    srcs = [
+        "boilerplate.py",
+        "boilerplate_test.py",
+    ],
+    data = glob([
+        "*.txt",
+        "test/*",
+    ]),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//hack/boilerplate/test:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

hack/boilerplate/boilerplate.Dockerfile.txt

@@ -11,3 +11,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+

hack/boilerplate/boilerplate.Makefile.txt

@@ -11,3 +11,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+

hack/boilerplate/boilerplate.bzl.txt

@@ -11,3 +11,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+

hack/boilerplate/boilerplate.generatebzl.txt

@@ -0,0 +1,14 @@
+# Copyright The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

hack/boilerplate.go.txt → hack/boilerplate/boilerplate.generatego.txt

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,3 +13,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+

hack/boilerplate/boilerplate.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python
+
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import argparse
+import datetime
+import difflib
+import glob
+import os
+import re
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "filenames",
+    help="list of files to check, all files if unspecified",
+    nargs='*')
+
+rootdir = os.path.dirname(__file__) + "/../../"
+rootdir = os.path.abspath(rootdir)
+parser.add_argument(
+    "--rootdir", default=rootdir, help="root directory to examine")
+
+default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate")
+parser.add_argument(
+    "--boilerplate-dir", default=default_boilerplate_dir)
+
+parser.add_argument(
+    "-v", "--verbose",
+    help="give verbose output regarding why a file does not pass",
+    action="store_true")
+
+args = parser.parse_args()
+
+verbose_out = sys.stderr if args.verbose else open("/dev/null", "w")
+
+def get_refs():
+    refs = {}
+
+    for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")):
+        extension = os.path.basename(path).split(".")[1]
+
+        ref_file = open(path, 'r')
+        ref = ref_file.read().splitlines()
+        ref_file.close()
+        refs[extension] = ref
+
+    return refs
+
+def is_generated_file(filename, data, regexs):
+    for d in skipped_ungenerated_files:
+        if d in filename:
+            return False
+
+    p = regexs["generated"]
+    return p.search(data)
+
+def file_passes(filename, refs, regexs):
+    try:
+        f = open(filename, 'r')
+    except Exception as exc:
+        print("Unable to open %s: %s" % (filename, exc), file=verbose_out)
+        return False
+
+    data = f.read()
+    f.close()
+
+    # determine if the file is automatically generated
+    generated = is_generated_file(filename, data, regexs)
+
+    basename = os.path.basename(filename)
+    extension = file_extension(filename)
+    if generated:
+        if extension == "go":
+            extension = "generatego"
+        elif extension == "bzl":
+            extension = "generatebzl"
+
+    if extension != "":
+        ref = refs[extension]
+    else:
+        ref = refs[basename]
+
+    # remove extra content from the top of files
+    if extension == "go" or extension == "generatego":
+        p = regexs["go_build_constraints"]
+        (data, found) = p.subn("", data, 1)
+    elif extension == "sh":
+        p = regexs["shebang"]
+        (data, found) = p.subn("", data, 1)
+
+    data = data.splitlines()
+
+    # if our test file is smaller than the reference it surely fails!
+    if len(ref) > len(data):
+        print('File %s smaller than reference (%d < %d)' %
+              (filename, len(data), len(ref)),
+              file=verbose_out)
+        return False
+
+    # trim our file to the same number of lines as the reference file
+    data = data[:len(ref)]
+
+    p = regexs["year"]
+    for d in data:
+        if p.search(d):
+            if generated:
+                print('File %s has the YEAR field, but it should not be in generated file' % filename, file=verbose_out)
+            else:
+                print('File %s has the YEAR field, but missing the year of date' % filename, file=verbose_out)
+            return False
+
+    if not generated:
+        # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR"
+        p = regexs["date"]
+        for i, d in enumerate(data):
+            (data[i], found) = p.subn('YEAR', d)
+            if found != 0:
+                break
+
+    # if we don't match the reference at this point, fail
+    if ref != data:
+        print("Header in %s does not match reference, diff:" % filename, file=verbose_out)
+        if args.verbose:
+            print(file=verbose_out)
+            for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''):
+                print(line, file=verbose_out)
+            print(file=verbose_out)
+        return False
+
+    return True
+
+def file_extension(filename):
+    return os.path.splitext(filename)[1].split(".")[-1].lower()
+
+skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', 'cluster/env.sh',
+                "vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test",
+                "pkg/kubectl/generated/bindata.go"]
+
+# list all the files contain 'DO NOT EDIT', but are not generated
+skipped_ungenerated_files = ['hack/lib/swagger.sh', 'hack/boilerplate/boilerplate.py']
+
+def normalize_files(files):
+    newfiles = []
+    for pathname in files:
+        if any(x in pathname for x in skipped_dirs):
+            continue
+        newfiles.append(pathname)
+    for i, pathname in enumerate(newfiles):
+        if not os.path.isabs(pathname):
+            newfiles[i] = os.path.join(args.rootdir, pathname)
+    return newfiles
+
+def get_files(extensions):
+    files = []
+    if len(args.filenames) > 0:
+        files = args.filenames
+    else:
+        for root, dirs, walkfiles in os.walk(args.rootdir):
+            # don't visit certain dirs. This is just a performance improvement
+            # as we would prune these later in normalize_files(). But doing it
+            # cuts down the amount of filesystem walking we do and cuts down
+            # the size of the file list
+            for d in skipped_dirs:
+                if d in dirs:
+                    dirs.remove(d)
+
+            for name in walkfiles:
+                pathname = os.path.join(root, name)
+                files.append(pathname)
+
+    files = normalize_files(files)
+    outfiles = []
+    for pathname in files:
+        basename = os.path.basename(pathname)
+        extension = file_extension(pathname)
+        if extension in extensions or basename in extensions:
+            outfiles.append(pathname)
+    return outfiles
+
+def get_dates():
+    years = datetime.datetime.now().year
+    return '(%s)' % '|'.join((str(year) for year in range(2014, years+1)))
+
+def get_regexs():
+    regexs = {}
+    # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing
+    regexs["year"] = re.compile( 'YEAR' )
+    # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year as a regex like: "(2014|2015|2016|2017|2018)";
+    # company holder names can be anything
+    regexs["date"] = re.compile(get_dates())
+    # strip // +build \n\n build constraints
+    regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE)
+    # strip #!.* from shell scripts
+    regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
+    # Search for generated files
+    regexs["generated"] = re.compile( 'DO NOT EDIT' )
+    return regexs
+
+def main():
+    regexs = get_regexs()
+    refs = get_refs()
+    filenames = get_files(refs.keys())
+
+    for filename in filenames:
+        if not file_passes(filename, refs, regexs):
+            print(filename, file=sys.stdout)
+
+    return 0
+
+if __name__ == "__main__":
+  sys.exit(main())

hack/boilerplate/boilerplate.py.txt

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # Copyright YEAR The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,3 +13,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+

hack/boilerplate/boilerplate.sh.txt

@@ -11,3 +11,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+