deploy: better diagnostics when driver deployment fails

With the recently introduced labels it becomes possible to describe
all objects that make up the driver deployment. This might be a lot of
output, but as this typically runs in a CI environment, it is better
to print too much than to little information because retrieving more
information later isn't possible.

Now also the logs of all containers are printed.

Author: pohly

deploy/kubernetes-distributed/deploy.sh

@@ -250,8 +250,18 @@ wait_for_daemonset () {
 
 # Wait until the DaemonSet is running on all nodes.
 if ! wait_for_daemonset default csi-hostpathplugin; then
+    echo
     echo "driver not ready"
-    kubectl describe daemonsets/csi-hostpathplugin
+    echo "Deployment:"
+    (set +e; set -x; kubectl describe all,role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver --all-namespaces -l app.kubernetes.io/instance=hostpath.csi.k8s.io)
+    echo
+    echo "Pod logs:"
+    kubectl get pods -l app.kubernetes.io/instance=hostpath.csi.k8s.io --all-namespaces -o=jsonpath='{range .items[*]}{"\n"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{" "}{end}{end}' | while read -r pod containers; do
+        for c in $containers; do
+            echo
+            (set +e; set -x; kubectl logs $pod $c)
+        done
+    done
     exit 1
 fi
 

deploy/util/deploy-hostpath.sh

@@ -247,10 +247,19 @@ expected_running_pods=6
 cnt=0
 while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt ${expected_running_pods} ]; do
     if [ $cnt -gt 30 ]; then
-        echo "$(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) running pods:"
-        kubectl describe pods
-
-        echo >&2 "ERROR: hostpath deployment not ready after over 5min"
+        echo "Expecting $expected_running_pods, have $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l)."
+        echo "Deployment:"
+        (set +e; set -x; kubectl describe all,role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver --all-namespaces -l app.kubernetes.io/instance=hostpath.csi.k8s.io)
+        echo
+        echo "Pod logs:"
+        kubectl get pods -l app.kubernetes.io/instance=hostpath.csi.k8s.io --all-namespaces -o=jsonpath='{range .items[*]}{"\n"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{" "}{end}{end}' | while read -r pod containers; do
+            for c in $containers; do
+                echo
+                (set +e; set -x; kubectl logs $pod $c)
+            done
+        done
+        echo
+        echo "ERROR: hostpath deployment not ready after over 5min"
         exit 1
     fi
     echo $(date +%H:%M:%S) "waiting for hostpath deployment to complete, attempt #$cnt"