fix: gpu utilization may not be streamed out

This PR also backs out assigning a GPU to the ray head.

This PR also adopts the hostname convention already used in pod-vmstat.sh and pod-vmstat-memory.sh

Author: starpit

guidebooks/ml/ray/aggregator/setup.md

@@ -41,5 +41,5 @@ export JOB_ENV=$(
 ```
 
 ```shell
-export NUM_GPUS=$(echo "$JOB_ENV" | jq -cr '.runtime_env.env_vars.NUM_GPUS')
+export NUM_GPUS=${NUM_GPUS-$(echo "$JOB_ENV" | jq -cr '.runtime_env.env_vars.NUM_GPUS')}
 ```

guidebooks/ml/ray/run/gpu-utilization.sh

@@ -13,13 +13,13 @@ if [ -z "$QUIET_CONSOLE" ]; then
     kubectl get pod -l ${KUBE_POD_LABEL_SELECTOR} ${KUBE_CONTEXT_ARG} ${KUBE_NS_ARG} -o name \
             --field-selector=status.phase==Running \
         | xargs ${REPLSIZE} -P128 -I {} -n1 \
-            sh -c "sleep 0.\$(shuf -i 100-2000 -n1); kubectl exec --pod-running-timeout=1h ${KUBE_CONTEXT_ARG} ${KUBE_NS_ARG} {} -- sh -c \"nvidia-smi --query-gpu=timestamp,utilization.gpu,utilization.memory,memory.total,temperature.gpu,name --format=csv,noheader -l 10 | awk -Winteractive -v pod=\\\$(hostname) -F, '{printf \\\"\n\033[31;1m%s \033[0;31mGPUType\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$6, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mUtilization.GPU\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$2, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mUtilization.Memory\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$3, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mMemory.Total\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$4, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mTemperature.GPU\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$5, pod, \\\$1; }'\"" \
+            sh -c "sleep 0.\$(shuf -i 100-2000 -n1); kubectl exec --pod-running-timeout=1h ${KUBE_CONTEXT_ARG} ${KUBE_NS_ARG} {} -- sh -c \"nvidia-smi --query-gpu=timestamp,utilization.gpu,utilization.memory,memory.total,temperature.gpu,name --format=csv,noheader -l 10 | awk -Winteractive -v pod=\\\$(hostname | sed -E 's/-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}//') -F, '{printf \\\"\n\033[31;1m%s \033[0;31mGPUType\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$6, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mUtilization.GPU\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$2, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mUtilization.Memory\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$3, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mMemory.Total\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$4, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mTemperature.GPU\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$5, pod, \\\$1; }'\"" \
             | tee "${STREAMCONSUMER_RESOURCES}gpu.txt" \
             1>&2
 else
     kubectl get pod -l ${KUBE_POD_LABEL_SELECTOR} ${KUBE_CONTEXT_ARG} ${KUBE_NS_ARG} -o name \
             --field-selector=status.phase==Running \
         | xargs ${REPLSIZE} -P128 -I {} -n1 \
-            sh -c "sleep 0.\$(shuf -i 100-2000 -n1); kubectl exec --pod-running-timeout=1h ${KUBE_CONTEXT_ARG} ${KUBE_NS_ARG} {} -- sh -c \"nvidia-smi --query-gpu=timestamp,utilization.gpu,utilization.memory,memory.total,temperature.gpu,name --format=csv,noheader -l 10 | awk -Winteractive -v pod=\\\$(hostname) -F, '{printf \\\"\n\033[31;1m%s \033[0;31mGPUType\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$6, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mUtilization.GPU\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$2, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mUtilization.Memory\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$3, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mMemory.Total\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$4, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mTemperature.GPU\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$5, pod, \\\$1; }'\"" \
+            sh -c "sleep 0.\$(shuf -i 100-2000 -n1); kubectl exec --pod-running-timeout=1h ${KUBE_CONTEXT_ARG} ${KUBE_NS_ARG} {} -- sh -c \"nvidia-smi --query-gpu=timestamp,utilization.gpu,utilization.memory,memory.total,temperature.gpu,name --format=csv,noheader -l 10 | awk -Winteractive -v pod=\\\$(hostname | sed -E 's/-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}//') -F, '{printf \\\"\n\033[31;1m%s \033[0;31mGPUType\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$6, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mUtilization.GPU\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$2, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mUtilization.Memory\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$3, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mMemory.Total\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$4, pod, \\\$1; printf \\\"\033[31;1m%s \033[0;31mTemperature.GPU\t\t\t\033[0;2m%s %s\033[0m\n\\\", \\\$5, pod, \\\$1; }'\"" \
             > "${STREAMCONSUMER_RESOURCES}gpu.txt"
 fi

guidebooks/ml/ray/start/kubernetes/install-via-helm.sh

@@ -184,7 +184,6 @@ cd $REPO/$SUBDIR && \
          --set clusterNamespace=${KUBE_NS_FOR_REAL-${KUBE_NS}} \
          --set podTypes.rayHeadType.CPU=${NUM_CPUS-1} \
          --set podTypes.rayHeadType.CPUInteger=${NUM_CPUS_INTEGER-1} \
-         --set podTypes.rayHeadType.GPU=${NUM_GPUS-0} \
          --set podTypes.rayHeadType.memory=${HEAD_MEMORY-1Gi} \
          --set podTypes.rayHeadType.storage=${RAY_EPHEMERAL_STORAGE-5Gi} \
          --set podTypes.rayWorkerType.CPU=${NUM_CPUS-1} \