Preserve logs, telemetry from all race configs after every nightly run (elastic#184)

Since /var/lib/jenkins/.rally/benchmarks/races gets fscked before every
night-rally invocation, by default, for our Jenkins triggered nightlies,
we still want to preserve important logs and telemetry output under each
race directory.

This commit copies the subdirectories telemetry, logs under each race id
to `$HOME/race_archive/<date_part_from_effective_time_stamp>`

Closes elastic#168

Author: dliappis

README.md

@@ -70,6 +70,12 @@ For more details, please issue `night-rally-admin delete annotation --help`.
 
 **Note:** The admin tool also supports a dry-run mode for all commands that would change the data store. Just append `--dry-run`.
 
+#### Find logs, telemetry or heapdumps from older nightly runs
+
+Each nightly execution wipes the data disk on target machines (`~/.var/lib/jenkins/rally/benchmarks/races`) as part of the initial [setup fixture](https://github.com/elastic/night-rally/tree/master/night_rally/fixtures/ansible/roles/initialize-data-disk).
+Important information from earlier nightly executions, specifically the `heapdump`, `logs` and `telemetry` subdirectories from each race id are copied under `/var/lib/jenkins/race_archive/<YYYYMMDD>` on each target machine, for future reference.
+This is done as the last step before the nightly run has finished. 
+
 #### Add a new track
 
 Benchmarks get executed in two environments, group-1 / group-2 (see [infra repo](https://github.com/elastic/infra/blob/master/ansible/inventory/production/hetzner/benchmarks)).

night_rally.sh

@@ -42,7 +42,7 @@ SKIP_ANSIBLE=NO
 # Optionally, allow skipping Rally's auto-update mechanism. Applicable in Vagrant only.
 SKIP_RALLY_UPDATE=${SKIP_RALLY_UPDATE:-NO}
 # We invoke Rally with the current (UTC) timestamp. This determines the version to checkout.
-EFFECTIVE_START_DATE=`date -u "+%Y-%m-%d %H:%M:%S"`
+EFFECTIVE_START_DATE=$(date -u "+%Y-%m-%d %H:%M:%S")
 # Deploy different Cloud credentials when running inside Vagrant
 IN_VAGRANT=${IN_VAGRANT:NO}
 MODE="nightly"
@@ -269,5 +269,14 @@ fi
 #****************************
 set -e
 
+# Preserve logs in a directory that doesn't get wiped by the initialize-data-disk fixture between invocations
+if [[ $SKIP_ANSIBLE == NO ]]
+then
+  pushd . >/dev/null 2>&1
+  cd "${NIGHT_RALLY_HOME}/night_rally/fixtures/ansible"
+  ansible-playbook -i inventory/production -u rally playbooks/copy-logs.yml --extra-vars "effective_start_date=${EFFECTIVE_START_DATE}"
+  popd >/dev/null 2>&1
+fi
+
 # Exit with the same exit code as night_rally.py
 exit ${exit_code}

night_rally/fixtures/ansible/playbooks/copy-logs.yml

@@ -0,0 +1,53 @@
+---
+- name: archive logs to a persistent directory
+# ============================================
+# Copy subdirectories of `/var/lib/jenkins/.rally/benchmarks/races/` with important data (logs, telemetry)
+# to a directory that doesn't get wiped away between night-rally invocations
+  hosts: target-hosts
+  gather_facts: true
+  become: true
+  become_user: jenkins
+
+  vars:
+    races_dir: '{{ ansible_env.HOME }}/.rally/benchmarks/races/'
+    current_date: '{{ effective_start_date.split(" ")[0] | replace("-", "") }}'
+    arch_dir: '{{ ansible_env.HOME }}/race_archive/{{ current_date }}'
+    dirs_to_preserve:
+      - 'heapdump'
+      - 'logs'
+      - 'telemetry'
+
+  tasks:
+    - name: ensure race archive dir is present
+      file:
+        path: '{{ arch_dir }}'
+        state: directory
+        mode: '0755'
+
+    - name: build list of race paths to preserve
+      find:
+        path: '{{ races_dir }}'
+        recurse: true
+        excludes: 'lost+found'
+        file_type: 'directory'
+        patterns: '{{ dirs_to_preserve }}'
+      register: race_paths
+
+    - name: ensure race archive dir is present
+      file:
+        path: '{{ arch_dir }}/{{ item["path"] | replace(races_dir, "") }}/'
+        state: directory
+        mode: '0755'
+      loop: '{{ race_paths.files }}'
+      no_log: true
+
+    - name: copy race paths to a persistent directory
+      synchronize:
+        src: '{{ item["path"] }}/'
+        dest: '{{ arch_dir }}/{{ item["path"] | replace(races_dir, "") }}/'
+      delegate_to: "{{ inventory_hostname }}"
+      loop: '{{ race_paths.files }}'
+      no_log: true
+
+    - debug:
+        msg: 'All race sub-directories [{{ dirs_to_preserve | join(",") }}] under [{{ races_dir }}] have been successfully copied under [{{ arch_dir }}]'