Read metrics store credentials from Vault

With this commit we allow Rally to use different metrics stores and read
the respective credentials from Vault. This change is also integrated
into the Vagrant workflow as well as night-rally-admin which can still
be used locally.

Relates elastic#103

Author: danielmitterdorfer

Makefile

@@ -1,7 +1,10 @@
 test:
 	python3 setup.py test
 
-install:
+install: configure
 	@ ./install.sh
 
-.PHONY: test install
+configure:
+	@ ./night_rally/fixtures/ansible/configure.sh
+
+.PHONY: test install configure

README.md

@@ -18,8 +18,9 @@ Night Rally is only tested on Mac OS X and Linux.
 
 1. Ensure that all prerequisites of [Rally](https://github.com/elastic/rally) are properly setup. Hint. It is *not* required to install Rally manually. Just ensure that its prerequisites are installed.
 2. Clone this repo: `git clone [email protected]:elastic/night-rally.git`
-3. Create a virtualenv with `python3 -m venv .venv` and activate it with `./.venv/bin/activate`
-4. Run `make install`
+3. [Setup Vault](https://github.com/elastic/infra/blob/master/docs/vault.md)
+4. Create a virtualenv with `python3 -m venv .venv` and activate it with `./.venv/bin/activate`
+5. Run `make install`
 
 Now you can invoke Night Rally regularly with the startup script `night_rally.sh` e.g. via cron. The script can also self-update if invoked as `night_rally.sh --self-update`.
 
@@ -131,10 +132,45 @@ To iterate on changes, always remember to re-run `./update_jenkins_night_rally.s
 6. `sudo -iu jenkins`
 7. Run `./test_nightly.sh`
 
-Results will be sent to the Elastic Cloud cluster `night-rally-tests` (details in LastPass).
-
 To iterate on changes, always remember to re-run `./update_jenkins_night_rally.sh` as user `vagrant`, before re-running tests.
 
+The Vagrant workflow retrieves credentials to the metrics store via Vault so ensure that it is [properly setup](https://github.com/elastic/infra/blob/master/docs/vault.md#github-auth). By default results will be sent to the Elastic Cloud cluster `night-rally-tests` (details in LastPass). In order to write to a different metrics store, you need to:
+
+1. Add credentials to Vault:
+
+    1.1 Create a file containing the cluster properties, e.g. in `~/cluster-creds.json`:
+
+    ```
+    {
+    "es_host": "cloud-host-name.eu-central-1.aws.cloud.es.io",
+    "es_password": "PASSWORD",
+    "es_username": "USERNAME",
+    "es_port": "9243",
+    "cloud_id": "CLOUD_ID_FROM_UI_AT_CLOUD_ELASTIC_CO",
+    "es_secure": "true"
+    }
+    ```
+    1.2 Add the key-value pairs to Vault. Please use `/secret/rally/cloud` as path prefix:
+
+    ```
+    vault write /secret/rally/cloud/your-metrics-cluster-name @cluster-creds.json
+    ```
+    1.3 Check that the data are present
+
+    ```
+    vault read /secret/rally/cloud/your-metrics-cluster-name
+    ```
+
+    1.4 Delete the cluster properties file
+
+    ```
+    rm ~/cluster-creds.json
+    ```
+
+2. `export VAULT_NIGHT_RALLY_METRICS_STORE_CREDENTIAL_PATH=/secret/rally/cloud/your-metrics-cluster-name`
+
+Afterwards you can start the Vagrant boxes.
+
 ##### Testing fixtures
 
 You can also simulate the application of fixtures like encryption-at-rest.

night_rally.sh

@@ -24,6 +24,11 @@ while [[ -h $SOURCE ]]; do # resolve $SOURCE until the file is no longer a symli
 done
 NIGHT_RALLY_HOME="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
+# Interactive users are required to be logged in already, others (like Jenkins) will expose VAULT_SECRET_ID env variable.
+if [[ -n $VAULT_SECRET_ID && -n $VAULT_ROLE_ID ]]; then
+    export VAULT_TOKEN=$( curl -s -X POST -H "Content-Type: application/json" -L -d "{\"role_id\":\"$VAULT_ROLE_ID\",\"secret_id\":\"$VAULT_SECRET_ID\"}" $VAULT_ADDR/v1/auth/approle/login | jq -r '.auth.client_token')
+fi
+RALLY_METRICS_STORE_CREDENTIAL_PATH=${RALLY_METRICS_STORE_CREDENTIAL_PATH:-"/secret/rally/cloud/nightly-rally-metrics"}
 
 ANSIBLE_ALL_TAGS=(encryption-at-rest initialize-data-disk trim drop-caches)
 ANSIBLE_SKIP_TAGS=( )

night_rally/admin.py

@@ -312,9 +312,9 @@ def positive_number(v):
 
 def main():
     parser = arg_parser()
-    es = client.create_client()
-
     args = parser.parse_args()
+
+    es = client.create_client(args.environment)
     if args.subcommand == "list":
         if args.configuration == "races":
             list_races(es, args)

night_rally/client.py

@@ -1,25 +1,22 @@
-def create_client(configuration_name=None):
+def create_client(configuration_name):
     import configparser
     import elasticsearch
     import certifi
     import pathlib
 
     def load():
         config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
-        if configuration_name:
-            # invoked from night-rally
-            _rally_ini_path = str(pathlib.Path.home()/".rally"/"rally-{}.ini".format(configuration_name))
-        else:
-            # invoked from e.g. night-rally-admin, ini files not deployed via Ansible
-            _rally_ini_path = str(pathlib.PurePath(__file__).parent/"fixtures/ansible/roles/rally-update/templates/rally.ini.j2")
-        config.read(_rally_ini_path)
+        config_file = pathlib.Path.home()/".rally"/"rally-{}.ini".format(configuration_name)
+        if not config_file.exists():
+            raise IOError("Config file [{}] is missing".format(config_file))
+        config.read(str(config_file))
         return config
 
     complete_cfg = load()
     cfg = complete_cfg["reporting"]
-    if cfg["datastore.secure"] == "True":
+    if cfg["datastore.secure"].lower() == "true":
         secure = True
-    elif cfg["datastore.secure"] == "False":
+    elif cfg["datastore.secure"].lower() == "false":
         secure = False
     else:
         raise ValueError("Setting [datastore.secure] is neither [True] nor [False] but [%s]" % cfg["datastore.secure"])

night_rally/fixtures/ansible/README.md

@@ -42,6 +42,7 @@ end
 
 def install_jenkins_user
   <<~EOF
+    set +x
     groupadd -g 1010 jenkins
     useradd -u 1010 -g 1010 -s /bin/bash -m -k /etc/skel -d /var/lib/jenkins jenkins
     mkdir /var/lib/jenkins/.ssh
@@ -102,7 +103,6 @@ def install_night_rally_test_script(base_ip, target_node_ip_addresses)
     export FIXTURES=${FIXTURES:-}
     export COORDINATING_NODES=#{base_ip}
     export TARGET_HOSTS="#{target_node_ip_addresses.join(',')}"
-    export IN_VAGRANT=YES
     export TEST_MODE=YES
     EOF2
 
@@ -146,6 +146,17 @@ def symlink_java_8
   "ln -s /usr/lib/jvm/java-8-openjdk-amd64 /var/lib/jenkins/.java/java8"
 end
 
+def pass_additional_jenkins_env_vars
+  <<~EOF
+    set +x
+    cat >>/var/lib/jenkins/.profile <<EOF2
+    export VAULT_ADDR=https://secrets.elastic.co:8200
+    export RALLY_METRICS_STORE_CREDENTIAL_PATH=${RALLY_METRICS_STORE_CREDENTIAL_PATH:-/secret/rally/cloud/vagrant-test-rally-metrics}
+    export IN_VAGRANT=YES
+    EOF2
+  EOF
+end
+
 Vagrant.configure(2) do |config|
   use_cachier = false
   if Vagrant.has_plugin?('vagrant-cachier') and not ENV['VAGRANT_NO_CACHIER']
@@ -183,12 +194,16 @@ Vagrant.configure(2) do |config|
         file_to_disk = File.realpath("vagrantdisk").to_s + "/#{node[:hostname]}_disk.vdi"
         vbox.customize ['createhd', '--filename', file_to_disk, '--size', 4*1024, '--format', 'VDI'] # Size is in MiB
         vbox.customize ['storageattach', :id, '--storagectl', 'IDE Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', file_to_disk]
+
+        if node[:hostname] == "coordinator"
+          override_config.vm.synced_folder "#{Dir.home}/", "/home/vagrant/", type: "rsync", rsync__args: ["-r", "--include=.vault-token", "--exclude=*"]
+        end
       end
 
       node_config.ssh.forward_agent = true
-
       node_config.vm.provision "shell", inline: tweak_es_sysctl
       node_config.vm.provision "shell", inline: install_jenkins_user
+      node_config.vm.provision "shell", inline: pass_additional_jenkins_env_vars, env: {"RALLY_METRICS_STORE_CREDENTIAL_PATH" => ENV["VAULT_NIGHT_RALLY_METRICS_STORE_CREDENTIAL_PATH"]}
       node_config.vm.provision "shell", inline: install_rally_user
       node_config.vm.provision "shell", inline: rally_sudoers
       node_config.vm.provision "shell", inline: install_rally_source
@@ -206,7 +221,7 @@ Vagrant.configure(2) do |config|
       # required because installing night_rally needs write access to the `.egg` directory
       if node == nodes.first
         node_config.vm.provision "shell", inline: <<-EOF
-          echo 'sudo rsync -r -o -g --chown=jenkins:jenkins --exclude ".vagrant" --exclude "vagrantdisk" /night_rally /var/lib/jenkins/' \
+          echo 'sudo rsync -r -o -g --chown=jenkins:jenkins --exclude ".vagrant" --exclude "vagrantdisk" /night_rally /home/vagrant/.vault-token /var/lib/jenkins/' \
             >/home/vagrant/update_jenkins_night_rally.sh
           chmod +x update_jenkins_night_rally.sh
           ./update_jenkins_night_rally.sh

night_rally/fixtures/ansible/Vagrantfile

@@ -4,6 +4,7 @@ display_skipped_hosts = false
 host_key_checking = false
 inventory = inventory/local
 library = library
+lookup_plugins = plugins/lookup_plugins
 nocows = 1
 retry_files_save_path = ~/.ansible
 roles_path = roles

night_rally/fixtures/ansible/ansible.cfg

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [[ -h $SOURCE ]]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ ${SOURCE} != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPT_HOME="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+cd ${SCRIPT_HOME}
+
+export RALLY_METRICS_STORE_CREDENTIAL_PATH=${RALLY_METRICS_STORE_CREDENTIAL_PATH:-"/secret/rally/cloud/nightly-rally-metrics"}
+# attempt to read so we can determine early on whether the user is logged in
+set +e
+vault read ${RALLY_METRICS_STORE_CREDENTIAL_PATH} > /dev/null 2>&1
+exit_code=$?
+set -e
+if [[ ${exit_code} -ne 0 ]]; then
+    echo "Failed to read from ${RALLY_METRICS_STORE_CREDENTIAL_PATH}. Please log in to Vault."
+    exit 1
+fi
+
+virtualenv --python=python2.7 .venv
+.venv/bin/pip install -q -r requirements.txt
+source ./.venv/bin/activate
+# Workaround for https://github.com/ansible/ansible/issues/31869 which is
+# basically a Python deficiency (see
+# http://sealiesoftware.com/blog/archive/2017/6/5/Objective-C_and_fork_in_macOS_1013.html
+# for details).
+export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
+
+ansible-playbook --connection=local playbooks/configure-rally.yml --extra-vars="rally_environment=nightly local_setup=true"
+ansible-playbook --connection=local playbooks/configure-rally.yml --extra-vars="rally_environment=release local_setup=true"
+# TODO: Remove this after the cutover period.
+ansible-playbook --connection=local playbooks/configure-rally.yml --extra-vars="rally_environment=release-new local_setup=true"
+ansible-playbook --connection=local playbooks/configure-rally.yml --extra-vars="rally_environment=nightly-new local_setup=true"
+unset OBJC_DISABLE_INITIALIZE_FORK_SAFETY
+deactivate
+cd - > /dev/null

night_rally/fixtures/ansible/configure.sh

@@ -0,0 +1,9 @@
+---
+
+- name: Configures Rally locally
+# ==============================
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  roles:
+    - { role: rally-configure, tags: rally-configure }

night_rally/fixtures/ansible/playbooks/configure-rally.yml

@@ -7,6 +7,7 @@
   hosts: coordinating-nodes, target-hosts
   gather_facts: true
   roles:
+    - { role: rally-configure, tags: rally-configure }
     - { role: rally-update, tags: rally-update }
   serial:
     - 1