Do not hardcode IP addresses in Ansible inventory groups

dliappis · web-flow · commit 050cc96f70d2 · 2018-07-02T11:36:28.000+03:00
Since we already define the IP addresses of the target hosts in Jenkins[1], there's no need to duplicate this information in static inventory files, when setting up fixtures with night-rally. Switch to a dynamic inventory script for setting up the needed host groups for Ansible fixtures. [1] https://github.com/elastic/infra/blob/8dd21b2e6e451407515186ab60e71b5d22114ba9/ci/jjb/elasticsearch-ci/globals/templates/elastic-elasticsearch/macrobenchmarks.yml#L26-L29 Relates elastic#75
diff --git a/night_rally/fixtures/ansible/Vagrantfile b/night_rally/fixtures/ansible/Vagrantfile
@@ -5,7 +5,8 @@ master_ip = '192.168.180.10'
 
 benchmark_boxes = %w(
 coordinator
-target1
+target01
+memorybenchmark01
 )
 
 nodes = benchmark_boxes.collect do |nodename|
diff --git a/night_rally/fixtures/ansible/inventory/local/vagrant.py b/night_rally/fixtures/ansible/inventory/local/vagrant.py
@@ -176,6 +176,7 @@ def update_the_cache():
             _group: list(ssh_config.keys()),
             'elasticsearch-ci-slaves-macrobenchmarks': [i for i in all_hostnames if 'coordinator' in i],
             'macrobenchmarks-targets': [i for i in all_hostnames if 'target' in i],
+            'memorybenchmarks': [i for i in all_hostnames if 'memorybenchmark' in i],
             '_meta': meta
         }, **vagrant_groups(ssh_config)))
     cache = open(cache_file, 'w')
diff --git a/night_rally/fixtures/ansible/inventory/production/inventory.py b/night_rally/fixtures/ansible/inventory/production/inventory.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+'''
+Extremely simple dynamic inventory script for Ansible.
+
+Expects TARGET_HOSTS env var to create the target-hosts group
+and COORDINATING_NODES env var to create the coordinating-nodes group.
+'''
+
+import argparse
+import os
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+ANSIBLE_SSH_PRIVATE_KEY_FILE = "/var/lib/jenkins/.ssh/rally"
+ANSIBLE_USER = "rally"
+
+class Inventory(object):
+    def __init__(self):
+        self.inventory = {}
+        self.read_cli_args()
+
+        # Called with `--list`
+        if self.args.list:
+            self.inventory = self.build_inventory()
+        # Called with `--host [hostname]`
+        elif self.args.host:
+            # Not implemented as `--list`` returns _meta
+            # see https://docs.ansible.com/ansible/latest/dev_guide/developing_inventory.html#tuning-the-external-inventory-script
+            self.inventory = self.empty_inventory()
+        # --refresh is a no-op as we don't cache results
+        elif self.args.refresh:
+            return
+        else:
+            self.inventory = self.empty_inventory()
+
+        print json.dumps(self.inventory);
+
+    def build_inventory(self):
+        inventory = self.dynamic_groups(group_name="target-hosts", env_var="TARGET_HOSTS").copy()
+        inventory.update(self.dynamic_groups(group_name="coordinating-nodes", env_var="COORDINATING_NODES"))
+        inventory.update(
+            {
+                '_meta': {
+                    'hostvars': {}
+                }
+            }
+        )
+
+        return inventory
+
+    def dynamic_groups(self, group_name=None, env_var=None):
+        if group_name is None or env_var is None or env_var not in os.environ:
+            return {}
+        else:
+            group_hosts = os.environ[env_var]
+
+        return {
+            group_name: {
+                'hosts': group_hosts.split(","),
+                'vars': {
+                    'ansible_ssh_user': ANSIBLE_USER,
+                    'ansible_ssh_private_key_file': ANSIBLE_SSH_PRIVATE_KEY_FILE
+                }
+            }
+        }
+
+    def empty_inventory(self):
+        return {'_meta': {'hostvars': {}}}
+
+    def read_cli_args(self):
+        parser = argparse.ArgumentParser()
+        parser.add_argument('--list', action = 'store_true')
+        parser.add_argument('--host', action = 'store')
+        parser.add_argument('--refresh', action = 'store_true')
+        self.args = parser.parse_args()
+
+Inventory()
diff --git a/night_rally/fixtures/ansible/inventory/production/macrobenchmarks b/night_rally/fixtures/ansible/inventory/production/macrobenchmarks
diff --git a/night_rally/fixtures/ansible/playbooks/setup.yml b/night_rally/fixtures/ansible/playbooks/setup.yml
@@ -1,8 +1,8 @@
 ---
 
-- name: run fixtures on macrobenchmarks targets
-# =============================================
-  hosts: macrobenchmarks-targets
+- name: run fixtures on target hosts
+# ==================================
+  hosts: target-hosts
   gather_facts: true
   roles:
     - { role: trim, tags: trim }
diff --git a/night_rally/fixtures/ansible/playbooks/update-rally.yml b/night_rally/fixtures/ansible/playbooks/update-rally.yml
@@ -4,7 +4,7 @@
 # ================================================================
 # First the Rally master (coordinating node which is running on the (only) Jenkins slave node, then the Rally slaves (running
 # on the target machines)
-  hosts: elasticsearch-ci-slaves-macrobenchmarks, macrobenchmarks-targets
+  hosts: coordinating-nodes, target-hosts
   gather_facts: true
   roles:
     - { role: rally-update, tags: rally-update }
diff --git a/night_rally/fixtures/ansible/roles/rally-update/tasks/main.yml b/night_rally/fixtures/ansible/roles/rally-update/tasks/main.yml
@@ -21,7 +21,7 @@
 
 
 - name: start rallyd
-  command: ./rallyd start --node-ip {{ inventory_hostname }} --coordinator-ip {{ groups['elasticsearch-ci-slaves-macrobenchmarks'][0] }}
+  command: ./rallyd start --node-ip {{ inventory_hostname }} --coordinator-ip {{ groups['coordinating-nodes'][0] }}
   args:
     chdir: /var/lib/jenkins/bin
   environment:
diff --git a/night_rally/night_rally.py b/night_rally/night_rally.py
@@ -11,8 +11,9 @@
 
 ROOT = os.path.dirname(os.path.realpath(__file__))
 RALLY_BINARY = "rally --skip-update"
-
 VERSION_PATTERN = re.compile("^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$")
+# The port that Elasticsearch is configured to use for rest requests
+TARGET_PORT = 39200
 
 # console logging
 logging.basicConfig(level=logging.INFO, format="[%(asctime)s][%(levelname)s] %(message)s")
@@ -472,6 +473,8 @@ def __str__(self):
 
 
 def run_rally(tracks, available_hosts, command, dry_run=False, skip_ansible=False, system=os.system):
+    # Build list of host:port pairs for target hosts
+    available_hosts_with_ports = list(map(lambda x: "{}:{}".format(x, TARGET_PORT), available_hosts))
     rally_failure = False
     if dry_run:
         runner = logger.info
@@ -480,7 +483,7 @@ def run_rally(tracks, available_hosts, command, dry_run=False, skip_ansible=Fals
     for track in tracks:
         track_name = track["track"]
         for combination in track["combinations"]:
-            race_cfg = RaceConfig(track_name, combination, available_hosts)
+            race_cfg = RaceConfig(track_name, combination, available_hosts_with_ports)
 
             if race_cfg.target_hosts:
                 if command.runnable(race_cfg):
@@ -495,7 +498,7 @@ def run_rally(tracks, available_hosts, command, dry_run=False, skip_ansible=Fals
                     logger.info("Running Rally on [%s]", race_cfg)
                     start = time.perf_counter()
                     try:
-                        wait_until_port_is_free(available_hosts)
+                        wait_until_port_is_free(available_hosts_with_ports)
                         if runner(command.command_line(race_cfg)):
                             rally_failure = True
                             logger.error("Failed to run [%s]. Please check the logs.", race_cfg)
diff --git a/tests/night_rally_test.py b/tests/night_rally_test.py
@@ -121,13 +121,13 @@ def test_run_two_challenges_successfully(self, mocked_wait_until_port_is_free):
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" "
                 "--car-params=\"verbose_iw_logging_enabled:true\" --pipeline=\"from-sources-complete\" "
                 "--revision=\"@2016-01-01T00:00:00Z\"",
 
-                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"4gheap\" --user-tag=\"env:bare,name:geonames-4g\" --runtime-jdk=\"8\" --pipeline=\"from-sources-skip-build\" "
                 "--revision=\"@2016-01-01T00:00:00Z\""
@@ -170,12 +170,12 @@ def test_run_two_tracks_successfully(self, mocked_wait_until_port_is_free):
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" "
                 "--pipeline=\"from-sources-complete\" --revision=\"@2016-10-01T00:00:00Z\"",
 
-                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"percolator\" --challenge=\"append-no-conflicts\" "
                 "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --runtime-jdk=\"8\" --pipeline=\"from-sources-skip-build\" "
                 "--revision=\"@2016-10-01T00:00:00Z\""
@@ -219,12 +219,12 @@ def test_run_adhoc_benchmark(self, mocked_wait_until_port_is_free):
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"lucene-7\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"lucene-7\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" "
                 "--pipeline=\"from-sources-complete\" --revision=\"66202dc\"",
 
-                "rally --skip-update --configuration-name=\"lucene-7\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"lucene-7\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"percolator\" --challenge=\"append-no-conflicts\" "
                 "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --runtime-jdk=\"8\" "
                 "--pipeline=\"from-sources-skip-build\" --revision=\"66202dc\""
@@ -261,12 +261,12 @@ def test_run_release_benchmark_without_plugins(self, mocked_wait_until_port_is_f
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" --distribution-version=\"5.3.0\" "
                 "--pipeline=\"from-distribution\"",
 
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"4gheap\" --user-tag=\"env:bare,name:geonames-4g\" --runtime-jdk=\"8\" --distribution-version=\"5.3.0\" "
                 "--pipeline=\"from-distribution\""
@@ -315,14 +315,14 @@ def test_run_release_benchmark_with_plugins(self, mocked_wait_until_port_is_free
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults\" --user-tag=\"env:x-pack,name:geonames-defaults,x-pack:true\" --runtime-jdk=\"8\" "
                 "--track-params=\"bulk_size:3000\" --client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally',"
                 "basic_auth_password:'rally-password'\" --elasticsearch-plugins=\"x-pack:security,monitoring\" "
                 "--distribution-version=\"5.3.0\" --pipeline=\"from-distribution\"",
 
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"4gheap\" --user-tag=\"env:x-pack,name:geonames-4g,x-pack:true\" --runtime-jdk=\"8\" "
                 "--track-params=\"bulk_size:2000\" --client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally',"
@@ -371,13 +371,13 @@ def test_run_release_benchmark_with_x_pack_module(self, mocked_wait_until_port_i
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults,x-pack-security,x-pack-monitoring\" --user-tag=\"env:x-pack,name:geonames-defaults,x-pack:true\" "
                 "--runtime-jdk=\"8\" --client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally',"
                 "basic_auth_password:'rally-password'\" --distribution-version=\"6.3.0\" --pipeline=\"from-distribution\"",
 
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"4gheap,x-pack-security,x-pack-monitoring\" --user-tag=\"env:x-pack,name:geonames-4g,x-pack:true\" "
                 "--runtime-jdk=\"8\" --client-options=\"timeout:60,use_ssl:true,verify_certs:false,basic_auth_user:'rally',"
@@ -412,7 +412,7 @@ def test_run_release_benchmark_with_transport_nio(self, mocked_wait_until_port_i
         self.assertEqual(1, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" "
                 "--elasticsearch-plugins=\"transport-nio:transport+http\" --distribution-version=\"7.0.0\" "
@@ -500,12 +500,12 @@ def test_run_docker_benchmark(self, mocked_wait_until_port_is_free):
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults\" --user-tag=\"env:docker,name:geonames-defaults\" --runtime-jdk=\"8\" "
                 "--distribution-version=\"5.3.0\" --pipeline=\"docker\"",
 
-                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"release\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-01-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"4gheap\" --user-tag=\"env:docker,name:geonames-4g\" --runtime-jdk=\"8\" "
                 "--distribution-version=\"5.3.0\" --pipeline=\"docker\""
@@ -550,12 +550,12 @@ def test_run_continues_on_error(self, mocked_wait_until_port_is_free):
         self.assertEqual(2, len(system_call.calls))
         self.assertEqual(
             [
-                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"geonames\" --challenge=\"append-no-conflicts\" "
                 "--car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\" --runtime-jdk=\"8\" "
                 "--pipeline=\"from-sources-complete\" --revision=\"@2016-10-01T00:00:00Z\"",
 
-                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost\" "
+                "rally --skip-update --configuration-name=\"nightly\" --quiet --target-host=\"localhost:39200\" "
                 "--effective-start-date=\"2016-10-01 00:00:00\" --track=\"percolator\" --challenge=\"append-no-conflicts\" "
                 "--car=\"4gheap\" --user-tag=\"env:bare,name:percolator-4g\" --runtime-jdk=\"8\" "
                 "--pipeline=\"from-sources-skip-build\" --revision=\"@2016-10-01T00:00:00Z\""
@@ -595,11 +595,10 @@ def test_run_with_telemetry(self, mocked_wait_until_port_is_free):
         self.assertEqual(
             [
                 "rally --skip-update --telemetry=\"jfr,gc,jit\" --telemetry-params=\"recording-template:profile\" "
-                "--configuration-name=\"nightly\" --quiet --target-host=\"localhost\" --effective-start-date=\"2016-01-01 00:00:00\" "
+                "--configuration-name=\"nightly\" --quiet --target-host=\"localhost:39200\" --effective-start-date=\"2016-01-01 00:00:00\" "
                 "--track=\"geonames\" --challenge=\"append-no-conflicts\" --car=\"defaults\" --user-tag=\"env:bare,name:geonames-defaults\""
                 " --runtime-jdk=\"8\" --pipeline=\"from-sources-complete\" --revision=\"@2016-01-01T00:00:00Z\""
             ]
             ,
             system_call.calls
         )
-

Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,8 @@ master_ip = '192.168.180.10'`
`5`	`5`
`6`	`6`	`benchmark_boxes = %w(`
`7`	`7`	`coordinator`
`8`		`-target1`
	`8`	`+target01`
	`9`	`+memorybenchmark01`
`9`	`10`	`)`
`10`	`11`
`11`	`12`	`nodes = benchmark_boxes.collect do \|nodename\|`