Support adhoc benchmarks

With this commit we add a new mode to night-rally: adhoc. It allows to
run our benchmark suite for arbitrary commit hashes and publish the
results. night-rally now also records the results of release-benchmarks
in the ES metrics store.

Closes elastic#20

Author: danielmitterdorfer

README.md

@@ -49,4 +49,29 @@ For this three steps are needed:
 2. Adjust the menu structure in all other files (if this happens more often, we should think about using a template engine for that...)
 3. Add your track and the challenges to run in the `tracks` hash in `night_rally.py`
 
-If you're finished, please submit a PR. After the PR is merged, the new track will show up after the next benchmark.
+If you're finished, please submit a PR. After the PR is merged, the new track will show up after the next benchmark.
+
+
+#### Run a release benchmark
+
+Suppose we want to replace the (already published) results of the Elasticsearch release `5.3.0` with release `5.3.1` on our benchmark page. 
+
+1. Replace "5.3.0" with "5.3.1" in the `versions` array in each `index.html` in `external/pages`. Commit and push your changes (commit message convention: "Update comparison charts to 5.3.1")
+2. On the benchmark machine, issue the following command:
+
+```
+night_rally.sh --target-host=target-551504.benchmark.hetzner-dc17.elasticnet.co:39200 --mode=comparison --release="5.3.1" --replace-release="5.3.0"
+```
+
+#### Run an ad-hoc benchmark
+
+_Note: An ad-hoc benchmark is a benchmark of an arbitrary git commit hash that should be published as a separate data series in our release comparison charts._
+
+Suppose we want to publish the results of the commit hash `66202dc` in the Elasticsearch repo as "Lucene 7 upgrade" on our benchmark page. 
+
+1. Add "Lucene 7 upgrade" to the `versions` array in each `index.html` in `external/pages`. Commit and push your changes.
+2. On the benchmark machine, issue the following command:
+
+```
+night_rally.sh --target-host=target-551504.benchmark.hetzner-dc17.elasticnet.co:39200 --mode=adhoc --revision=66202dc --release="Lucene 7" --replace-release="Lucene 7
+```

night_rally.py

@@ -114,19 +114,30 @@ def ensure_dir(directory):
             raise
 
 
-def configure_rally(dry_run):
+def sanitize(text):
+    """
+    Sanitizes the input text so it is safe to use as an environment name in Rally.
+    
+    :param text: A text to sanitize
+    """
+    return text.lower().replace(" ", "-").replace(".", "_")
+
+
+def configure_rally(configuration_name, dry_run):
     user_home = os.getenv("HOME")
     root = os.path.dirname(os.path.realpath(__file__))
-    source = "%s/resources/rally-nightly.ini" % root
-    destination = "%s/.rally/rally-nightly.ini" % user_home
+    source = "%s/resources/rally-template.ini" % root
+    destination = "%s/.rally/rally-%s.ini" % (user_home, configuration_name)
     logger.info("Copying rally configuration from [%s] to [%s]" % (source, destination))
     if not dry_run:
         ensure_dir("%s/.rally" % user_home)
         shutil.copyfile(source, destination)
-        # materialize current user home
+        # materialize current user home and set environment name
         with fileinput.input(files=destination, inplace=True) as f:
             for line in f:
-                print(line.replace("~", user_home))
+                print(line
+                      .replace("~", user_home)
+                      .replace("<<ENVIRONMENT>>", configuration_name))
 
 
 class BaseCommand:
@@ -149,37 +160,52 @@ def command_line(self, track, challenge, car):
         raise NotImplementedError("abstract method")
 
 
-class NightlyCommand(BaseCommand):
-    def __init__(self, effective_start_date, target_host, root_dir, override_src_dir=None):
+class SourceBasedCommand(BaseCommand):
+    def __init__(self, effective_start_date, target_host, root_dir, revision, configuration_name, override_src_dir=None):
         super().__init__(effective_start_date, target_host, root_dir)
-        self.revision_ts = to_iso8601(effective_start_date)
+        self.revision = revision
+        self.configuration_name = configuration_name
         self.pipeline = "from-sources-complete"
         if override_src_dir is not None:
             self.override = " --override-src-dir=%s" % override_src_dir
         else:
             self.override = ""
 
     def command_line(self, track, challenge, car):
-        cmd = "rally --configuration-name=nightly --target-host={8} --pipeline={6} --quiet --revision \"@{0}\" " \
+        cmd = "rally --configuration-name={9} --target-host={8} --pipeline={6} --quiet --revision \"{0}\" " \
               "--effective-start-date \"{1}\" --track={2} --challenge={3} --car={4} --report-format=csv --report-file={5}{7}". \
-            format(self.revision_ts, self.ts, track, challenge, car, self.report_path(track, challenge, car), self.pipeline, self.override,
-                   self.target_host)
+            format(self.revision, self.ts, track, challenge, car, self.report_path(track, challenge, car), self.pipeline, self.override,
+                   self.target_host, self.configuration_name)
         # after we've executed the first benchmark, there is no reason to build again from sources
         self.pipeline = "from-sources-skip-build"
         return cmd
 
 
+class NightlyCommand(SourceBasedCommand):
+    CONFIG_NAME = "nightly"
+
+    def __init__(self, effective_start_date, target_host, root_dir, override_src_dir=None):
+        super().__init__(effective_start_date, target_host, root_dir, "@%s" % to_iso8601(effective_start_date),
+                         NightlyCommand.CONFIG_NAME, override_src_dir)
+
+
+class AdHocCommand(SourceBasedCommand):
+    def __init__(self, revision, effective_start_date, target_host, root_dir, configuration_name, override_src_dir=None):
+        super().__init__(effective_start_date, target_host, root_dir, revision, configuration_name, override_src_dir)
+
+
 class ReleaseCommand(BaseCommand):
-    def __init__(self, effective_start_date, target_host, root_dir, distribution_version):
+    def __init__(self, effective_start_date, target_host, root_dir, distribution_version, configuration_name):
         super().__init__(effective_start_date, target_host, root_dir)
+        self.configuration_name = configuration_name
         self.pipeline = "from-distribution"
         self.distribution_version = distribution_version
 
     def command_line(self, track, challenge, car):
-        cmd = "rally --target-host={7} --pipeline={6} --quiet --distribution-version={0} --effective-start-date \"{1}\" " \
-              "--track={2} --challenge={3} --car={4} --report-format=csv --report-file={5}". \
+        cmd = "rally --configuration-name={8} --target-host={7} --pipeline={6} --quiet --distribution-version={0} " \
+              "--effective-start-date \"{1}\" --track={2} --challenge={3} --car={4} --report-format=csv --report-file={5}". \
             format(self.distribution_version, self.ts, track, challenge, car, self.report_path(track, challenge, car), self.pipeline,
-                   self.target_host)
+                   self.target_host, self.configuration_name)
         return cmd
 
 
@@ -528,7 +554,11 @@ def parse_args():
         "--mode",
         help="In which mode to run?",
         default="nightly",
-        choices=["nightly", "comparison"])
+        choices=["nightly", "comparison", "adhoc"])
+    parser.add_argument(
+        "--revision",
+        help="Specify the source code revision to build for adhoc benchmarks.",
+        default="latest")
     parser.add_argument(
         "--release",
         help="Specify release string to use for comparison reports",
@@ -543,16 +573,23 @@ def parse_args():
 def main():
     args = parse_args()
     compare_mode = args.mode == "comparison"
+    adhoc_mode = args.mode == "adhoc"
     root_dir = config["root.dir"] if not args.override_root_dir else args.override_root_dir
     if compare_mode:
+        env_name = sanitize(args.release)
+        configure_rally(env_name, args.dry_run)
         if args.release.startswith("Docker"):
             command = DockerCommand(args.effective_start_date, args.target_host, root_dir, args.release)
         else:
             command = ReleaseCommand(args.effective_start_date, args.target_host, root_dir, args.release)
+    elif adhoc_mode:
+        env_name = sanitize(args.release)
+        command = AdHocCommand(args.revision, args.effective_start_date, args.target_host, root_dir, env_name, args.override_src_dir)
     else:
-        configure_rally(args.dry_run)
+        env_name = NightlyCommand.CONFIG_NAME
         command = NightlyCommand(args.effective_start_date, args.target_host, root_dir, args.override_src_dir)
 
+    configure_rally(env_name, args.dry_run)
     rally_failure = run_rally(tracks, command, args.dry_run)
     replace_release = args.replace_release if args.replace_release else args.release
     report(args.effective_start_date, tracks, defaults, replace_release, args.release, root_dir, compare_mode)

night_rally.sh

@@ -35,6 +35,8 @@ DRY_RUN=NO
 START_DATE=`date -u "+%Y-%m-%d %H:%M:%S"`
 MODE="nightly"
 RELEASE="master"
+# only needed for ad-hoc benchmarks
+REVISION="latest"
 REPLACE_RELEASE=${RELEASE}
 TARGET_HOST="localhost:9200"
 
@@ -61,6 +63,10 @@ case ${i} in
     MODE="${i#*=}"
     shift # past argument=value
     ;;
+    --revision=*)
+    REVISION="${i#*=}"
+    shift # past argument=value
+    ;;
     --release=*)
     RELEASE="${i#*=}"
     shift # past argument=value
@@ -136,7 +142,7 @@ fi
 #****************************
 set +e
 # Avoid failing before we transferred all results. Usually only a single benchmark trial run fails but lots of other succeed.
-python3 ${NIGHT_RALLY_HOME}/night_rally.py --target-host=${TARGET_HOST} --effective-start-date="${START_DATE}" ${NIGHT_RALLY_OVERRIDE}  --mode=${MODE} ${NIGHT_RALLY_DRY_RUN} --release="${RELEASE}" --replace-release="${REPLACE_RELEASE}"
+python3 ${NIGHT_RALLY_HOME}/night_rally.py --target-host=${TARGET_HOST} --effective-start-date="${START_DATE}" ${NIGHT_RALLY_OVERRIDE} --mode=${MODE} ${NIGHT_RALLY_DRY_RUN} --revision="${REVISION}" --release="${RELEASE}" --replace-release="${REPLACE_RELEASE}"
 exit_code=$?
 
 echo "Killing any lingering Rally processes"

resources/rally-nightly.ini renamed to resources/rally-template.ini

@@ -5,7 +5,7 @@ config.version = 8
 root.dir = ~/.rally/benchmarks
 
 [system]
-env.name = nightly
+env.name = <<ENVIRONMENT>>
 
 [source]
 local.src.dir = ~/.rally/benchmarks/src

tests.py

@@ -1,6 +1,8 @@
 import collections
 import datetime
 import unittest
+import os
+import os.path
 
 if __name__ == "__main__" and __package__ is None:
     __package__ = "night_rally"
@@ -148,6 +150,22 @@ def __call__(self, *args, **kwargs):
 
 
 class NightRallyTests(unittest.TestCase):
+    def test_sanitize(self):
+        self.assertEqual("lucene-7-upgrade", night_rally.sanitize("Lucene 7 Upgrade"))
+        self.assertEqual("lucene-7-upgrade", night_rally.sanitize("lucene-7-upgrade"))
+        self.assertEqual("elasticsearch-6_0_0-alpha1-docker", night_rally.sanitize("Elasticsearch 6.0.0-alpha1 Docker"))
+
+    def test_configure_rally(self):
+        path = "%s/.rally/rally-night-rally-unit-test.ini" % os.getenv("HOME")
+        try:
+            night_rally.configure_rally("night-rally-unit-test", dry_run=False)
+            self.assertTrue(os.path.isfile(path), "configuration routine did not create [%s]" % path)
+        finally:
+            try:
+                os.remove(path)
+            except Exception:
+                pass
+
     def test_run_two_challenges_successfully(self):
         system_call = RecordingSystemCall(return_value=False)
 
@@ -202,6 +220,61 @@ def test_run_two_tracks_successfully(self):
             system_call.calls
         )
 
+    def test_run_adhoc_benchmark(self):
+        system_call = RecordingSystemCall(return_value=False)
+
+        tracks = collections.OrderedDict()
+        tracks["geonames"] = [["append-no-conflicts", "defaults"]]
+        tracks["percolator"] = [["append-no-conflicts", "4gheap"]]
+
+        start_date = datetime.datetime(2016, 10, 1)
+        cmd = night_rally.AdHocCommand("66202dc", start_date, "localhost", "/rally_root", "lucene-7", override_src_dir="~/src/")
+        night_rally.run_rally(tracks, cmd, system=system_call)
+        self.assertEqual(2, len(system_call.calls))
+        self.assertEqual(
+            [
+                "rally --configuration-name=lucene-7 --target-host=localhost --pipeline=from-sources-complete --quiet "
+                "--revision \"66202dc\" --effective-start-date \"2016-10-01 00:00:00\" --track=geonames "
+                "--challenge=append-no-conflicts --car=defaults --report-format=csv "
+                "--report-file=/rally_root/reports/rally/2016-10-01-00-00-00/geonames/append-no-conflicts/defaults/report.csv "
+                "--override-src-dir=~/src/",
+                "rally --configuration-name=lucene-7 --target-host=localhost --pipeline=from-sources-skip-build --quiet "
+                "--revision \"66202dc\" --effective-start-date \"2016-10-01 00:00:00\" --track=percolator "
+                "--challenge=append-no-conflicts --car=4gheap --report-format=csv "
+                "--report-file=/rally_root/reports/rally/2016-10-01-00-00-00/percolator/append-no-conflicts/4gheap/report.csv "
+                "--override-src-dir=~/src/"]
+            ,
+            system_call.calls
+        )
+
+
+    def test_run_release_benchmark(self):
+        system_call = RecordingSystemCall(return_value=False)
+
+        tracks = {"geonames": [
+            ["append-no-conflicts", "defaults"],
+            ["append-no-conflicts", "4gheap"]
+        ]}
+        start_date = datetime.datetime(2016, 1, 1)
+        cmd = night_rally.ReleaseCommand(start_date, "localhost", "/rally_root", "5.3.0", "5_3_0")
+        night_rally.run_rally(tracks, cmd, system=system_call)
+        self.assertEqual(2, len(system_call.calls))
+        self.assertEqual(
+            [
+                "rally --configuration-name=5_3_0 --target-host=localhost --pipeline=from-distribution --quiet "
+                "--distribution-version=5.3.0 --effective-start-date \"2016-01-01 00:00:00\" --track=geonames "
+                "--challenge=append-no-conflicts --car=defaults --report-format=csv "
+                "--report-file=/rally_root/reports/rally/2016-01-01-00-00-00/geonames/append-no-conflicts/defaults/report.csv",
+
+                "rally --configuration-name=5_3_0 --target-host=localhost --pipeline=from-distribution --quiet "
+                "--distribution-version=5.3.0 --effective-start-date \"2016-01-01 00:00:00\" --track=geonames "
+                "--challenge=append-no-conflicts --car=4gheap --report-format=csv "
+                "--report-file=/rally_root/reports/rally/2016-01-01-00-00-00/geonames/append-no-conflicts/4gheap/report.csv"
+            ]
+            ,
+            system_call.calls
+        )
+
     def test_run_continues_on_error(self):
         self.maxDiff = None
         system_call = RecordingSystemCall(return_value=True)