diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
index 429c40e4282ba..b25f9393af573 100644
--- a/qa/rolling-upgrade/build.gradle
+++ b/qa/rolling-upgrade/build.gradle
@@ -72,6 +72,7 @@ for (Version version : bwcVersions.wireCompatible) {
   Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
   oldClusterTestRunner.configure {
     systemProperty 'tests.rest.suite', 'old_cluster'
+    systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
   }
 
   Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure unicastSeed ->
@@ -96,6 +97,7 @@ for (Version version : bwcVersions.wireCompatible) {
   oneThirdUpgradedTestRunner.configure {
     systemProperty 'tests.rest.suite', 'mixed_cluster'
     systemProperty 'tests.first_round', 'true'
+    systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
     finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
   }
 
@@ -108,6 +110,7 @@ for (Version version : bwcVersions.wireCompatible) {
   twoThirdsUpgradedTestRunner.configure {
     systemProperty 'tests.rest.suite', 'mixed_cluster'
     systemProperty 'tests.first_round', 'false'
+    systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
     finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
   }
 
@@ -119,6 +122,7 @@ for (Version version : bwcVersions.wireCompatible) {
   Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
   upgradedClusterTestRunner.configure {
     systemProperty 'tests.rest.suite', 'upgraded_cluster'
+    systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
     /*
      * Force stopping all the upgraded nodes after the test runner
      * so they are alive during the test.
diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java
index 3ed98a5d1f772..f87eb783680d3 100644
--- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java
+++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java
@@ -19,15 +19,22 @@
 package org.elasticsearch.upgrades;
 
 import org.apache.http.util.EntityUtils;
-import org.elasticsearch.common.Booleans;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.junit.Before;
+import org.elasticsearch.Version;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.Response;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.junit.Assume.assumeThat;
 
 /**
@@ -36,11 +43,9 @@
  */
 public class XPackIT extends AbstractRollingTestCase {
     @Before
-    public void skipIfNotXPack() {
+    public void skipIfNotZip() {
         assumeThat("test is only supported if the distribution contains xpack",
                 System.getProperty("tests.distribution"), equalTo("zip"));
-        assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
-                CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
         /*
          * *Mostly* we want this for when we're upgrading from pre-6.3's
          * zip distribution which doesn't contain xpack to post 6.3's zip
@@ -50,11 +55,81 @@ public void skipIfNotXPack() {
     }
 
     /**
-     * Test a basic feature (SQL) which doesn't require any trial license.
-     * Note that the test methods on this class can run in any order so we
-     * <strong>might</strong> have already installed a trial license.
+     * Tests that xpack is able to work itself into a sane state during the
+     * upgrade by testing that it is able to create all of the templates that
+     * it needs. This isn't a very strong assertion of sanity, but it is better
+     * than nothing and should catch a few sad cases.
+     * <p>
+     * The trouble is that when xpack isn't able to create the templates that
+     * it needs it retries over and over and over again. This can
+     * <strong>really</strong> slow things down. This test asserts that xpack
+     * was able to create the templates so it <strong>shouldn't</strong> be
+     * spinning trying to create things and slowing down the rest of the
+     * system.
+     */
+    public void testIndexTemplatesCreated() throws Exception {
+        Version upgradeFromVersion =
+                Version.fromString(System.getProperty("tests.upgrade_from_version"));
+        boolean upgradeFromVersionHasXPack = upgradeFromVersion.onOrAfter(Version.V_6_3_0);
+        assumeFalse("this test doesn't really prove anything if the starting version has xpack and it is *much* more complex to maintain",
+                upgradeFromVersionHasXPack);
+        assumeFalse("since we're upgrading from a version without x-pack it won't have any templates",
+                CLUSTER_TYPE == ClusterType.OLD);
+
+        List<String> expectedTemplates = new ArrayList<>();
+        // Watcher creates its templates as soon as the first watcher node connects
+        expectedTemplates.add(".triggered_watches");
+        expectedTemplates.add(".watch-history-8");
+        expectedTemplates.add(".watches");
+        if (masterIsNewVersion()) {
+            // Everything else waits until the master is upgraded to create its templates
+            expectedTemplates.add(".ml-anomalies-");
+            expectedTemplates.add(".ml-meta");
+            expectedTemplates.add(".ml-notifications");
+            expectedTemplates.add(".ml-state");
+            expectedTemplates.add("logstash-index-template");
+            expectedTemplates.add("security-index-template");
+            expectedTemplates.add("security_audit_log");
+        }
+        Collections.sort(expectedTemplates);
+
+        /*
+         * The index templates are created asynchronously after startup and
+         * while this is usually fast we use assertBusy here just in case
+         * they aren't created by the time this test is run.
+         */
+        assertBusy(() -> {
+            List<String> actualTemplates;
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                    client().performRequest(new Request("GET", "/_template")).getEntity().getContent())) {
+                actualTemplates = new ArrayList<>(parser.map().keySet());
+            }
+            Collections.sort(actualTemplates);
+            /*
+             * This test asserts that the templates match *exactly* to force
+             * us to keep the list of templates up to date. Most templates
+             * aren't likely to cause a problem on upgrade but it is better
+             * to be safe and make sure they are all created than to be sorry
+             * and miss a bug that causes one to be missed on upgrade.
+             *
+             * We sort the templates so the error message is easy to read.
+             */
+            assertEquals(expectedTemplates, actualTemplates);
+        });
+    }
+
+    /**
+     * Test a basic feature (SQL) after the upgrade which only requires the
+     * "default" basic license. Note that the test methods on this class can
+     * run in any order so we <strong>might</strong> have already installed a
+     * trial license.
      */
-    public void testBasicFeature() throws IOException {
+    public void testBasicFeatureAfterUpgrade() throws IOException {
+        assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
+                CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
+
         Request bulk = new Request("POST", "/sql_test/doc/_bulk");
         bulk.setJsonEntity(
               "{\"index\":{}}\n"
@@ -71,16 +146,20 @@ public void testBasicFeature() throws IOException {
     }
 
     /**
-     * Test creating a trial license and using it. This is interesting because
-     * our other tests test cover starting a new cluster with the default
-     * distribution and enabling the trial license but this test is the only
-     * one that can upgrade from the oss distribution to the default
-     * distribution with xpack and the create a trial license. We don't
-     * <strong>do</strong> a lot with the trial license because for the most
-     * part those things are tested elsewhere, off in xpack. But we do use the
-     * trial license a little bit to make sure that it works.
+     * Test creating a trial license after the upgrade and a feature (ML) that
+     * requires the license. Our other tests test cover starting a new cluster
+     * with the default distribution and enabling the trial license but this
+     * test is the only one tests the rolling upgrade from the oss distribution
+     * to the default distribution with xpack and then creating of a trial
+     * license. We don't <strong>do</strong> a lot with the trial license
+     * because for the most part those things are tested elsewhere, off in
+     * xpack. But we do use the trial license a little bit to make sure that
+     * creating it worked properly.
      */
     public void testTrialLicense() throws IOException {
+        assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway",
+                CLUSTER_TYPE, equalTo(ClusterType.UPGRADED));
+
         Request startTrial = new Request("POST", "/_xpack/license/start_trial");
         startTrial.addParameter("acknowledge", "true");
         client().performRequest(startTrial);
@@ -108,4 +187,22 @@ public void testTrialLicense() throws IOException {
                 + "}\n");
         client().performRequest(createJob);
     }
+
+    /**
+     * Has the master been upgraded to the new version?
+     */
+    private boolean masterIsNewVersion() throws IOException {
+        Map<?, ?> map;
+        try (XContentParser parser = JsonXContent.jsonXContent.createParser(
+                NamedXContentRegistry.EMPTY,
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                client().performRequest(new Request("GET", "/_nodes/_master")).getEntity().getContent())) {
+            map = parser.map();
+        }
+        map = (Map<?, ?>) map.get("nodes");
+        assertThat(map.values(), hasSize(1));
+        map = (Map<?, ?>) map.values().iterator().next();
+        Version masterVersion = Version.fromString(map.get("version").toString());
+        return Version.CURRENT.equals(masterVersion);
+    }
 }