diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
index a9fd5f1305037..c598c224bf721 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle
+++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle
@@ -119,6 +119,7 @@ yamlRestTest {
     'ml/evaluate_data_frame/Test classification auc_roc given actual_field is never equal to fish',
     'ml/evaluate_data_frame/Test classification auc_roc given predicted_class_field is never equal to mouse',
     'ml/evaluate_data_frame/Test classification auc_roc with missing class_name',
+    'ml/evaluate_data_frame/Test classification auc_roc with old results format',
     'ml/evaluate_data_frame/Test classification accuracy with missing predicted_field',
     'ml/evaluate_data_frame/Test regression given evaluation with empty metrics',
     'ml/evaluate_data_frame/Test regression given missing actual_field',
diff --git a/x-pack/plugin/ml/qa/ml-with-security/roles.yml b/x-pack/plugin/ml/qa/ml-with-security/roles.yml
index 5407121ad705d..29bb28e9735e6 100644
--- a/x-pack/plugin/ml/qa/ml-with-security/roles.yml
+++ b/x-pack/plugin/ml/qa/ml-with-security/roles.yml
@@ -7,7 +7,7 @@ minimal:
     # Give all users involved in these tests access to the indices where the data to
     # be analyzed is stored, because the ML roles alone do not provide access to
     # non-ML indices
-    - names: [ 'airline-data', 'index-*', 'unavailable-data', 'utopia', 'store' ]
+    - names: [ 'airline-data', 'index-*', 'unavailable-data', 'utopia*', 'store' ]
       privileges:
         - create_index
         - indices:admin/refresh
diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml
index 94282acec9ac1..279bc552d9fed 100644
--- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml
+++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/evaluate_data_frame.yml
@@ -186,6 +186,23 @@ setup:
             "foo": 0.24
           }
 
+  - do:
+      indices.create:
+        index: utopia_old_results_format
+
+  # This document is indexed to utopia_old_results_format index *without* nested mapping for ml.top_classes field
+  - do:
+      index:
+        index: utopia_old_results_format
+        body:  >
+          {
+            "classification_field_act": "cat",
+            "ml.top_classes": [
+              {"class_name": "dog", "class_probability": 0.9},
+              {"class_name": "cat", "class_probability": 0.1}
+            ]
+          }
+
   - do:
       indices.refresh: {}
 
@@ -750,6 +767,25 @@ setup:
   - match: { classification.auc_roc.doc_count: 8 }
   - is_false: classification.auc_roc.curve
 ---
+"Test classification auc_roc with old results format":
+  - do:
+      catch: /\[nested\] nested object under path \[ml.top_classes\] is not of nested type/
+      ml.evaluate_data_frame:
+        body:  >
+          {
+            "index": "utopia_old_results_format",
+            "evaluation": {
+              "classification": {
+                "actual_field": "classification_field_act.keyword",
+                "metrics": {
+                  "auc_roc": {
+                    "class_name": "cat"
+                  }
+                }
+              }
+            }
+          }
+---
 "Test classification accuracy with missing predicted_field":
   - do:
       catch: /\[classification\] must define \[predicted_field\] as required by the following metrics \[accuracy\]/