[ML] Fix error when a nested field is also incompatible (#71736)

This fixes a bug that was introduced in #71400.

The bug occurs when the _explain API is called for a
data frame analytics job and there are nested fields
that are also incompatible. In particular, we end
up calling `iterator.remove()` twice which throws
`IllegalStateException`.

I also took the chance to move the nested field check
first as I think it's more informative to explain a
field is not included due to being nested than because
it has an incompatible type in this case.

The PR is marked as `non-issue` as this has not been
released yet.

Author: dimitris-athanasiou

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java

@@ -253,14 +253,13 @@ private void removeFieldsWithIncompatibleTypes(Set<String> fields, Set<FieldSele
         Iterator<String> fieldsIterator = fields.iterator();
         while (fieldsIterator.hasNext()) {
             String field = fieldsIterator.next();
-            if (hasCompatibleType(field) == false) {
-                addExcludedField(field, "unsupported type; supported types are " + getSupportedTypes(), fieldSelection);
-                fieldsIterator.remove();
-            }
             Optional<String> matchingNestedFieldPattern = findMatchingNestedFieldPattern(field);
             if (matchingNestedFieldPattern.isPresent()) {
                 addExcludedNestedPattern(matchingNestedFieldPattern.get(), fieldSelection);
                 fieldsIterator.remove();
+            } else if (hasCompatibleType(field) == false) {
+                addExcludedField(field, "unsupported type; supported types are " + getSupportedTypes(), fieldSelection);
+                fieldsIterator.remove();
             }
         }
     }

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java

@@ -949,6 +949,27 @@ public void testDetect_GivenNestedFields() {
         );
     }
 
+    public void testDetect_GivenNestedFieldThatAlsoHasIncompatibleType() {
+        FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
+            .addAggregatableField("float_field", "float")
+            .addNonAggregatableField("nested_field_1", "nested")
+            .addAggregatableField("nested_field_1.a", "definitely_not_supported")
+            .build();
+
+        ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(
+            buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap());
+        Tuple<ExtractedFields, List<FieldSelection>> fieldExtraction = extractedFieldsDetector.detect();
+
+        List<ExtractedField> allFields = fieldExtraction.v1().getAllFields();
+        assertThat(allFields, hasSize(1));
+        assertThat(allFields.get(0).getName(), equalTo("float_field"));
+
+        assertFieldSelectionContains(fieldExtraction.v2(),
+            FieldSelection.included("float_field", Collections.singleton("float"), false, FieldSelection.FeatureType.NUMERICAL),
+            FieldSelection.excluded("nested_field_1.*", Collections.singleton("nested"), "nested fields are not supported")
+        );
+    }
+
     public void testDetect_GivenAnalyzedFieldIncludesObjectField() {
         FieldCapabilitiesResponse fieldCapabilities = new MockFieldCapsResponseBuilder()
             .addAggregatableField("float_field", "float")