Much faster indices lookup on metadata (#123749)

We mostly need the map for lookups in very tight loops but also rarely rely on the sortedness of this thing. Getting the best of both worlds at surprisingly only a ~10% increase in build time and ~25% increase in the heap consumption of this structure provides a massive speedup to e.g. search or field_caps over large numbers of indices thanks to the up to an order of magnitude cheaper lookups in index name resolution and security (as well as speedups in other areas).

Author: original-brownbear

server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java

@@ -46,6 +46,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -1761,15 +1762,129 @@ static SortedMap<String, IndexAbstraction> buildIndicesLookup(
             if (indices.isEmpty()) {
                 return Collections.emptySortedMap();
             }
-            SortedMap<String, IndexAbstraction> indicesLookup = new TreeMap<>();
+            Map<String, IndexAbstraction> indicesLookup = new HashMap<>();
             Map<String, DataStream> indexToDataStreamLookup = new HashMap<>();
             collectDataStreams(dataStreamMetadata, indicesLookup, indexToDataStreamLookup);
 
             Map<String, List<IndexMetadata>> aliasToIndices = new HashMap<>();
             collectIndices(indices, indexToDataStreamLookup, indicesLookup, aliasToIndices);
             collectAliases(aliasToIndices, indicesLookup);
 
-            return Collections.unmodifiableSortedMap(indicesLookup);
+            // We do a ton of lookups on this map but also need its sorted properties at times.
+            // Using this hybrid of a sorted and a hash-map trades some heap overhead relative to just using a TreeMap
+            // for much faster O(1) lookups in large clusters.
+            return new SortedMap<>() {
+
+                private final SortedMap<String, IndexAbstraction> sortedMap = Collections.unmodifiableSortedMap(
+                    new TreeMap<>(indicesLookup)
+                );
+
+                @Override
+                public Comparator<? super String> comparator() {
+                    return sortedMap.comparator();
+                }
+
+                @Override
+                public SortedMap<String, IndexAbstraction> subMap(String fromKey, String toKey) {
+                    return sortedMap.subMap(fromKey, toKey);
+                }
+
+                @Override
+                public SortedMap<String, IndexAbstraction> headMap(String toKey) {
+                    return sortedMap.headMap(toKey);
+                }
+
+                @Override
+                public SortedMap<String, IndexAbstraction> tailMap(String fromKey) {
+                    return sortedMap.tailMap(fromKey);
+                }
+
+                @Override
+                public String firstKey() {
+                    return sortedMap.firstKey();
+                }
+
+                @Override
+                public String lastKey() {
+                    return sortedMap.lastKey();
+                }
+
+                @Override
+                public Set<String> keySet() {
+                    return sortedMap.keySet();
+                }
+
+                @Override
+                public Collection<IndexAbstraction> values() {
+                    return sortedMap.values();
+                }
+
+                @Override
+                public Set<Entry<String, IndexAbstraction>> entrySet() {
+                    return sortedMap.entrySet();
+                }
+
+                @Override
+                public int size() {
+                    return indicesLookup.size();
+                }
+
+                @Override
+                public boolean isEmpty() {
+                    return indicesLookup.isEmpty();
+                }
+
+                @Override
+                public boolean containsKey(Object key) {
+                    return indicesLookup.containsKey(key);
+                }
+
+                @Override
+                public boolean containsValue(Object value) {
+                    return indicesLookup.containsValue(value);
+                }
+
+                @Override
+                public IndexAbstraction get(Object key) {
+                    return indicesLookup.get(key);
+                }
+
+                @Override
+                public IndexAbstraction put(String key, IndexAbstraction value) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public IndexAbstraction remove(Object key) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public void putAll(Map<? extends String, ? extends IndexAbstraction> m) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public void clear() {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public boolean equals(Object obj) {
+                    if (obj == null) {
+                        return false;
+                    }
+                    if (getClass() != obj.getClass()) {
+                        return false;
+                    }
+                    return indicesLookup.equals(obj);
+                }
+
+                @Override
+                public int hashCode() {
+                    return indicesLookup.hashCode();
+                }
+            };
         }
 
         private static void collectAliases(Map<String, List<IndexMetadata>> aliasToIndices, Map<String, IndexAbstraction> indicesLookup) {