[ML] Check invariants after restoration (elastic#1737)

Add checking of invariants after restoration of several classes in the
model library.

Also change a number of conditions where an error was emitted during the
restoration process to be hard stops with LOG_ABORT.

Author: edsavage

lib/model/CDetectorEqualizer.cc

@@ -45,8 +45,7 @@ bool CDetectorEqualizer::acceptRestoreTraverser(core::CStateRestoreTraverser& tr
                                /**/)
         if (name == SKETCH_TAG) {
             if (!detector) {
-                LOG_ERROR(<< "Expected the detector label first");
-                return false;
+                LOG_ABORT(<< "Expected the detector label first");
             }
             m_Sketches.emplace_back(
                 *detector, maths::CQuantileSketch(SKETCH_INTERPOLATION, SKETCH_SIZE));

lib/model/CEventRateBucketGatherer.cc

@@ -231,10 +231,9 @@ bool restoreAttributePeopleData(core::CStateRestoreTraverser& traverser, TSizeUS
                 data.resize(lastCid + 1);
             }
         } else if (name == PERSON_TAG) {
-            if (!seenCid) {
-                LOG_ERROR(<< "Incorrect format - person ID before attribute ID in "
+            if (seenCid == false) {
+                LOG_ABORT(<< "Incorrect format - person ID before attribute ID in "
                           << traverser.value());
-                return false;
             }
             std::size_t pid = 0;
             if (core::CStringUtils::stringToType(traverser.value(), pid) == false) {

lib/model/CIndividualModel.cc

@@ -416,6 +416,8 @@ bool CIndividualModel::doAcceptRestoreTraverser(core::CStateRestoreTraverser& tr
         }
     }
 
+    VIOLATES_INVARIANT(m_FirstBucketTimes.size(), !=, m_LastBucketTimes.size());
+
     return true;
 }
 

lib/model/CPopulationModel.cc

@@ -323,6 +323,9 @@ bool CPopulationModel::doAcceptRestoreTraverser(core::CStateRestoreTraverser& tr
         }
     } while (traverser.next());
 
+    VIOLATES_INVARIANT(m_AttributeFirstBucketTimes.size(), !=,
+                       m_AttributeLastBucketTimes.size());
+
     return true;
 }
 

lib/model/CTokenListCategory.cc

@@ -10,6 +10,7 @@
 #include <core/CStatePersistInserter.h>
 #include <core/CStateRestoreTraverser.h>
 #include <core/CStringUtils.h>
+#include <core/RestoreMacros.h>
 
 #include <functional>
 
@@ -100,9 +101,8 @@ bool CTokenListCategory::acceptRestoreTraverser(core::CStateRestoreTraverser& tr
             m_BaseTokenIds.push_back(tokenAndWeight);
         } else if (name == BASE_TOKEN_WEIGHT) {
             if (m_BaseTokenIds.empty()) {
-                LOG_ERROR(<< "Base token weight precedes base token ID in "
+                LOG_ABORT(<< "Base token weight precedes base token ID in "
                           << traverser.value());
-                return false;
             }
 
             TSizeSizePr& tokenAndWeight = m_BaseTokenIds.back();
@@ -162,12 +162,12 @@ bool CTokenListCategory::acceptRestoreTraverser(core::CStateRestoreTraverser& tr
         } else if (name == ORIG_UNIQUE_TOKEN_WEIGHT) {
             if (core::CStringUtils::stringToType(traverser.value(),
                                                  m_OrigUniqueTokenWeight) == false) {
-                LOG_ERROR(<< "Invalid maximum string length in " << traverser.value());
+                LOG_ERROR(<< "Invalid unique token weight in " << traverser.value());
                 return false;
             }
         } else if (name == NUM_MATCHES) {
             if (core::CStringUtils::stringToType(traverser.value(), m_NumMatches) == false) {
-                LOG_ERROR(<< "Invalid maximum string length in " << traverser.value());
+                LOG_ERROR(<< "Invalid number of matches in " << traverser.value());
                 return false;
             }
         } else if (name == BASE_RAW_STRING_LENGTH) {
@@ -178,6 +178,9 @@ bool CTokenListCategory::acceptRestoreTraverser(core::CStateRestoreTraverser& tr
         }
     } while (traverser.next());
 
+    // Ensure that m_CommonUniqueTokenIds is sorted in ascending order.
+    std::sort(m_CommonUniqueTokenIds.begin(), m_CommonUniqueTokenIds.end(), CTokenIdLess{});
+
     // m_BaseRawStringLen will only have been persisted by 7.9 and above.
     // In this case the absolute maximum set at the beginning of the method
     // will still be set.  A reasonable compromise that will result in