[ML] Fix off-by-one error in usurped categories

The mistake was that m_CategoriesByCount has indexes into
m_Categories, not category IDs (which are one more than
the indices so that they start at 1 rather than 0).

Fixes elastic#1121

Author: droberts195

include/model/CTokenListDataCategorizerBase.h

@@ -324,7 +324,8 @@ class MODEL_EXPORT CTokenListDataCategorizerBase : public CDataCategorizer {
     TTokenListCategoryVec m_Categories;
 
     //! List of match count/index into category vector in descending order of
-    //! match count
+    //! match count.  Note that the second element is an index into m_Categories,
+    //! not a category ID.
     TSizeSizePrVec m_CategoriesByCount;
 
     //! Used for looking up tokens to a unique ID

lib/model/CTokenListDataCategorizerBase.cc

@@ -200,15 +200,18 @@ bool CTokenListDataCategorizerBase::createReverseSearch(int categoryId,
             return false;
         }
 
+        // Ensure all reference arguments are set before returning true
+        wasCached = false;
+        maxMatchingLength = 0;
         return true;
     }
 
     CTokenListCategory& category{m_Categories[categoryId - 1]};
     maxMatchingLength = category.maxMatchingStringLen();
 
     // If we can retrieve cached reverse search terms we'll save a lot of time
-    if (category.cachedReverseSearch(part1, part2) == true) {
-        wasCached = true;
+    wasCached = category.cachedReverseSearch(part1, part2);
+    if (wasCached) {
         return true;
     }
 
@@ -634,27 +637,28 @@ CDataCategorizer::TIntVec CTokenListDataCategorizerBase::usurpedCategories(int c
     }
     auto iter = std::find_if(m_CategoriesByCount.begin(), m_CategoriesByCount.end(),
                              [categoryId](const TSizeSizePr& pr) {
-                                 return pr.second == static_cast<std::size_t>(categoryId);
+                                 return pr.second ==
+                                        static_cast<std::size_t>(categoryId - 1);
                              });
     if (iter == m_CategoriesByCount.end()) {
         LOG_WARN(<< "Could not find category definition for category: " << categoryId);
         return usurped;
     }
-    ++iter;
+
     const CTokenListCategory& category{m_Categories[categoryId - 1]};
-    for (; iter != m_CategoriesByCount.end(); ++iter) {
-        const CTokenListCategory& lessFrequentCategory{
-            m_Categories[static_cast<int>(iter->second) - 1]};
+    for (++iter; iter != m_CategoriesByCount.end(); ++iter) {
+        const CTokenListCategory& lessFrequentCategory{m_Categories[iter->second]};
         bool matchesSearch{category.maxMatchingStringLen() >=
                                lessFrequentCategory.maxMatchingStringLen() &&
                            category.isMissingCommonTokenWeightZero(
                                lessFrequentCategory.commonUniqueTokenIds()) &&
                            category.containsCommonInOrderTokensInOrder(
                                lessFrequentCategory.baseTokenIds())};
         if (matchesSearch) {
-            usurped.emplace_back(static_cast<int>(iter->second));
+            usurped.emplace_back(1 + static_cast<int>(iter->second));
         }
     }
+    std::sort(usurped.begin(), usurped.end());
     return usurped;
 }
 

lib/model/unittest/CTokenListDataCategorizerTest.cc

@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+#include <core/CContainerPrinter.h>
 #include <core/CLogger.h>
 #include <core/CRapidXmlParser.h>
 #include <core/CRapidXmlStatePersistInserter.h>
@@ -542,13 +543,13 @@ BOOST_FIXTURE_TEST_CASE(testUsurpedCategories, CTestFixture) {
                                                        500));
 
     BOOST_REQUIRE_EQUAL(2, categorizer.numMatches(1));
-    std::vector<int> expected{2, 3, 4, 5, 6};
-    std::vector<int> actual = categorizer.usurpedCategories(1);
 
-    BOOST_REQUIRE_EQUAL(expected.size(), actual.size());
-    for (std::size_t i = 0; i < actual.size(); i++) {
-        BOOST_REQUIRE_EQUAL(expected[i], actual[i]);
-    }
+    using TIntVec = std::vector<int>;
+    TIntVec expected{2, 3, 4, 5, 6, 7};
+    TIntVec actual{categorizer.usurpedCategories(1)};
+
+    BOOST_REQUIRE_EQUAL(ml::core::CContainerPrinter::print(expected),
+                        ml::core::CContainerPrinter::print(actual));
     checkMemoryUsageInstrumentation(categorizer);
 }