[7.8][ML] Improve bad input handling in distribution models (#1119)

Backport #1114.

Author: tveasey

docs/CHANGELOG.asciidoc

@@ -38,6 +38,7 @@
   (See {ml-pull}1111[1111].)
 * Adds new `num_matches` and `preferred_to_categories` fields to category output.
   (See {ml-pull}1062[#1062])
+* Improve robustness of anomaly detection to bad input data. (See {ml-pull}1114[#1114].)
 
 == {es} version 7.7.0
 

lib/maths/CGammaRateConjugate.cc

@@ -874,13 +874,15 @@ void CGammaRateConjugate::addSamples(const TDouble1Vec& samples,
     try {
         double shift = boost::math::digamma(m_LikelihoodShape);
         for (std::size_t i = 0u; i < samples.size(); ++i) {
+            double x = samples[i] + m_Offset;
             double n = maths_t::countForUpdate(weights[i]);
             double varianceScale = maths_t::seasonalVarianceScale(weights[i]) *
                                    maths_t::countVarianceScale(weights[i]);
 
-            double x = samples[i] + m_Offset;
-            if (!CMathsFuncs::isFinite(x) || x <= 0.0) {
-                LOG_ERROR(<< "Discarding " << x << " it's not gamma");
+            if (x <= 0.0 || !CMathsFuncs::isFinite(x) || !CMathsFuncs::isFinite(n) ||
+                !CMathsFuncs::isFinite(varianceScale)) {
+                LOG_ERROR(<< "Discarding sample = " << x << ", weight = " << n
+                          << ", variance scale = " << varianceScale);
                 continue;
             }
 

lib/maths/CLogNormalMeanPrecConjugate.cc

@@ -576,12 +576,13 @@ class CLogSampleSquareDeviation : core::CNonCopyable {
     bool operator()(double x, double& result) const {
         result = 0.0;
         for (std::size_t i = 0u; i < m_Samples.size(); ++i) {
-            double residual = m_Samples[i];
-            if (residual <= 0.0) {
+            double residual = m_Samples[i] + x;
+            double n = maths_t::countForUpdate(m_Weights[i]);
+            if (residual <= 0.0 || !CMathsFuncs::isFinite(residual) ||
+                !CMathsFuncs::isFinite(n)) {
                 continue;
             }
-            double n = maths_t::countForUpdate(m_Weights[i]);
-            residual = std::log(residual + x) - m_Mean;
+            residual = std::log(residual) - m_Mean;
             result += n * CTools::pow2(residual);
         }
         return true;
@@ -779,10 +780,17 @@ void CLogNormalMeanPrecConjugate::addSamples(const TDouble1Vec& samples,
 
         TMeanAccumulator logSamplesMean_;
         for (std::size_t i = 0u; i < samples.size(); ++i) {
+            double x = samples[i] + m_Offset;
             double n = maths_t::countForUpdate(weights[i]);
             double varianceScale = maths_t::seasonalVarianceScale(weights[i]) *
                                    maths_t::countVarianceScale(weights[i]);
-            double x = samples[i] + m_Offset;
+            if (x <= 0.0 || !CMathsFuncs::isFinite(x) || !CMathsFuncs::isFinite(n) ||
+                !CMathsFuncs::isFinite(varianceScale)) {
+                LOG_ERROR(<< "Discarding sample = " << x << ", weight = " << n
+                          << ", variance scale = " << varianceScale);
+                continue;
+            }
+
             numberSamples += n;
             double t = varianceScale == 1.0
                            ? r
@@ -814,14 +822,17 @@ void CLogNormalMeanPrecConjugate::addSamples(const TDouble1Vec& samples,
     } else {
         TMeanVarAccumulator logSamplesMoments;
         for (std::size_t i = 0u; i < samples.size(); ++i) {
+            double x = samples[i] + m_Offset;
             double n = maths_t::countForUpdate(weights[i]);
             double varianceScale = maths_t::seasonalVarianceScale(weights[i]) *
                                    maths_t::countVarianceScale(weights[i]);
-            double x = samples[i] + m_Offset;
-            if (x <= 0.0) {
-                LOG_ERROR(<< "Discarding " << x << " it's not log-normal");
+            if (x <= 0.0 || !CMathsFuncs::isFinite(x) || !CMathsFuncs::isFinite(n) ||
+                !CMathsFuncs::isFinite(varianceScale)) {
+                LOG_ERROR(<< "Discarding sample = " << x << ", weight = " << n
+                          << ", variance scale = " << varianceScale);
                 continue;
             }
+
             numberSamples += n;
             double t = varianceScale == 1.0
                            ? r

lib/maths/CNormalMeanPrecConjugate.cc

@@ -613,9 +613,16 @@ void CNormalMeanPrecConjugate::addSamples(const TDouble1Vec& samples,
     double numberSamples = 0.0;
     TMeanVarAccumulator sampleMoments;
     for (std::size_t i = 0u; i < samples.size(); ++i) {
+        double x = samples[i];
         double n = maths_t::countForUpdate(weights[i]);
         double varianceScale = maths_t::seasonalVarianceScale(weights[i]) *
                                maths_t::countVarianceScale(weights[i]);
+        if (!CMathsFuncs::isFinite(x) || !CMathsFuncs::isFinite(n) ||
+            !CMathsFuncs::isFinite(varianceScale)) {
+            LOG_ERROR(<< "Discarding sample = " << x << ", weight = " << n
+                      << ", variance scale = " << varianceScale);
+            continue;
+        }
         numberSamples += n;
         sampleMoments.add(samples[i], n / varianceScale);
     }

lib/maths/COneOfNPrior.cc

@@ -310,7 +310,11 @@ void COneOfNPrior::addSamples(const TDouble1Vec& samples,
     n = this->numberSamples() - n;
 
     for (std::size_t i = 0; i < samples.size(); ++i) {
-        m_SampleMoments.add(samples[i], maths_t::countForUpdate(weights[i]));
+        double xi = samples[i];
+        double ni = maths_t::countForUpdate(weights[i]);
+        if (CMathsFuncs::isFinite(xi) && CMathsFuncs::isFinite(ni)) {
+            m_SampleMoments.add(xi, ni);
+        }
     }
 
     // For this 1-of-n model we assume that all the data come from one

lib/maths/CPoissonMeanConjugate.cc

@@ -332,10 +332,10 @@ void CPoissonMeanConjugate::addSamples(const TDouble1Vec& samples,
     double numberSamples = 0.0;
     double sampleSum = 0.0;
     for (std::size_t i = 0u; i < samples.size(); ++i) {
-        double n = maths_t::countForUpdate(weights[i]);
         double x = samples[i] + m_Offset;
-        if (!CMathsFuncs::isFinite(x) || x < 0.0) {
-            LOG_ERROR(<< "Discarding " << x << " it's not Poisson");
+        double n = maths_t::countForUpdate(weights[i]);
+        if (x < 0.0 || !CMathsFuncs::isFinite(x) || !CMathsFuncs::isFinite(n)) {
+            LOG_ERROR(<< "Discarding sample = " << x << ", weight = " << n);
             continue;
         }
         numberSamples += n;

lib/maths/CPrior.cc

@@ -92,11 +92,15 @@ double CPrior::offsetMargin() const {
     return 0.0;
 }
 
-void CPrior::addSamples(const TDouble1Vec& /*samples*/, const TDoubleWeightsAry1Vec& weights) {
+void CPrior::addSamples(const TDouble1Vec& samples, const TDoubleWeightsAry1Vec& weights) {
     double n = 0.0;
     try {
-        for (const auto& weight : weights) {
-            n += maths_t::countForUpdate(weight);
+        for (std::size_t i = 0; i < weights.size(); ++i) {
+            double xi = samples[i];
+            double ni = maths_t::countForUpdate(weights[i]);
+            if (CMathsFuncs::isFinite(xi) && CMathsFuncs::isFinite(ni)) {
+                n += ni;
+            }
         }
     } catch (const std::exception& e) {
         LOG_ERROR(<< "Failed to extract sample counts: " << e.what());