[ML] Scale regularisers for final train (#1755)

tveasey · web-flow · commit 0614b01a749e · 2021-02-23T18:05:51.000Z
As we move towards training for hyperparameter tuning on a small fraction of the data set and final training on more
we will suffer issues with overfitting if we don't address the bias this introduces estimating regularisers. Interestingly,
we already see a mismatch in train and test errors on larger data sets where we only use two-folds. I tested this
correction, which is the one we use when we downsample, on a variety of data sets and we ended up with lower
mismatch between train and test errors.
diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc
@@ -45,6 +45,10 @@
 
 * Speed up training of regression and classification model training for data sets
   with many features. (See {ml-pull}1746[#1746].)
+* Avoid overfitting in final training by scaling regularizers to account for the
+  difference in the number of training examples. This results in a better match
+  between train and test error for classification and regression and often slightly
+  improved test errors. (See {ml-pull}1755[#1755].)
 
 == {es} version 7.12.0
 
diff --git a/include/maths/CBoostedTreeImpl.h b/include/maths/CBoostedTreeImpl.h
@@ -311,6 +311,9 @@ class MATHS_EXPORT CBoostedTreeImpl final {
     //! Set the hyperparamaters from the best recorded.
     void restoreBestHyperparameters();
 
+    //! Scale the regulariser multipliers by \p scale.
+    void scaleRegularizers(double scale);
+
     //! Check invariants which are assumed to hold after restoring.
     void checkRestoredInvariants() const;
 
diff --git a/lib/maths/CBoostedTreeFactory.cc b/lib/maths/CBoostedTreeFactory.cc
@@ -769,32 +769,23 @@ void CBoostedTreeFactory::initializeUnsetDownsampleFactor(core::CDataFrame& fram
                     (logMinDownsampleFactor + logMaxDownsampleFactor) / 2.0};
                 LOG_TRACE(<< "mean log downsample factor = " << meanLogDownSampleFactor);
 
-                double previousDownsampleFactor{m_TreeImpl->m_DownsampleFactor};
-                double previousDepthPenaltyMultiplier{
+                double initialDownsampleFactor{m_TreeImpl->m_DownsampleFactor};
+                double initialDepthPenaltyMultiplier{
                     m_TreeImpl->m_Regularization.depthPenaltyMultiplier()};
-                double previousTreeSizePenaltyMultiplier{
+                double initialTreeSizePenaltyMultiplier{
                     m_TreeImpl->m_Regularization.treeSizePenaltyMultiplier()};
-                double previousLeafWeightPenaltyMultiplier{
+                double initialLeafWeightPenaltyMultiplier{
                     m_TreeImpl->m_Regularization.leafWeightPenaltyMultiplier()};
 
                 // We need to scale the regularisation terms to account for the difference
                 // in the downsample factor compared to the value used in the line search.
                 auto scaleRegularizers = [&](CBoostedTreeImpl& tree, double downsampleFactor) {
-                    double scale{previousDownsampleFactor / downsampleFactor};
-                    if (tree.m_RegularizationOverride.depthPenaltyMultiplier() == boost::none) {
-                        tree.m_Regularization.depthPenaltyMultiplier(
-                            scale * previousDepthPenaltyMultiplier);
-                    }
-                    if (tree.m_RegularizationOverride.treeSizePenaltyMultiplier() ==
-                        boost::none) {
-                        tree.m_Regularization.treeSizePenaltyMultiplier(
-                            scale * previousTreeSizePenaltyMultiplier);
-                    }
-                    if (tree.m_RegularizationOverride.leafWeightPenaltyMultiplier() ==
-                        boost::none) {
-                        tree.m_Regularization.leafWeightPenaltyMultiplier(
-                            scale * previousLeafWeightPenaltyMultiplier);
-                    }
+                    double scale{initialDownsampleFactor / downsampleFactor};
+                    tree.m_Regularization.depthPenaltyMultiplier(initialDepthPenaltyMultiplier);
+                    tree.m_Regularization.treeSizePenaltyMultiplier(initialTreeSizePenaltyMultiplier);
+                    tree.m_Regularization.leafWeightPenaltyMultiplier(
+                        initialLeafWeightPenaltyMultiplier);
+                    tree.scaleRegularizers(scale);
                     return scale;
                 };
 
diff --git a/lib/maths/CBoostedTreeImpl.cc b/lib/maths/CBoostedTreeImpl.cc
@@ -261,6 +261,8 @@ void CBoostedTreeImpl::train(core::CDataFrame& frame,
         LOG_TRACE(<< "Test loss = " << m_BestForestTestLoss);
 
         this->restoreBestHyperparameters();
+        this->scaleRegularizers(allTrainingRowsMask.manhattan() /
+                                m_TrainingRowMasks[0].manhattan());
         this->startProgressMonitoringFinalTrain();
         std::tie(m_BestForest, std::ignore, std::ignore) = this->trainForest(
             frame, allTrainingRowsMask, allTrainingRowsMask, m_TrainingProgress);
@@ -1452,6 +1454,21 @@ void CBoostedTreeImpl::restoreBestHyperparameters() {
               << ", feature bag fraction* = " << m_FeatureBagFraction);
 }
 
+void CBoostedTreeImpl::scaleRegularizers(double scale) {
+    if (m_RegularizationOverride.depthPenaltyMultiplier() == boost::none) {
+        m_Regularization.depthPenaltyMultiplier(
+            scale * m_Regularization.depthPenaltyMultiplier());
+    }
+    if (m_RegularizationOverride.treeSizePenaltyMultiplier() == boost::none) {
+        m_Regularization.treeSizePenaltyMultiplier(
+            scale * m_Regularization.treeSizePenaltyMultiplier());
+    }
+    if (m_RegularizationOverride.leafWeightPenaltyMultiplier() == boost::none) {
+        m_Regularization.leafWeightPenaltyMultiplier(
+            scale * m_Regularization.leafWeightPenaltyMultiplier());
+    }
+}
+
 std::size_t CBoostedTreeImpl::numberHyperparametersToTune() const {
     return m_RegularizationOverride.countNotSet() +
            (m_DownsampleFactorOverride != boost::none ? 0 : 1) +
diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc
@@ -882,7 +882,7 @@ BOOST_AUTO_TEST_CASE(testCategoricalRegressors) {
     LOG_DEBUG(<< "bias = " << modelBias);
     LOG_DEBUG(<< " R^2 = " << modelRSquared);
     BOOST_REQUIRE_CLOSE_ABSOLUTE(0.0, modelBias, 0.16);
-    BOOST_TEST_REQUIRE(modelRSquared > 0.97);
+    BOOST_TEST_REQUIRE(modelRSquared > 0.95);
 }
 
 BOOST_AUTO_TEST_CASE(testFeatureBags) {
@@ -1301,13 +1301,13 @@ BOOST_AUTO_TEST_CASE(testBinomialLogisticRegression) {
         LOG_DEBUG(<< "log relative error = "
                   << maths::CBasicStatistics::mean(logRelativeError));
 
-        BOOST_TEST_REQUIRE(maths::CBasicStatistics::mean(logRelativeError) < 0.681);
+        BOOST_TEST_REQUIRE(maths::CBasicStatistics::mean(logRelativeError) < 0.69);
         meanLogRelativeError.add(maths::CBasicStatistics::mean(logRelativeError));
     }
 
     LOG_DEBUG(<< "mean log relative error = "
               << maths::CBasicStatistics::mean(meanLogRelativeError));
-    BOOST_TEST_REQUIRE(maths::CBasicStatistics::mean(meanLogRelativeError) < 0.51);
+    BOOST_TEST_REQUIRE(maths::CBasicStatistics::mean(meanLogRelativeError) < 0.52);
 }
 
 BOOST_AUTO_TEST_CASE(testImbalancedClasses) {
@@ -1389,7 +1389,7 @@ BOOST_AUTO_TEST_CASE(testImbalancedClasses) {
     LOG_DEBUG(<< "recalls    = " << core::CContainerPrinter::print(recalls));
 
     BOOST_TEST_REQUIRE(std::fabs(precisions[0] - precisions[1]) < 0.1);
-    BOOST_TEST_REQUIRE(std::fabs(recalls[0] - recalls[1]) < 0.11);
+    BOOST_TEST_REQUIRE(std::fabs(recalls[0] - recalls[1]) < 0.13);
 }
 
 BOOST_AUTO_TEST_CASE(testClassificationWeightsOverride) {
