[ML] Line search feature bag fraction for classification and regression model training (#1761)

Following on from #1733, we can get further speedups by line searching for the best feature bag fraction for data sets
where we only need a fraction of the features per tree. For example, training time on Higgs 1M drops from 2585s to
1742s and we actually get a small improvement in accuracy because our hyperparameter search region is better
initialised.

This makes three changes:
1. Adds a line search for the best initial feature bag fraction to use.
2. Adds a small linear penalty at most 1% minimum loss to encourage larger down sample factors and smaller feature
   bag fractions.
3. Handles better the case we have many features and relatively few training examples.

Author: tveasey

docs/CHANGELOG.asciidoc

@@ -28,6 +28,13 @@
 
 //=== Regressions
 
+== {es} version 7.13.0
+
+=== Enhancements
+
+* Speed up training of regression and classification model training for data sets
+  with many features. (See {ml-pull}1746[#1746].)
+
 == {es} version 7.12.0
 
 === Enhancements

include/maths/CBoostedTreeFactory.h

@@ -119,7 +119,7 @@ class MATHS_EXPORT CBoostedTreeFactory final {
     //! Set the number of training examples we need per feature we'll include.
     CBoostedTreeFactory& numberTopShapValues(std::size_t numberTopShapValues);
     //! Set the flag to enable or disable early stopping.
-    CBoostedTreeFactory& earlyStoppingEnabled(bool earlyStoppingEnabled);
+    CBoostedTreeFactory& earlyStoppingEnabled(bool enable);
 
     //! Set pointer to the analysis instrumentation.
     CBoostedTreeFactory&
@@ -147,7 +147,8 @@ class MATHS_EXPORT CBoostedTreeFactory final {
     using TOptionalVector = boost::optional<TVector>;
     using TPackedBitVectorVec = std::vector<core::CPackedBitVector>;
     using TBoostedTreeImplUPtr = std::unique_ptr<CBoostedTreeImpl>;
-    using TApplyRegularizer = std::function<bool(CBoostedTreeImpl&, double)>;
+    using TApplyParameter = std::function<bool(CBoostedTreeImpl&, double)>;
+    using TAdjustTestLoss = std::function<double(double, double, double)>;
 
 private:
     CBoostedTreeFactory(std::size_t numberThreads, TLossFunctionUPtr loss);
@@ -190,6 +191,9 @@ class MATHS_EXPORT CBoostedTreeFactory final {
     //! search bounding box.
     void initializeUnsetRegularizationHyperparameters(core::CDataFrame& frame);
 
+    //! Estimate a good central value for the feature bag fraction search interval.
+    void initializeUnsetFeatureBagFraction(core::CDataFrame& frame);
+
     //! Estimates a good central value for the downsample factor search interval.
     void initializeUnsetDownsampleFactor(core::CDataFrame& frame);
 
@@ -208,11 +212,12 @@ class MATHS_EXPORT CBoostedTreeFactory final {
     //! \return The interval to search during the main hyperparameter optimisation
     //! loop or null if this couldn't be found.
     TOptionalVector testLossLineSearch(core::CDataFrame& frame,
-                                       const TApplyRegularizer& applyRegularizerStep,
+                                       const TApplyParameter& applyParameterStep,
                                        double intervalLeftEnd,
                                        double intervalRightEnd,
                                        double returnedIntervalLeftEndOffset,
-                                       double returnedIntervalRightEndOffset) const;
+                                       double returnedIntervalRightEndOffset,
+                                       const TAdjustTestLoss& adjustTestLoss = noopAdjustTestLoss) const;
 
     //! Initialize the state for hyperparameter optimisation.
     void initializeHyperparameterOptimisation() const;
@@ -264,8 +269,8 @@ class MATHS_EXPORT CBoostedTreeFactory final {
     //! Stubs out persistence.
     static void noopRecordTrainingState(CBoostedTree::TPersistFunc);
 
-    //! Stop hyperparameter optimization early if the process is not promising.
-    void stopHyperparameterOptimizationEarly(bool stopEarly);
+    //! Stubs out test loss adjustment.
+    static double noopAdjustTestLoss(double, double, double testLoss);
 
 private:
     TOptionalDouble m_MinimumFrequencyToOneHotEncode;
@@ -280,6 +285,7 @@ class MATHS_EXPORT CBoostedTreeFactory final {
     std::size_t m_NumberThreads;
     TBoostedTreeImplUPtr m_TreeImpl;
     TVector m_LogDownsampleFactorSearchInterval;
+    TVector m_LogFeatureBagFractionInterval;
     TVector m_LogDepthPenaltyMultiplierSearchInterval;
     TVector m_LogTreeSizePenaltyMultiplierSearchInterval;
     TVector m_LogLeafWeightPenaltyMultiplierSearchInterval;

include/maths/CBoostedTreeImpl.h

@@ -192,8 +192,9 @@ class MATHS_EXPORT CBoostedTreeImpl final {
         E_TreeSizePenaltyMultiplierInitialized = 3,
         E_LeafWeightPenaltyMultiplierInitialized = 4,
         E_DownsampleFactorInitialized = 5,
-        E_EtaInitialized = 6,
-        E_FullyInitialized = 7
+        E_FeatureBagFractionInitialized = 6,
+        E_EtaInitialized = 7,
+        E_FullyInitialized = 8
     };
 
 private: