From 0ffe193b4a6b729338024cf4e7c7d854ab9291c0 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 1 Oct 2019 18:38:53 +0100 Subject: [PATCH 01/20] Logistic regression loss function --- include/maths/CBoostedTree.h | 85 +++++++++- include/maths/CBoostedTreeFactory.h | 2 + include/maths/CTools.h | 3 +- lib/maths/CBoostedTree.cc | 179 +++++++++++++++++++- lib/maths/CBoostedTreeFactory.cc | 16 +- lib/maths/CBoostedTreeImpl.cc | 53 +++--- lib/maths/unittest/CBoostedTreeTest.cc | 225 ++++++++++++++++++++++++- lib/maths/unittest/CBoostedTreeTest.h | 2 + 8 files changed, 532 insertions(+), 33 deletions(-) diff --git a/include/maths/CBoostedTree.h b/include/maths/CBoostedTree.h index c460d4a99b..c16af92003 100644 --- a/include/maths/CBoostedTree.h +++ b/include/maths/CBoostedTree.h @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -29,18 +30,28 @@ class CEncodedDataFrameRowRef; namespace boosted_tree_detail { class MATHS_EXPORT CArgMinLossImpl { public: + CArgMinLossImpl(double lambda); virtual ~CArgMinLossImpl() = default; virtual std::unique_ptr clone() const = 0; + virtual bool nextPass() = 0; virtual void add(double prediction, double actual) = 0; virtual void merge(const CArgMinLossImpl& other) = 0; virtual double value() const = 0; + +protected: + double lambda() const; + +private: + double m_Lambda; }; //! \brief Finds the value to add to a set of predictions which minimises the MSE. class MATHS_EXPORT CArgMinMseImpl final : public CArgMinLossImpl { public: + CArgMinMseImpl(double lambda); std::unique_ptr clone() const override; + bool nextPass() override; void add(double prediction, double actual) override; void merge(const CArgMinLossImpl& other) override; double value() const override; @@ -51,6 +62,46 @@ class MATHS_EXPORT CArgMinMseImpl final : public CArgMinLossImpl { private: TMeanAccumulator m_MeanError; }; + +//! \brief Finds the value to add to the argument of the logistic function which +//! minimises the cross entropy loss. +class MATHS_EXPORT CArgMinLogisticImpl final : public CArgMinLossImpl { +public: + CArgMinLogisticImpl(double lambda); + std::unique_ptr clone() const override; + bool nextPass() override; + void add(double prediction, double actual) override; + void merge(const CArgMinLossImpl& other) override; + double value() const override; + +private: + using TMinMaxAccumulator = CBasicStatistics::CMinMax; + using TSizeVector = CVectorNx1; + using TSizeVectorVec = std::vector; + +private: + std::size_t bucket(double prediction) const { + double bucket{(prediction - m_MinMaxPrediction.min()) / this->bucketWidth()}; + return std::min(static_cast(bucket), + m_BucketCategoryCounts.size() - 1); + } + + double bucketCentre(std::size_t bucket) const { + return m_MinMaxPrediction.min() + + (static_cast(bucket) + 0.5) * this->bucketWidth(); + } + + double bucketWidth() const { + return m_MinMaxPrediction.range() / + static_cast(m_BucketCategoryCounts.size()); + } + +private: + std::size_t m_CurrentPass = 0; + TMinMaxAccumulator m_MinMaxPrediction; + TSizeVector m_CategoryCounts; + TSizeVectorVec m_BucketCategoryCounts; +}; } namespace boosted_tree { @@ -64,6 +115,9 @@ class MATHS_EXPORT CArgMinLoss { CArgMinLoss& operator=(const CArgMinLoss& other); CArgMinLoss& operator=(CArgMinLoss&& other) = default; + //! The number of passes over the data this needs. + bool nextPass() const; + //! Update with a point prediction and actual value. void add(double prediction, double actual); @@ -94,6 +148,8 @@ class MATHS_EXPORT CArgMinLoss { class MATHS_EXPORT CLoss { public: virtual ~CLoss() = default; + //! Clone the loss. + virtual std::unique_ptr clone() const = 0; //! The value of the loss function. virtual double value(double prediction, double actual) const = 0; //! The slope of the loss function. @@ -103,7 +159,7 @@ class MATHS_EXPORT CLoss { //! Returns true if the loss curvature is constant. virtual bool isCurvatureConstant() const = 0; //! Get an object which computes the leaf value that minimises loss. - virtual CArgMinLoss minimizer() const = 0; + virtual CArgMinLoss minimizer(double lambda) const = 0; //! Get the name of the loss function virtual const std::string& name() const = 0; @@ -114,11 +170,36 @@ class MATHS_EXPORT CLoss { //! \brief The MSE loss function. class MATHS_EXPORT CMse final : public CLoss { public: + std::unique_ptr clone() const override; + double value(double prediction, double actual) const override; + double gradient(double prediction, double actual) const override; + double curvature(double prediction, double actual) const override; + bool isCurvatureConstant() const override; + CArgMinLoss minimizer(double lambda) const override; + const std::string& name() const override; + +public: + static const std::string NAME; +}; + +//! \brief Implements loss for logistic regression for binary classification. +//! +//! DESCRIPTION:\n +//! This targets the cross entropy loss using the logistic function of the sum of +//! the of the tree predictions to estimate the probability of one of the classes +//! for a binary classification task +//! 
+//!   \f$\displaystyle l_i(p) = -(1 - a_i) \log(1 - S(p)) - a_i \log(S(p))\f$
+//!
+//! where \f$a_i\f$ denotes the actual class of the i'th example, \f$p\f$ is the +//! prediction and \f$S(\cdot)\f$ denotes the logistic function. +class MATHS_EXPORT CLogistic final : public CLoss { + std::unique_ptr clone() const override; double value(double prediction, double actual) const override; double gradient(double prediction, double actual) const override; double curvature(double prediction, double actual) const override; bool isCurvatureConstant() const override; - CArgMinLoss minimizer() const override; + CArgMinLoss minimizer(double lambda) const override; const std::string& name() const override; public: diff --git a/include/maths/CBoostedTreeFactory.h b/include/maths/CBoostedTreeFactory.h index 01582d8d10..5f4a76f3b3 100644 --- a/include/maths/CBoostedTreeFactory.h +++ b/include/maths/CBoostedTreeFactory.h @@ -177,6 +177,8 @@ class MATHS_EXPORT CBoostedTreeFactory final { TOptionalDouble m_MinimumFrequencyToOneHotEncode; TOptionalSize m_BayesianOptimisationRestarts; bool m_Restored = false; + std::size_t m_NumberThreads; + TLossFunctionUPtr m_Loss; TBoostedTreeImplUPtr m_TreeImpl; TVector m_LogDepthPenaltyMultiplierSearchInterval; TVector m_LogTreeSizePenaltyMultiplierSearchInterval; diff --git a/include/maths/CTools.h b/include/maths/CTools.h index 079a9304c6..d51f760564 100644 --- a/include/maths/CTools.h +++ b/include/maths/CTools.h @@ -678,7 +678,8 @@ class MATHS_EXPORT CTools : private core::CNonInstantiatable { //! \param[in] width The step width. //! \param[in] x0 The centre of the step. //! \param[in] sign Determines whether it's a step up or down. - static double logisticFunction(double x, double width, double x0 = 0.0, double sign = 1.0) { + static double + logisticFunction(double x, double width = 1.0, double x0 = 0.0, double sign = 1.0) { return sigmoid(std::exp(std::copysign(1.0, sign) * (x - x0) / width)); } diff --git a/lib/maths/CBoostedTree.cc b/lib/maths/CBoostedTree.cc index 7c9f1c4528..0049edf47c 100644 --- a/lib/maths/CBoostedTree.cc +++ b/lib/maths/CBoostedTree.cc @@ -10,6 +10,11 @@ #include #include +#include +#include +#include + +#include #include #include @@ -27,10 +32,24 @@ const std::string SPLIT_VALUE_TAG{"split_value"}; namespace boosted_tree_detail { +CArgMinLossImpl::CArgMinLossImpl(double lambda) : m_Lambda{lambda} { +} + +double CArgMinLossImpl::lambda() const { + return m_Lambda; +} + +CArgMinMseImpl::CArgMinMseImpl(double lambda) : CArgMinLossImpl{lambda} { +} + std::unique_ptr CArgMinMseImpl::clone() const { return std::make_unique(*this); } +bool CArgMinMseImpl::nextPass() { + return false; +} + void CArgMinMseImpl::add(double prediction, double actual) { m_MeanError.add(actual - prediction); } @@ -43,7 +62,118 @@ void CArgMinMseImpl::merge(const CArgMinLossImpl& other) { } double CArgMinMseImpl::value() const { - return CBasicStatistics::mean(m_MeanError); + double count{CBasicStatistics::count(m_MeanError)}; + return count == 0.0 + ? 0.0 + : count / (count + this->lambda()) * CBasicStatistics::mean(m_MeanError); +} + +CArgMinLogisticImpl::CArgMinLogisticImpl(double lambda) + : CArgMinLossImpl{lambda}, m_CategoryCounts{0}, + m_BucketCategoryCounts(128, TSizeVector{0}) { +} + +std::unique_ptr CArgMinLogisticImpl::clone() const { + return std::make_unique(*this); +} + +bool CArgMinLogisticImpl::nextPass() { + m_CurrentPass += this->bucketWidth() > 0.0 ? 1 : 2; + return m_CurrentPass < 2; +} + +void CArgMinLogisticImpl::add(double prediction, double actual) { + switch (m_CurrentPass) { + case 0: { + m_MinMaxPrediction.add(prediction); + ++m_CategoryCounts(static_cast(actual)); + break; + } + case 1: { + auto& count = m_BucketCategoryCounts[this->bucket(prediction)]; + ++count(static_cast(actual)); + break; + } + default: + break; + } +} + +void CArgMinLogisticImpl::merge(const CArgMinLossImpl& other) { + const auto* logistic = dynamic_cast(&other); + if (logistic != nullptr) { + switch (m_CurrentPass) { + case 0: + m_MinMaxPrediction += logistic->m_MinMaxPrediction; + m_CategoryCounts += logistic->m_CategoryCounts; + break; + case 1: + for (std::size_t i = 0; i < m_BucketCategoryCounts.size(); ++i) { + m_BucketCategoryCounts[i] += logistic->m_BucketCategoryCounts[i]; + } + break; + default: + break; + } + } +} + +double CArgMinLogisticImpl::value() const { + + std::function objective; + double minWeight; + double maxWeight; + + if (this->bucketWidth() == 0.0) { + objective = [this](double weight) { + double p{CTools::logisticFunction(weight)}; + std::size_t c0{m_CategoryCounts(0)}; + std::size_t c1{m_CategoryCounts(1)}; + return this->lambda() * CTools::pow2(weight) - + static_cast(c0) * CTools::fastLog(1.0 - p) - + static_cast(c1) * CTools::fastLog(p); + }; + + // Weight shrinkage means the optimal weight will be somewhere + // between the logit of the empirical probability and zero. + std::size_t c0{m_CategoryCounts(0) + 1}; + std::size_t c1{m_CategoryCounts(1) + 1}; + double p{static_cast(c1) / static_cast(c0 + c1)}; + minWeight = p < 0.5 ? std::log(p / (1.0 - p)) : 0.0; + maxWeight = p < 0.5 ? 0.0 : std::log(p / (1.0 - p)); + + } else { + objective = [this](double weight) { + double loss{0.0}; + for (std::size_t i = 0; i < m_BucketCategoryCounts.size(); ++i) { + double bucketPrediction{this->bucketCentre(i)}; + double p{CTools::logisticFunction(bucketPrediction + weight)}; + std::size_t c0{m_BucketCategoryCounts[i](0)}; + std::size_t c1{m_BucketCategoryCounts[i](1)}; + loss -= static_cast(c0) * CTools::fastLog(1.0 - p) + + static_cast(c1) * CTools::fastLog(p); + } + return loss + this->lambda() * CTools::pow2(weight); + }; + + // Choose a weight interval in which all probabilites vary from close to + // zero to close to one. + minWeight = -m_MinMaxPrediction.max() - 2.0; + maxWeight = -m_MinMaxPrediction.min() + 2.0; + } + + if (minWeight == maxWeight) { + return minWeight; + } + + double minimum; + double objectiveAtMinimum; + std::size_t maxIterations{10}; + CSolvers::minimize(minWeight, maxWeight, objective(minWeight), objective(maxWeight), + objective, 1e-3, maxIterations, minimum, objectiveAtMinimum); + LOG_TRACE(<< "minimum = " << minimum << " objective(minimum) = " << objectiveAtMinimum); + + return minimum; } } @@ -61,6 +191,10 @@ CArgMinLoss& CArgMinLoss::operator=(const CArgMinLoss& other) { return *this; } +bool CArgMinLoss::nextPass() const { + return m_Impl->nextPass(); +} + void CArgMinLoss::add(double prediction, double actual) { return m_Impl->add(prediction, actual); } @@ -80,6 +214,10 @@ CArgMinLoss CLoss::makeMinimizer(const boosted_tree_detail::CArgMinLossImpl& imp return {impl}; } +std::unique_ptr CMse::clone() const { + return std::make_unique(*this); +} + double CMse::value(double prediction, double actual) const { return CTools::pow2(prediction - actual); } @@ -96,8 +234,8 @@ bool CMse::isCurvatureConstant() const { return true; } -CArgMinLoss CMse::minimizer() const { - return this->makeMinimizer(CArgMinMseImpl{}); +CArgMinLoss CMse::minimizer(double lambda) const { + return this->makeMinimizer(CArgMinMseImpl{lambda}); } const std::string& CMse::name() const { @@ -105,6 +243,41 @@ const std::string& CMse::name() const { } const std::string CMse::NAME{"mse"}; + +std::unique_ptr CLogistic::clone() const { + return std::make_unique(*this); +} + +double CLogistic::value(double prediction, double actual) const { + // Cross entropy + prediction = CTools::logisticFunction(prediction); + return -((1.0 - actual) * CTools::fastLog(1.0 - prediction) + + actual * CTools::fastLog(prediction)); +} + +double CLogistic::gradient(double prediction, double actual) const { + prediction = CTools::logisticFunction(prediction); + return prediction - actual; +} + +double CLogistic::curvature(double prediction, double /*actual*/) const { + prediction = CTools::logisticFunction(prediction); + return prediction * (1.0 - prediction); +} + +bool CLogistic::isCurvatureConstant() const { + return false; +} + +CArgMinLoss CLogistic::minimizer(double lambda) const { + return this->makeMinimizer(CArgMinLogisticImpl{lambda}); +} + +const std::string& CLogistic::name() const { + return NAME; +} + +const std::string CLogistic::NAME{"logistic"}; } std::size_t CBoostedTreeNode::leafIndex(const CEncodedDataFrameRowRef& row, diff --git a/lib/maths/CBoostedTreeFactory.cc b/lib/maths/CBoostedTreeFactory.cc index e4b7e2e7cf..b302660cbb 100644 --- a/lib/maths/CBoostedTreeFactory.cc +++ b/lib/maths/CBoostedTreeFactory.cc @@ -80,9 +80,10 @@ CBoostedTreeFactory::buildFor(core::CDataFrame& frame, std::size_t dependentVari } } - // TODO can only use factory to create one object since this is moved. This seems trappy. + auto treeImpl = std::make_unique(m_NumberThreads, m_Loss->clone()); + std::swap(m_TreeImpl, treeImpl); return TBoostedTreeUPtr{new CBoostedTree{frame, m_RecordProgress, m_RecordMemoryUsage, - m_RecordTrainingState, std::move(m_TreeImpl)}}; + m_RecordTrainingState, std::move(treeImpl)}}; } std::size_t CBoostedTreeFactory::numberHyperparameterTuningRounds() const { @@ -534,7 +535,6 @@ CBoostedTreeFactory::testLossLineSearch(core::CDataFrame& frame, double testLoss{m_TreeImpl->meanLoss(frame, testRowMask, forest)}; leastSquaresQuadraticTestLoss.add(static_cast(i) * stepSize, testLoss); testLosses[i] = testLoss; - m_TreeImpl->m_TrainingProgress.increment(); } LOG_TRACE(<< "test losses = " << core::CContainerPrinter::print(testLosses)); @@ -616,8 +616,8 @@ CBoostedTreeFactory CBoostedTreeFactory::constructFromString(std::istream& jsonS } CBoostedTreeFactory::CBoostedTreeFactory(bool restored, std::size_t numberThreads, TLossFunctionUPtr loss) - : m_Restored{restored}, m_TreeImpl{std::make_unique(numberThreads, - std::move(loss))}, + : m_Restored{restored}, m_NumberThreads{numberThreads}, m_Loss{loss->clone()}, + m_TreeImpl{std::make_unique(numberThreads, std::move(loss))}, m_LogDepthPenaltyMultiplierSearchInterval{0.0}, m_LogTreeSizePenaltyMultiplierSearchInterval{0.0}, m_LogLeafWeightPenaltyMultiplierSearchInterval{0.0} { } @@ -793,12 +793,16 @@ void CBoostedTreeFactory::initializeTrainingProgressMonitoring() { // This comprises: // - The cost of category encoding and feature selection which we count as // one unit, + // - One unit for estimating the expected gain and sum curvature per node, // - INITIAL_REGULARIZER_SEARCH_ITERATIONS units per regularization parameter // which isn't user defined, // - The main optimisation loop which costs number folds units per iteration, // - The cost of the final train which we count as number folds units. - std::size_t totalNumberSteps{1}; + std::size_t totalNumberSteps{2}; + if (m_TreeImpl->m_RegularizationOverride.depthPenaltyMultiplier() == boost::none) { + totalNumberSteps += INITIAL_REGULARIZER_SEARCH_ITERATIONS; + } if (m_TreeImpl->m_RegularizationOverride.treeSizePenaltyMultiplier() == boost::none) { totalNumberSteps += INITIAL_REGULARIZER_SEARCH_ITERATIONS; } diff --git a/lib/maths/CBoostedTreeImpl.cc b/lib/maths/CBoostedTreeImpl.cc index 692aff017c..ecdc68edb9 100644 --- a/lib/maths/CBoostedTreeImpl.cc +++ b/lib/maths/CBoostedTreeImpl.cc @@ -433,7 +433,6 @@ CBoostedTreeImpl::crossValidateForest(core::CDataFrame& frame, lossMoments.add(loss); LOG_TRACE(<< "fold = " << i << " forest size = " << forest.size() << " test set loss = " << loss); - m_TrainingProgress.increment(); } LOG_TRACE(<< "test mean loss = " << CBasicStatistics::mean(lossMoments) << ", sigma = " << std::sqrt(CBasicStatistics::mean(lossMoments))); @@ -521,6 +520,8 @@ CBoostedTreeImpl::trainForest(core::CDataFrame& frame, LOG_TRACE(<< "Trained one forest"); + m_TrainingProgress.increment(); + return forest; } @@ -733,27 +734,39 @@ void CBoostedTreeImpl::refreshPredictionsAndLossDerivatives(core::CDataFrame& fr using TArgMinLossVec = std::vector; - auto result = frame.readRows( - m_NumberThreads, 0, frame.numberRows(), - core::bindRetrievableState( - [&](TArgMinLossVec& leafValues, TRowItr beginRows, TRowItr endRows) { - for (auto itr = beginRows; itr != endRows; ++itr) { - const TRowRef& row{*itr}; - double prediction{readPrediction(row)}; - double actual{readActual(row, m_DependentVariable)}; - leafValues[root(tree).leafIndex(m_Encoder->encode(row), tree)] - .add(prediction, actual); - } - }, - TArgMinLossVec(tree.size(), m_Loss->minimizer())), - &trainingRowMask); + TArgMinLossVec leafValues( + tree.size(), m_Loss->minimizer(m_Regularization.leafWeightPenaltyMultiplier())); + auto nextPass = [&] { + bool done{true}; + for (const auto& value : leafValues) { + done &= (value.nextPass() == false); + } + return done == false; + }; - auto leafValues = std::move(result.first[0].s_FunctionState); - for (std::size_t i = 1; i < result.first.size(); ++i) { - for (std::size_t j = 0; j < leafValues.size(); ++j) { - leafValues[j].merge(result.first[i].s_FunctionState[j]); + do { + auto result = frame.readRows( + m_NumberThreads, 0, frame.numberRows(), + core::bindRetrievableState( + [&](TArgMinLossVec& leafValues_, TRowItr beginRows, TRowItr endRows) { + for (auto itr = beginRows; itr != endRows; ++itr) { + const TRowRef& row{*itr}; + double prediction{readPrediction(row)}; + double actual{readActual(row, m_DependentVariable)}; + leafValues_[root(tree).leafIndex(m_Encoder->encode(row), tree)] + .add(prediction, actual); + } + }, + std::move(leafValues)), + &trainingRowMask); + + leafValues = std::move(result.first[0].s_FunctionState); + for (std::size_t i = 1; i < result.first.size(); ++i) { + for (std::size_t j = 0; j < leafValues.size(); ++j) { + leafValues[j].merge(result.first[i].s_FunctionState[j]); + } } - } + } while (nextPass()); for (std::size_t i = 0; i < tree.size(); ++i) { tree[i].value(eta * leafValues[i].value()); diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index f45d735cfd..12e77efe10 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include @@ -71,13 +73,14 @@ template void fillDataFrame(std::size_t trainRows, std::size_t testRows, std::size_t cols, + const TBoolVec& categoricalColumns, const TDoubleVecVec& regressors, const TDoubleVec& noise, const F& target, core::CDataFrame& frame) { std::size_t rows{trainRows + testRows}; - frame.categoricalColumns(TBoolVec(cols, false)); + frame.categoricalColumns(categoricalColumns); for (std::size_t i = 0; i < rows; ++i) { frame.writeRow([&](core::CDataFrame::TFloatVecItr column, std::int32_t&) { for (std::size_t j = 0; j < cols - 1; ++j, ++column) { @@ -96,6 +99,18 @@ void fillDataFrame(std::size_t trainRows, }); } +template +void fillDataFrame(std::size_t trainRows, + std::size_t testRows, + std::size_t cols, + const TDoubleVecVec& regressors, + const TDoubleVec& noise, + const F& target, + core::CDataFrame& frame) { + fillDataFrame(trainRows, testRows, cols, TBoolVec(cols, false), regressors, + noise, target, frame); +} + template auto predictAndComputeEvaluationMetrics(const F& generateFunction, test::CRandomNumbers& rng, @@ -809,6 +824,210 @@ void CBoostedTreeTest::testDepthBasedRegularization() { } } +void CBoostedTreeTest::testLogisticMinimizer() { + + // Test that we a good approximation of the minimizing additive weight for + // the cross entropy objective of logistic regression. + + using maths::boosted_tree_detail::CArgMinLogisticImpl; + + test::CRandomNumbers rng; + + TDoubleVec labels; + TDoubleVec weights; + + // All predictions equal + { + CArgMinLogisticImpl argmin{0.0}; + argmin.add(0.0, 0.0); + argmin.add(0.0, 1.0); + argmin.add(0.0, 1.0); + argmin.add(0.0, 0.0); + argmin.nextPass(); + CPPUNIT_ASSERT_EQUAL(0.0, argmin.value()); + } + { + rng.generateUniformSamples(0.0, 1.0, 1000, labels); + for (auto& label : labels) { + label = std::floor(label + 0.3); + } + weights.resize(labels.size(), 0.0); + + CArgMinLogisticImpl argmin{0.0}; + std::size_t numberPasses{0}; + std::size_t counts[2]{0, 0}; + + do { + ++numberPasses; + for (std::size_t i = 0; i < labels.size(); ++i) { + argmin.add(weights[i], labels[i]); + ++counts[static_cast(labels[i])]; + } + } while (argmin.nextPass()); + + double p{static_cast(counts[1]) / 1000.0}; + double expected{std::log(p / (1.0 - p))}; + double actual{argmin.value()}; + + CPPUNIT_ASSERT_EQUAL(std::size_t{1}, numberPasses); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, actual, 0.01 * std::fabs(expected)); + } + + for (auto lambda : {0.0, 10.0}) { + + LOG_DEBUG(<< "lambda = " << lambda); + auto objective = [&](double weight) { + double loss{0.0}; + for (std::size_t i = 0; i < labels.size(); ++i) { + double p{maths::CTools::logisticFunction(weights[i] + weight)}; + loss -= (1.0 - labels[i]) * maths::CTools::fastLog(1.0 - p) + + labels[i] * maths::CTools::fastLog(p); + } + return loss + lambda * maths::CTools::pow2(weight); + }; + + for (std::size_t t = 0; t < 10; ++t) { + + double min{std::numeric_limits::max()}; + double max{-min}; + + rng.generateUniformSamples(0.0, 1.0, 1000, labels); + for (auto& label : labels) { + label = std::floor(label + 0.5); + } + weights.clear(); + for (const auto& label : labels) { + TDoubleVec weight; + rng.generateNormalSamples(label, 2.0, 1, weight); + weights.push_back(weight[0]); + min = std::min(min, weight[0]); + max = std::max(max, weight[0]); + } + + double expected; + double objectiveAtExpected; + std::size_t maxIterations{20}; + maths::CSolvers::minimize(-max, -min, objective(-max), objective(-min), + objective, 1e-3, maxIterations, expected, + objectiveAtExpected); + LOG_DEBUG(<< "expected = " << expected + << " objective at expected = " << objectiveAtExpected); + + CArgMinLogisticImpl argmin{lambda}; + CArgMinLogisticImpl argminPartition[2]{{lambda}, {lambda}}; + auto nextPass = [&] { + bool done{argmin.nextPass() == false}; + done &= (argminPartition[0].nextPass() == false); + done &= (argminPartition[1].nextPass() == false); + return done; + }; + + do { + for (std::size_t i = 0; i < labels.size() / 2; ++i) { + argmin.add(weights[i], labels[i]); + argminPartition[0].add(weights[i], labels[i]); + } + for (std::size_t i = labels.size() / 2; i < labels.size(); ++i) { + argmin.add(weights[i], labels[i]); + argminPartition[1].add(weights[i], labels[i]); + } + argminPartition[0].merge(argminPartition[1]); + argminPartition[1] = argminPartition[0]; + } while (nextPass()); + + double actual{argmin.value()}; + double actualPartition{argminPartition[0].value()}; + LOG_DEBUG(<< "actual = " << actual + << " objective at actual = " << objective(actual)); + + CPPUNIT_ASSERT_EQUAL(actual, actualPartition); + CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, actual, 0.01 * std::fabs(expected)); + CPPUNIT_ASSERT_DOUBLES_EQUAL(objectiveAtExpected, objective(actual), + 1e-5 * objectiveAtExpected); + } + } +} + +void CBoostedTreeTest::testLogisticRegression() { + + // Test we approximately minimise the cross entropy if the category labels + // are generated from log odds which are a linaer function of the regressors. + + test::CRandomNumbers rng; + + std::size_t trainRows{1000}; + std::size_t rows{1200}; + std::size_t cols{4}; + std::size_t capacity{600}; + + TMeanAccumulator meanExcessCrossEntropy; + for (std::size_t test = 0; test < 3; ++test) { + auto probability = [&] { + TDoubleVec weights; + rng.generateUniformSamples(-2.0, 2.0, cols - 1, weights); + TDoubleVec noise; + rng.generateNormalSamples(0.0, 1.0, rows, noise); + return [=](const TRowRef& row) { + double x{0.0}; + for (std::size_t i = 0; i < cols - 1; ++i) { + x += weights[i] * row[i]; + } + return maths::CTools::logisticFunction(x + noise[row.index()]); + }; + }(); + + auto target = [&] { + TDoubleVec uniform01; + rng.generateUniformSamples(0.0, 1.0, rows, uniform01); + return [=](const TRowRef& row) { + return uniform01[row.index()] < probability(row) ? 1.0 : 0.0; + }; + }(); + + TDoubleVecVec x(cols - 1); + for (std::size_t i = 0; i < cols - 1; ++i) { + rng.generateUniformSamples(0.0, 4.0, rows, x[i]); + } + + auto frame = core::makeMainStorageDataFrame(cols, capacity).first; + + fillDataFrame(trainRows, rows - trainRows, cols, {false, false, false, true}, + x, TDoubleVec(rows, 0.0), target, *frame); + + auto regression = maths::CBoostedTreeFactory::constructFromParameters( + 1, std::make_unique()) + .buildFor(*frame, cols - 1); + + regression->train(); + regression->predict(); + + double actualCrossEntropy{0.0}; + double minimumCrossEntropy{0.0}; + frame->readRows(1, [&](TRowItr beginRows, TRowItr endRows) { + for (auto row = beginRows; row != endRows; ++row) { + if (row->index() >= trainRows) { + std::size_t index{ + regression->columnHoldingPrediction(row->numberColumns())}; + actualCrossEntropy -= + probability(*row) * + std::log(maths::CTools::logisticFunction((*row)[index])); + minimumCrossEntropy -= probability(*row) * + std::log(probability(*row)); + } + } + }); + + LOG_DEBUG(<< "actual cross entropy = " << actualCrossEntropy + << " minimum cross entropy = " << minimumCrossEntropy); + CPPUNIT_ASSERT(actualCrossEntropy < 1.35 * minimumCrossEntropy); + meanExcessCrossEntropy.add(actualCrossEntropy / minimumCrossEntropy); + } + + LOG_DEBUG(<< "mean excess cross entropy " + << maths::CBasicStatistics::mean(meanExcessCrossEntropy)); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanExcessCrossEntropy) < 1.25); +} + void CBoostedTreeTest::testEstimateMemoryUsedByTrain() { // Test estimation of the memory used training a model. @@ -1122,6 +1341,10 @@ CppUnit::Test* CBoostedTreeTest::suite() { suiteOfTests->addTest(new CppUnit::TestCaller( "CBoostedTreeTest::testDepthBasedRegularization", &CBoostedTreeTest::testDepthBasedRegularization)); + suiteOfTests->addTest(new CppUnit::TestCaller( + "CBoostedTreeTest::testLogisticMinimizer", &CBoostedTreeTest::testLogisticMinimizer)); + suiteOfTests->addTest(new CppUnit::TestCaller( + "CBoostedTreeTest::testLogisticRegression", &CBoostedTreeTest::testLogisticRegression)); suiteOfTests->addTest(new CppUnit::TestCaller( "CBoostedTreeTest::testEstimateMemoryUsedByTrain", &CBoostedTreeTest::testEstimateMemoryUsedByTrain)); diff --git a/lib/maths/unittest/CBoostedTreeTest.h b/lib/maths/unittest/CBoostedTreeTest.h index 6c098bdc0b..7abe8bfa18 100644 --- a/lib/maths/unittest/CBoostedTreeTest.h +++ b/lib/maths/unittest/CBoostedTreeTest.h @@ -22,6 +22,8 @@ class CBoostedTreeTest : public CppUnit::TestFixture { void testSingleSplit(); void testTranslationInvariance(); void testDepthBasedRegularization(); + void testLogisticMinimizer(); + void testLogisticRegression(); void testEstimateMemoryUsedByTrain(); void testProgressMonitoring(); void testMissingData(); From 801fa7b14b89d01d2341b1aa2ddfcf01592b14dd Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 2 Oct 2019 11:54:14 +0100 Subject: [PATCH 02/20] Typo --- lib/maths/unittest/CBoostedTreeTest.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index 12e77efe10..bb827dca7f 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -951,7 +951,7 @@ void CBoostedTreeTest::testLogisticMinimizer() { void CBoostedTreeTest::testLogisticRegression() { // Test we approximately minimise the cross entropy if the category labels - // are generated from log odds which are a linaer function of the regressors. + // are generated from log odds which are a linear function of the regressors. test::CRandomNumbers rng; @@ -1018,12 +1018,12 @@ void CBoostedTreeTest::testLogisticRegression() { }); LOG_DEBUG(<< "actual cross entropy = " << actualCrossEntropy - << " minimum cross entropy = " << minimumCrossEntropy); + << ", minimum cross entropy = " << minimumCrossEntropy); CPPUNIT_ASSERT(actualCrossEntropy < 1.35 * minimumCrossEntropy); meanExcessCrossEntropy.add(actualCrossEntropy / minimumCrossEntropy); } - LOG_DEBUG(<< "mean excess cross entropy " + LOG_DEBUG(<< "mean excess cross entropy = " << maths::CBasicStatistics::mean(meanExcessCrossEntropy)); CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanExcessCrossEntropy) < 1.25); } From b7decfdd7fc60c8226bfa5d6a8696275db7b934c Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 2 Oct 2019 12:12:41 +0100 Subject: [PATCH 03/20] Better comments --- include/maths/CBoostedTree.h | 13 ++++++------- lib/maths/unittest/CBoostedTreeTest.cc | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/include/maths/CBoostedTree.h b/include/maths/CBoostedTree.h index c16af92003..76a367c1af 100644 --- a/include/maths/CBoostedTree.h +++ b/include/maths/CBoostedTree.h @@ -46,7 +46,8 @@ class MATHS_EXPORT CArgMinLossImpl { double m_Lambda; }; -//! \brief Finds the value to add to a set of predictions which minimises the MSE. +//! \brief Finds the value to add to a set of predictions which minimises the +//! regularized MSE w.r.t. the actual values. class MATHS_EXPORT CArgMinMseImpl final : public CArgMinLossImpl { public: CArgMinMseImpl(double lambda); @@ -63,8 +64,8 @@ class MATHS_EXPORT CArgMinMseImpl final : public CArgMinLossImpl { TMeanAccumulator m_MeanError; }; -//! \brief Finds the value to add to the argument of the logistic function which -//! minimises the cross entropy loss. +//! \brief Finds the value to add to a set of predicted log-odds which minimises +//! regularised the cross entropy loss w.r.t. the actual categories. class MATHS_EXPORT CArgMinLogisticImpl final : public CArgMinLossImpl { public: CArgMinLogisticImpl(double lambda); @@ -182,12 +183,10 @@ class MATHS_EXPORT CMse final : public CLoss { static const std::string NAME; }; -//! \brief Implements loss for logistic regression for binary classification. +//! \brief Implements loss for binomial logistic regression. //! //! DESCRIPTION:\n -//! This targets the cross entropy loss using the logistic function of the sum of -//! the of the tree predictions to estimate the probability of one of the classes -//! for a binary classification task +//! This targets the cross entropy loss using the tree to predict class log-odds: //! 
 //!   \f$\displaystyle l_i(p) = -(1 - a_i) \log(1 - S(p)) - a_i \log(S(p))\f$
 //!
diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index bb827dca7f..596c2fb073 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -826,8 +826,8 @@ void CBoostedTreeTest::testDepthBasedRegularization() { void CBoostedTreeTest::testLogisticMinimizer() { - // Test that we a good approximation of the minimizing additive weight for - // the cross entropy objective of logistic regression. + // Test that we a good approximation of the additive term for the log-odds + // which minimises the cross entropy objective. using maths::boosted_tree_detail::CArgMinLogisticImpl; @@ -836,7 +836,7 @@ void CBoostedTreeTest::testLogisticMinimizer() { TDoubleVec labels; TDoubleVec weights; - // All predictions equal + // All predictions equal and zero. { CArgMinLogisticImpl argmin{0.0}; argmin.add(0.0, 0.0); @@ -846,6 +846,7 @@ void CBoostedTreeTest::testLogisticMinimizer() { argmin.nextPass(); CPPUNIT_ASSERT_EQUAL(0.0, argmin.value()); } + // All predictions are equal. { rng.generateUniformSamples(0.0, 1.0, 1000, labels); for (auto& label : labels) { @@ -876,6 +877,8 @@ void CBoostedTreeTest::testLogisticMinimizer() { for (auto lambda : {0.0, 10.0}) { LOG_DEBUG(<< "lambda = " << lambda); + + // The true objective. auto objective = [&](double weight) { double loss{0.0}; for (std::size_t i = 0; i < labels.size(); ++i) { @@ -940,6 +943,8 @@ void CBoostedTreeTest::testLogisticMinimizer() { LOG_DEBUG(<< "actual = " << actual << " objective at actual = " << objective(actual)); + // We should be within 1% for the value and 0.001% for the objective + // at the value. CPPUNIT_ASSERT_EQUAL(actual, actualPartition); CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, actual, 0.01 * std::fabs(expected)); CPPUNIT_ASSERT_DOUBLES_EQUAL(objectiveAtExpected, objective(actual), @@ -1016,15 +1021,18 @@ void CBoostedTreeTest::testLogisticRegression() { } } }); - LOG_DEBUG(<< "actual cross entropy = " << actualCrossEntropy << ", minimum cross entropy = " << minimumCrossEntropy); + + // We should be with 35% of the minimum possible cross entropy. CPPUNIT_ASSERT(actualCrossEntropy < 1.35 * minimumCrossEntropy); meanExcessCrossEntropy.add(actualCrossEntropy / minimumCrossEntropy); } LOG_DEBUG(<< "mean excess cross entropy = " << maths::CBasicStatistics::mean(meanExcessCrossEntropy)); + + // We should be within 25% of the minimum possible cross entropy on average. CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanExcessCrossEntropy) < 1.25); } From 885da7d4066ede18e7a5fb3e826be30c75196c79 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 2 Oct 2019 13:13:02 +0100 Subject: [PATCH 04/20] Update tests --- lib/maths/CBoostedTreeFactory.cc | 7 +++++-- lib/maths/unittest/CBoostedTreeTest.cc | 17 +++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/maths/CBoostedTreeFactory.cc b/lib/maths/CBoostedTreeFactory.cc index b302660cbb..55639ad458 100644 --- a/lib/maths/CBoostedTreeFactory.cc +++ b/lib/maths/CBoostedTreeFactory.cc @@ -80,7 +80,8 @@ CBoostedTreeFactory::buildFor(core::CDataFrame& frame, std::size_t dependentVari } } - auto treeImpl = std::make_unique(m_NumberThreads, m_Loss->clone()); + auto treeImpl = std::make_unique( + m_NumberThreads, m_Loss != nullptr ? m_Loss->clone() : nullptr); std::swap(m_TreeImpl, treeImpl); return TBoostedTreeUPtr{new CBoostedTree{frame, m_RecordProgress, m_RecordMemoryUsage, m_RecordTrainingState, std::move(treeImpl)}}; @@ -616,7 +617,9 @@ CBoostedTreeFactory CBoostedTreeFactory::constructFromString(std::istream& jsonS } CBoostedTreeFactory::CBoostedTreeFactory(bool restored, std::size_t numberThreads, TLossFunctionUPtr loss) - : m_Restored{restored}, m_NumberThreads{numberThreads}, m_Loss{loss->clone()}, + : m_Restored{restored}, m_NumberThreads{numberThreads}, m_Loss{loss != nullptr + ? loss->clone() + : nullptr}, m_TreeImpl{std::make_unique(numberThreads, std::move(loss))}, m_LogDepthPenaltyMultiplierSearchInterval{0.0}, m_LogTreeSizePenaltyMultiplierSearchInterval{0.0}, m_LogLeafWeightPenaltyMultiplierSearchInterval{0.0} { diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index 596c2fb073..2f968dda1c 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -227,12 +227,13 @@ void CBoostedTreeTest::testPiecewiseConstant() { // Unbiased... CPPUNIT_ASSERT_DOUBLES_EQUAL( 0.0, modelBias[i][0], - 7.0 * std::sqrt(noiseVariance / static_cast(trainRows))); + 4.0 * std::sqrt(noiseVariance / static_cast(trainRows))); // Good R^2... - CPPUNIT_ASSERT(modelRSquared[i][0] > 0.95); + CPPUNIT_ASSERT(modelRSquared[i][0] > 0.96); meanModelRSquared.add(modelRSquared[i][0]); } + LOG_DEBUG(<< "mean R^2 = " << maths::CBasicStatistics::mean(meanModelRSquared)); CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanModelRSquared) > 0.97); } @@ -282,14 +283,14 @@ void CBoostedTreeTest::testLinear() { // Unbiased... CPPUNIT_ASSERT_DOUBLES_EQUAL( 0.0, modelBias[i][0], - 5.0 * std::sqrt(noiseVariance / static_cast(trainRows))); + 4.0 * std::sqrt(noiseVariance / static_cast(trainRows))); // Good R^2... CPPUNIT_ASSERT(modelRSquared[i][0] > 0.97); meanModelRSquared.add(modelRSquared[i][0]); } LOG_DEBUG(<< "mean R^2 = " << maths::CBasicStatistics::mean(meanModelRSquared)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanModelRSquared) > 0.97); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanModelRSquared) > 0.98); } void CBoostedTreeTest::testNonLinear() { @@ -349,14 +350,14 @@ void CBoostedTreeTest::testNonLinear() { // Unbiased... CPPUNIT_ASSERT_DOUBLES_EQUAL( 0.0, modelBias[i][0], - 8.0 * std::sqrt(noiseVariance / static_cast(trainRows))); + 4.0 * std::sqrt(noiseVariance / static_cast(trainRows))); // Good R^2... - CPPUNIT_ASSERT(modelRSquared[i][0] > 0.95); + CPPUNIT_ASSERT(modelRSquared[i][0] > 0.97); meanModelRSquared.add(modelRSquared[i][0]); } LOG_DEBUG(<< "mean R^2 = " << maths::CBasicStatistics::mean(meanModelRSquared)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanModelRSquared) > 0.96); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanModelRSquared) > 0.97); } void CBoostedTreeTest::testThreading() { @@ -922,7 +923,7 @@ void CBoostedTreeTest::testLogisticMinimizer() { bool done{argmin.nextPass() == false}; done &= (argminPartition[0].nextPass() == false); done &= (argminPartition[1].nextPass() == false); - return done; + return done == false; }; do { From 55dc6af9e52521e2aa26d4de9a53e3f3b573a436 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 2 Oct 2019 14:11:17 +0100 Subject: [PATCH 05/20] Docs --- docs/CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index c2fedcf6ba..8e87bba36b 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -43,6 +43,7 @@ boosted tree training. Hard depth based regularization is often the strategy of choice to prevent over fitting for XGBoost. By smoothing we can make better tradeoffs. Also, the parameters of the penalty function are mode suited to optimising with our Bayesian optimisation based hyperparameter search. (See {ml-pull}698[#698].) +* Binomial logistic regression targeting cross entropy. (See {ml-pull}713[#713].) == {es} version 7.4.1 From 72010b428b120ecf255e18cfc899a3dfbb5e2cc1 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 2 Oct 2019 14:19:27 +0100 Subject: [PATCH 06/20] Relax test thresholds for other platforms --- lib/maths/unittest/CBoostedTreeTest.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index 2f968dda1c..89890e0883 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -352,7 +352,7 @@ void CBoostedTreeTest::testNonLinear() { 0.0, modelBias[i][0], 4.0 * std::sqrt(noiseVariance / static_cast(trainRows))); // Good R^2... - CPPUNIT_ASSERT(modelRSquared[i][0] > 0.97); + CPPUNIT_ASSERT(modelRSquared[i][0] > 0.96); meanModelRSquared.add(modelRSquared[i][0]); } @@ -818,7 +818,7 @@ void CBoostedTreeTest::testDepthBasedRegularization() { TMeanAccumulator meanDepth; for (const auto& tree : regression->trainedModel()) { CPPUNIT_ASSERT(maxDepth(tree, tree[0], 0) <= static_cast(targetDepth)); - meanDepth.add(maxDepth(tree, tree[0], 0)); + meanDepth.add(static_cast(maxDepth(tree, tree[0], 0))); } LOG_DEBUG(<< "mean depth = " << maths::CBasicStatistics::mean(meanDepth)); CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanDepth) > targetDepth - 1.0); @@ -1025,8 +1025,8 @@ void CBoostedTreeTest::testLogisticRegression() { LOG_DEBUG(<< "actual cross entropy = " << actualCrossEntropy << ", minimum cross entropy = " << minimumCrossEntropy); - // We should be with 35% of the minimum possible cross entropy. - CPPUNIT_ASSERT(actualCrossEntropy < 1.35 * minimumCrossEntropy); + // We should be with 40% of the minimum possible cross entropy. + CPPUNIT_ASSERT(actualCrossEntropy < 1.4 * minimumCrossEntropy); meanExcessCrossEntropy.add(actualCrossEntropy / minimumCrossEntropy); } From d0f6246e695b2c0950f1ece0b87567648ed45285 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 2 Oct 2019 14:22:32 +0100 Subject: [PATCH 07/20] Another one --- lib/maths/unittest/CBoostedTreeTest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index 89890e0883..ecb273feb5 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -290,7 +290,7 @@ void CBoostedTreeTest::testLinear() { meanModelRSquared.add(modelRSquared[i][0]); } LOG_DEBUG(<< "mean R^2 = " << maths::CBasicStatistics::mean(meanModelRSquared)); - CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanModelRSquared) > 0.98); + CPPUNIT_ASSERT(maths::CBasicStatistics::mean(meanModelRSquared) > 0.97); } void CBoostedTreeTest::testNonLinear() { From 28624bba5f946166e3d27836643998c67275bbf6 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Thu, 3 Oct 2019 14:28:11 +0100 Subject: [PATCH 08/20] Fix and overhaul test. Break hidden dependency on state names. Override correct parameters. --- include/maths/CBoostedTree.h | 13 + include/maths/CBoostedTreeImpl.h | 11 + lib/api/unittest/CDataFrameAnalyzerTest.cc | 370 +++++++++++---------- lib/api/unittest/CDataFrameAnalyzerTest.h | 34 -- lib/maths/CBoostedTree.cc | 16 + lib/maths/CBoostedTreeImpl.cc | 19 ++ 6 files changed, 250 insertions(+), 213 deletions(-) diff --git a/include/maths/CBoostedTree.h b/include/maths/CBoostedTree.h index 76a367c1af..3d1374fc41 100644 --- a/include/maths/CBoostedTree.h +++ b/include/maths/CBoostedTree.h @@ -18,6 +18,8 @@ #include #include +#include +#include namespace ml { namespace core { @@ -328,6 +330,7 @@ class MATHS_EXPORT CBoostedTreeNode final { //! proposed by Reshef for this purpose. See CDataFrameCategoryEncoder for more details. class MATHS_EXPORT CBoostedTree final : public CDataFrameRegressionModel { public: + using TStrVec = std::vector; using TRowRef = core::CDataFrame::TRowRef; using TLossFunctionUPtr = std::unique_ptr; using TDataFramePtr = core::CDataFrame*; @@ -365,6 +368,16 @@ class MATHS_EXPORT CBoostedTree final : public CDataFrameRegressionModel { //! Get the model produced by training if it has been run. const TNodeVecVec& trainedModel() const; + //! The name of the object holding the best hyperaparameters in the state document. + static const std::string& bestHyperparametersName(); + + //! The name of the object holding the best regularisation hyperparameters in the + //! state document. + static const std::string& bestRegularizationHyperparametersName(); + + //! A list of the names of the best individual hyperparameters in the state document. + static TStrVec bestHyperparameterNames(); + //! Persist by passing information to \p inserter. void acceptPersistInserter(core::CStatePersistInserter& inserter) const; diff --git a/include/maths/CBoostedTreeImpl.h b/include/maths/CBoostedTreeImpl.h index 3b5bcad000..7e25af7293 100644 --- a/include/maths/CBoostedTreeImpl.h +++ b/include/maths/CBoostedTreeImpl.h @@ -48,6 +48,7 @@ inline std::size_t predictionColumn(std::size_t numberColumns) { class MATHS_EXPORT CBoostedTreeImpl final { public: using TDoubleVec = std::vector; + using TStrVec = std::vector; using TMeanAccumulator = CBasicStatistics::SSampleMean::TAccumulator; using TMeanVarAccumulator = CBasicStatistics::SSampleMeanVar::TAccumulator; using TBayesinOptimizationUPtr = std::unique_ptr; @@ -101,6 +102,16 @@ class MATHS_EXPORT CBoostedTreeImpl final { //! frame with \p numberRows row and \p numberColumns columns will use. std::size_t estimateMemoryUsage(std::size_t numberRows, std::size_t numberColumns) const; + //! The name of the object holding the best hyperaparameters in the state document. + static const std::string& bestHyperparametersName(); + + //! The name of the object holding the best regularisation hyperparameters in the + //! state document. + static const std::string& bestRegularizationHyperparametersName(); + + //! A list of the names of the best individual hyperparameters in the state document. + static TStrVec bestHyperparameterNames(); + //! Persist by passing information to \p inserter. void acceptPersistInserter(core::CStatePersistInserter& inserter) const; diff --git a/lib/api/unittest/CDataFrameAnalyzerTest.cc b/lib/api/unittest/CDataFrameAnalyzerTest.cc index 8b11db920f..b653e3bdf9 100644 --- a/lib/api/unittest/CDataFrameAnalyzerTest.cc +++ b/lib/api/unittest/CDataFrameAnalyzerTest.cc @@ -47,6 +47,10 @@ using TRowItr = core::CDataFrame::TRowItr; using TPoint = maths::CDenseVector; using TPointVec = std::vector; using TDataFrameUPtr = std::unique_ptr; +using TDataAdderUPtr = std::unique_ptr; +using TPersisterSupplier = std::function; +using TDataSearcherUPtr = std::unique_ptr; +using TRestoreSearcherSupplier = std::function; class CTestDataSearcher : public core::CDataSearcher { public: @@ -56,6 +60,7 @@ class CTestDataSearcher : public core::CDataSearcher { virtual TIStreamP search(size_t /*currentDocNum*/, size_t /*limit*/) { std::istringstream* intermediateStateStream{ static_cast(m_Stream.get())}; + // Discard first line, which contains the state id. intermediateStateStream->ignore(256, '\n'); std::string intermediateState; std::getline(*intermediateStateStream, intermediateState); @@ -66,26 +71,8 @@ class CTestDataSearcher : public core::CDataSearcher { TIStreamP m_Stream; }; -class CTestDataAdder : public core::CDataAdder { -public: - CTestDataAdder() : m_Stream(new std::ostringstream) {} - - virtual TOStreamP addStreamed(const std::string& /*index*/, const std::string& /*id*/) { - return m_Stream; - } - - virtual bool streamComplete(TOStreamP& /*strm*/, bool /*force*/) { - return true; - } - - TOStreamP getStream() { return m_Stream; } - -private: - TOStreamP m_Stream; -}; - -std::vector streamToStringVector(std::stringstream&& tokenStream) { - std::vector results; +TStrVec splitOnNull(std::stringstream&& tokenStream) { + TStrVec results; std::string token; while (std::getline(tokenStream, token, '\0')) { results.push_back(token); @@ -106,6 +93,16 @@ rapidjson::Document treeToJsonDocument(const maths::CBoostedTree& tree) { return results; } +auto restoreTree(std::string persistedState, TDataFrameUPtr& frame, std::size_t dependentVariable) { + CTestDataSearcher dataSearcher(persistedState); + auto decompressor = std::make_unique(dataSearcher); + decompressor->setStateRestoreSearch(api::ML_STATE_INDEX, + api::getRegressionStateId("testJob")); + auto stream = decompressor->search(1, 1); + return maths::CBoostedTreeFactory::constructFromString(*stream).buildFor( + *frame, dependentVariable); +} + auto outlierSpec(std::size_t rows = 110, std::size_t memoryLimit = 100000, std::string method = "", @@ -151,21 +148,35 @@ auto regressionSpec(std::string dependentVariable, std::size_t numberRoundsPerHyperparameter = 0, std::size_t bayesianOptimisationRestarts = 0, const TStrVec& categoricalFieldNames = TStrVec{}, + double alpha = -1.0, double lambda = -1.0, double gamma = -1.0, + double softTreeDepthLimit = -1.0, + double softTreeDepthTolerance = -1.0, double eta = -1.0, std::size_t maximumNumberTrees = 0, double featureBagFraction = -1.0, - CDataFrameAnalyzerTest::TPersisterSupplier* persisterSupplier = nullptr, - CDataFrameAnalyzerTest::TRestoreSearcherSupplier* restoreSearcherSupplier = nullptr) { + TPersisterSupplier* persisterSupplier = nullptr, + TRestoreSearcherSupplier* restoreSearcherSupplier = nullptr) { std::string parameters = "{\n\"dependent_variable\": \"" + dependentVariable + "\""; + if (alpha >= 0.0) { + parameters += ",\n\"alpha\": " + core::CStringUtils::typeToString(alpha); + } if (lambda >= 0.0) { parameters += ",\n\"lambda\": " + core::CStringUtils::typeToString(lambda); } if (gamma >= 0.0) { parameters += ",\n\"gamma\": " + core::CStringUtils::typeToString(gamma); } + if (softTreeDepthLimit >= 0.0) { + parameters += ",\n\"soft_tree_depth_limit\": " + + core::CStringUtils::typeToString(softTreeDepthLimit); + } + if (softTreeDepthTolerance >= 0.0) { + parameters += ",\n\"soft_tree_depth_tolerance\": " + + core::CStringUtils::typeToString(softTreeDepthTolerance); + } if (eta > 0.0) { parameters += ",\n\"eta\": " + core::CStringUtils::typeToString(eta); } @@ -317,8 +328,11 @@ void addRegressionTestData(const TStrVec& fieldNames, api::CDataFrameAnalyzer& analyzer, TDoubleVec& expectedPredictions, std::size_t numberExamples = 100, + double alpha = -1.0, double lambda = -1.0, double gamma = -1.0, + double softTreeDepthLimit = -1.0, + double softTreeDepthTolerance = -1.0, double eta = 0.0, std::size_t maximumNumberTrees = 0, double featureBagFraction = 0.0) { @@ -333,12 +347,21 @@ void addRegressionTestData(const TStrVec& fieldNames, maths::CBoostedTreeFactory treeFactory{maths::CBoostedTreeFactory::constructFromParameters( 1, std::make_unique())}; + if (alpha >= 0.0) { + treeFactory.depthPenaltyMultiplier(alpha); + } if (lambda >= 0.0) { treeFactory.leafWeightPenaltyMultiplier(lambda); } if (gamma >= 0.0) { treeFactory.treeSizePenaltyMultiplier(gamma); } + if (softTreeDepthLimit >= 0.0) { + treeFactory.softTreeDepthLimit(softTreeDepthLimit); + } + if (softTreeDepthTolerance >= 0.0) { + treeFactory.softTreeDepthTolerance(softTreeDepthTolerance); + } if (eta > 0.0) { treeFactory.eta(eta); } @@ -361,6 +384,88 @@ void addRegressionTestData(const TStrVec& fieldNames, } }); } + +template +void testOneRunOfBoostedTreeTrainingWithStateRecovery(F makeSpec, std::size_t iterationToRestartFrom) { + + std::stringstream outputStream; + auto outputWriterFactory = [&outputStream]() { + return std::make_unique(outputStream); + }; + + std::size_t numberExamples{200}; + TStrVec fieldNames{"c1", "c2", "c3", "c4", "c5", ".", "."}; + TStrVec fieldValues{"", "", "", "", "", "0", ""}; + TDoubleVec weights{0.1, 2.0, 0.4, -0.5}; + TDoubleVec values; + test::CRandomNumbers rng; + rng.generateUniformSamples(-10.0, 10.0, weights.size() * numberExamples, values); + + auto persistenceStream = std::make_shared(); + TPersisterSupplier persisterSupplier = [&persistenceStream]() -> TDataAdderUPtr { + return std::make_unique(persistenceStream); + }; + + // Compute expected tree. + + api::CDataFrameAnalyzer analyzer{ + makeSpec("c5", numberExamples, persisterSupplier), outputWriterFactory}; + std::size_t dependentVariable( + std::find(fieldNames.begin(), fieldNames.end(), "c5") - fieldNames.begin()); + + auto frame = passDataToAnalyzer(fieldNames, fieldValues, analyzer, weights, values); + analyzer.handleRecord(fieldNames, {"", "", "", "", "", "", "$"}); + + TStrVec persistedStates{ + splitOnNull(std::stringstream{std::move(persistenceStream->str())})}; + auto expectedTree = restoreTree(std::move(persistedStates.back()), frame, dependentVariable); + + // Compute actual tree. + + persistenceStream->str(""); + + std::istringstream intermediateStateStream{persistedStates[iterationToRestartFrom]}; + TRestoreSearcherSupplier restoreSearcherSupplier = [&intermediateStateStream]() -> TDataSearcherUPtr { + return std::make_unique(intermediateStateStream.str()); + }; + + api::CDataFrameAnalyzer restoredAnalyzer{ + makeSpec("c5", numberExamples, persisterSupplier), outputWriterFactory}; + + passDataToAnalyzer(fieldNames, fieldValues, restoredAnalyzer, weights, values); + restoredAnalyzer.handleRecord(fieldNames, {"", "", "", "", "", "", "$"}); + + persistedStates = splitOnNull(std::stringstream{std::move(persistenceStream->str())}); + auto actualTree = restoreTree(std::move(persistedStates.back()), frame, dependentVariable); + + // Compare hyperparameters. + + rapidjson::Document expectedResults{treeToJsonDocument(*expectedTree)}; + const auto& expectedHyperparameters = + expectedResults[maths::CBoostedTree::bestHyperparametersName()]; + const auto& expectedRegularizationHyperparameters = + expectedHyperparameters[maths::CBoostedTree::bestRegularizationHyperparametersName()]; + + rapidjson::Document actualResults{treeToJsonDocument(*actualTree)}; + const auto& actualHyperparameters = + actualResults[maths::CBoostedTree::bestHyperparametersName()]; + const auto& actualRegularizationHyperparameters = + actualHyperparameters[maths::CBoostedTree::bestRegularizationHyperparametersName()]; + + for (const auto& key : maths::CBoostedTree::bestHyperparameterNames()) { + if (expectedHyperparameters.HasMember(key)) { + double expected{std::stod(expectedHyperparameters[key].GetString())}; + double actual{std::stod(actualHyperparameters[key].GetString())}; + CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, actual, 1e-4 * expected); + } else if (expectedRegularizationHyperparameters.HasMember(key)) { + double expected{std::stod(expectedRegularizationHyperparameters[key].GetString())}; + double actual{std::stod(actualRegularizationHyperparameters[key].GetString())}; + CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, actual, 1e-4 * expected); + } else { + CPPUNIT_FAIL("Missing " + key); + } + } +} } void CDataFrameAnalyzerTest::testWithoutControlMessages() { @@ -664,8 +769,11 @@ void CDataFrameAnalyzerTest::testRunBoostedTreeTrainingWithParams() { // Test the regression hyperparameter settings are correctly propagated to the // analysis runner. + double alpha{2.0}; double lambda{1.0}; double gamma{10.0}; + double softTreeDepthLimit{3.0}; + double softTreeDepthTolerance{0.1}; double eta{0.9}; std::size_t maximumNumberTrees{1}; double featureBagFraction{0.3}; @@ -676,7 +784,8 @@ void CDataFrameAnalyzerTest::testRunBoostedTreeTrainingWithParams() { }; api::CDataFrameAnalyzer analyzer{ - regressionSpec("c5", 100, 5, 3000000, 0, 0, {}, lambda, gamma, eta, + regressionSpec("c5", 100, 5, 3000000, 0, 0, {}, alpha, lambda, gamma, + softTreeDepthLimit, softTreeDepthTolerance, eta, maximumNumberTrees, featureBagFraction), outputWriterFactory}; @@ -684,8 +793,9 @@ void CDataFrameAnalyzerTest::testRunBoostedTreeTrainingWithParams() { TStrVec fieldNames{"c1", "c2", "c3", "c4", "c5", ".", "."}; TStrVec fieldValues{"", "", "", "", "", "0", ""}; - addRegressionTestData(fieldNames, fieldValues, analyzer, expectedPredictions, 100, - lambda, gamma, eta, maximumNumberTrees, featureBagFraction); + addRegressionTestData(fieldNames, fieldValues, analyzer, expectedPredictions, + 100, alpha, lambda, gamma, softTreeDepthLimit, softTreeDepthTolerance, + eta, maximumNumberTrees, featureBagFraction); analyzer.handleRecord(fieldNames, {"", "", "", "", "", "", "$"}); rapidjson::Document results; @@ -773,6 +883,61 @@ void CDataFrameAnalyzerTest::testRunBoostedTreeTrainingWithRowsMissingTargetValu CPPUNIT_ASSERT_EQUAL(std::size_t{50}, numberResults); } +void CDataFrameAnalyzerTest::testRunBoostedTreeTrainingWithStateRecovery() { + + struct SHyperparameters { + SHyperparameters(double alpha = 2.0, double lambda = 1.0, double gamma = 10.0) + : s_Alpha{alpha}, s_Lambda{lambda}, s_Gamma{gamma} {} + + std::size_t numberUnset() const { + return (s_Alpha < 0.0 ? 1 : 0) + (s_Lambda < 0.0 ? 1 : 0) + + (s_Gamma < 0.0 ? 1 : 0); + } + + double s_Alpha; + double s_Lambda; + double s_Gamma; + double s_SoftTreeDepthLimit = 3.0; + double s_SoftTreeDepthTolerance = 0.15; + double s_Eta = 0.9; + std::size_t s_MaximumNumberTrees = 2; + double s_FeatureBagFraction = 0.3; + }; + + std::size_t numberRoundsPerHyperparameter{3}; + + TSizeVec intermediateIterations; + std::size_t finalIteration{0}; + + test::CRandomNumbers rng; + + // TODO re-enable case that all parameters are set. + for (const auto& params : + {/*SHyperparameters{},*/ SHyperparameters{-1.0}, + SHyperparameters{-1.0, -1.0}, SHyperparameters{-1.0, -1.0, -1.0}}) { + + LOG_DEBUG(<< "Number parameters to search = " << params.numberUnset()); + + auto makeSpec = [&](const std::string& dependentVariable, std::size_t numberExamples, + TPersisterSupplier persisterSupplier) { + return regressionSpec(dependentVariable, numberExamples, 5, 15000000, + numberRoundsPerHyperparameter, 12, {}, + params.s_Alpha, params.s_Lambda, params.s_Gamma, + params.s_SoftTreeDepthLimit, params.s_SoftTreeDepthTolerance, + params.s_Eta, params.s_MaximumNumberTrees, + params.s_FeatureBagFraction, &persisterSupplier); + }; + + finalIteration = params.numberUnset() * numberRoundsPerHyperparameter - 1; + rng.generateUniformSamples(0, finalIteration - 1, 3, intermediateIterations); + + for (auto intermediateIteration : intermediateIterations) { + LOG_DEBUG(<< "restart from " << intermediateIteration); + testOneRunOfBoostedTreeTrainingWithStateRecovery(makeSpec, intermediateIteration); + } + } +} + void CDataFrameAnalyzerTest::testFlushMessage() { // Test that white space is just ignored. @@ -985,8 +1150,7 @@ void CDataFrameAnalyzerTest::testCategoricalFields() { std::size_t rows{api::CDataFrameAnalyzer::MAX_CATEGORICAL_CARDINALITY + 3}; api::CDataFrameAnalyzer analyzer{ - regressionSpec("x5", rows, 5, 8000000000, 0, 0, {"x1"}, 0, 0, 0, 0, 0), - outputWriterFactory}; + regressionSpec("x5", rows, 5, 8000000000, 0, 0, {"x1"}), outputWriterFactory}; TStrVec fieldNames{"x1", "x2", "x3", "x4", "x5", ".", "."}; TStrVec fieldValues{"", "", "", "", "", "", ""}; @@ -1061,155 +1225,3 @@ CppUnit::Test* CDataFrameAnalyzerTest::suite() { return suiteOfTests; } - -void CDataFrameAnalyzerTest::testRunBoostedTreeTrainingWithStateRecovery() { - - // no hyperparameter search - double lambda{1.0}; - double gamma{10.0}; - double eta{0.9}; - std::size_t maximumNumberTrees{2}; - double featureBagFraction{0.3}; - std::size_t numberRoundsPerHyperparameter{5}; - - TSizeVec intermediateIterations; - std::size_t finalIteration{0}; - - test::CRandomNumbers rng; - - // TODO reactivate this test case - // LOG_DEBUG(<< "No hyperparameters to search") - // testRunBoostedTreeTrainingWithStateRecoverySubroutine( - // lambda, gamma, eta, maximumNumberTrees, featureBagFraction, - // numberRoundsPerHyperparameter, 0, finalIteration); - - LOG_DEBUG(<< "One hyperparameter to search"); - lambda = -1.0; - gamma = 10.0; - finalIteration = 1 * numberRoundsPerHyperparameter - 1; - rng.generateUniformSamples(1, finalIteration - 1, 3, intermediateIterations); - for (auto intermediateIteration : intermediateIterations) { - LOG_DEBUG(<< "restart from " << intermediateIteration); - testRunBoostedTreeTrainingWithStateRecoverySubroutine( - lambda, gamma, eta, maximumNumberTrees, featureBagFraction, - numberRoundsPerHyperparameter, intermediateIteration); - } - - LOG_DEBUG(<< "Two hyperparameters to search"); - lambda = -1.0; - gamma = -1.0; - finalIteration = 2 * numberRoundsPerHyperparameter - 1; - rng.generateUniformSamples(finalIteration / 2, finalIteration - 1, 3, intermediateIterations); - for (auto intermediateIteration : intermediateIterations) { - LOG_DEBUG(<< "restart from " << intermediateIteration); - testRunBoostedTreeTrainingWithStateRecoverySubroutine( - lambda, gamma, eta, maximumNumberTrees, featureBagFraction, - numberRoundsPerHyperparameter, intermediateIteration); - } -} - -void CDataFrameAnalyzerTest::testRunBoostedTreeTrainingWithStateRecoverySubroutine( - double lambda, - double gamma, - double eta, - std::size_t maximumNumberTrees, - double featureBagFraction, - std::size_t numberRoundsPerHyperparameter, - std::size_t iterationToRestartFrom) const { - std::stringstream outputStream; - auto outputWriterFactory = [&outputStream]() { - return std::make_unique(outputStream); - }; - - std::size_t numberExamples{200}; - TStrVec fieldNames{"c1", "c2", "c3", "c4", "c5", ".", "."}; - TStrVec fieldValues{"", "", "", "", "", "0", ""}; - TDoubleVec weights{0.1, 2.0, 0.4, -0.5}; - TDoubleVec values; - test::CRandomNumbers rng; - rng.generateUniformSamples(-10.0, 10.0, weights.size() * numberExamples, values); - - auto persistenceStream{std::make_shared()}; - TPersisterSupplier persisterSupplier = [&persistenceStream]() -> TDataAdderUPtr { - return std::make_unique(persistenceStream); - }; - - // compute expected tree - - api::CDataFrameAnalyzer analyzer{ - regressionSpec("c5", numberExamples, 5, 15000000, - numberRoundsPerHyperparameter, 12, {}, lambda, gamma, eta, - maximumNumberTrees, featureBagFraction, &persisterSupplier), - outputWriterFactory}; - std::size_t dependentVariable( - std::find(fieldNames.begin(), fieldNames.end(), "c5") - fieldNames.begin()); - - auto frame{passDataToAnalyzer(fieldNames, fieldValues, analyzer, weights, values)}; - analyzer.handleRecord(fieldNames, {"", "", "", "", "", "", "$"}); - - TStrVec persistedStatesString{ - streamToStringVector(std::stringstream(persistenceStream->str()))}; - - auto expectedTree{this->getFinalTree(persistedStatesString, frame, dependentVariable)}; - - // Compute actual tree - persistenceStream->str(""); - - std::istringstream intermediateStateStream{persistedStatesString[iterationToRestartFrom]}; - TRestoreSearcherSupplier restoreSearcherSupplier = [&intermediateStateStream]() -> TDataSearcherUPtr { - return std::make_unique(intermediateStateStream.str()); - }; - - api::CDataFrameAnalyzer analyzerToRestore{ - regressionSpec("c5", numberExamples, 5, 15000000, numberRoundsPerHyperparameter, - 12, {}, lambda, gamma, eta, maximumNumberTrees, featureBagFraction, - &persisterSupplier, &restoreSearcherSupplier), - outputWriterFactory}; - - passDataToAnalyzer(fieldNames, fieldValues, analyzerToRestore, weights, values); - analyzerToRestore.handleRecord(fieldNames, {"", "", "", "", "", "", "$"}); - - persistedStatesString = - streamToStringVector(std::stringstream(persistenceStream->str())); - auto actualTree{this->getFinalTree(persistedStatesString, frame, dependentVariable)}; - - // compare hyperparameter - - // TODO avoid implicit dependency on state names - - rapidjson::Document expectedResults{treeToJsonDocument(*expectedTree)}; - const auto& expectedHyperparameters = expectedResults["best_hyperparameters"]; - const auto& expectedRegularizationHyperparameters = - expectedHyperparameters["hyperparam_regularization"]; - - rapidjson::Document actualResults{treeToJsonDocument(*actualTree)}; - const auto& actualHyperparameters = actualResults["best_hyperparameters"]; - const auto& actualRegularizationHyperparameters = - actualHyperparameters["hyperparam_regularization"]; - - for (const auto& key : {"hyperparam_eta", "hyperparam_eta_growth_rate_per_tree", - "hyperparam_feature_bag_fraction"}) { - double expected{std::stod(expectedHyperparameters[key].GetString())}; - double actual{std::stod(actualHyperparameters[key].GetString())}; - CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, actual, 1e-4 * expected); - } - for (const auto& key : {"regularization_tree_size_penalty_multiplier", - "regularization_leaf_weight_penalty_multiplier"}) { - double expected{std::stod(expectedRegularizationHyperparameters[key].GetString())}; - double actual{std::stod(actualRegularizationHyperparameters[key].GetString())}; - CPPUNIT_ASSERT_DOUBLES_EQUAL(expected, actual, 1e-4 * expected); - } -} - -maths::CBoostedTreeFactory::TBoostedTreeUPtr -CDataFrameAnalyzerTest::getFinalTree(const TStrVec& persistedStates, - std::unique_ptr& frame, - std::size_t dependentVariable) const { - CTestDataSearcher dataSearcher(persistedStates.back()); - auto decompressor{std::make_unique(dataSearcher)}; - decompressor->setStateRestoreSearch(api::ML_STATE_INDEX, - api::getRegressionStateId("testJob")); - auto stream{decompressor->search(1, 1)}; - return maths::CBoostedTreeFactory::constructFromString(*stream).buildFor( - *frame, dependentVariable); -} diff --git a/lib/api/unittest/CDataFrameAnalyzerTest.h b/lib/api/unittest/CDataFrameAnalyzerTest.h index 7943653c08..60948f9d01 100644 --- a/lib/api/unittest/CDataFrameAnalyzerTest.h +++ b/lib/api/unittest/CDataFrameAnalyzerTest.h @@ -7,24 +7,9 @@ #ifndef INCLUDED_CDataFrameAnalyzerTest_h #define INCLUDED_CDataFrameAnalyzerTest_h -#include -#include - -#include -#include - -#include - #include class CDataFrameAnalyzerTest : public CppUnit::TestFixture { -public: - using TDataAdderUPtr = std::unique_ptr; - using TPersisterSupplier = std::function; - using TDataSearcherUPtr = std::unique_ptr; - using TRestoreSearcherSupplier = std::function; - using TDataFrameUPtr = std::unique_ptr; - public: void testWithoutControlMessages(); void testRunOutlierDetection(); @@ -41,25 +26,6 @@ class CDataFrameAnalyzerTest : public CppUnit::TestFixture { void testCategoricalFields(); static CppUnit::Test* suite(); - -private: - using TDoubleVec = std::vector; - using TStrVec = std::vector; - -private: - void testRunBoostedTreeTrainingWithStateRecoverySubroutine( - double lambda, - double gamma, - double eta, - std::size_t maximumNumberTrees, - double featureBagFraction, - std::size_t numberRoundsPerHyperparameter, - std::size_t iterationToRestartFrom) const; - - ml::maths::CBoostedTreeFactory::TBoostedTreeUPtr - getFinalTree(const TStrVec& persistedStates, - TDataFrameUPtr& frame, - std::size_t dependentVariable) const; }; #endif // INCLUDED_CDataFrameAnalyzerTest_h diff --git a/lib/maths/CBoostedTree.cc b/lib/maths/CBoostedTree.cc index 0049edf47c..c4181985b9 100644 --- a/lib/maths/CBoostedTree.cc +++ b/lib/maths/CBoostedTree.cc @@ -464,6 +464,22 @@ const CBoostedTree::TNodeVecVec& CBoostedTree::trainedModel() const { return m_Impl->trainedModel(); } +const std::string& CBoostedTree::bestHyperparametersName() { + return CBoostedTreeImpl::bestHyperparametersName(); +} + +const std::string& CBoostedTree::bestRegularizationHyperparametersName() { + return CBoostedTreeImpl::bestRegularizationHyperparametersName(); +} + +CBoostedTree::TStrVec CBoostedTree::bestHyperparameterNames() { + return CBoostedTreeImpl::bestHyperparameterNames(); +} + +bool CBoostedTree::acceptRestoreTraverser(core::CStateRestoreTraverser& traverser) { + return m_Impl->acceptRestoreTraverser(traverser); +} + void CBoostedTree::acceptPersistInserter(core::CStatePersistInserter& inserter) const { m_Impl->acceptPersistInserter(inserter); } diff --git a/lib/maths/CBoostedTreeImpl.cc b/lib/maths/CBoostedTreeImpl.cc index ecdc68edb9..31f8d4072c 100644 --- a/lib/maths/CBoostedTreeImpl.cc +++ b/lib/maths/CBoostedTreeImpl.cc @@ -1023,6 +1023,25 @@ const std::string HYPERPARAM_FEATURE_BAG_FRACTION_TAG{"hyperparam_feature_bag_fr const std::string HYPERPARAM_REGULARIZATION_TAG{"hyperparam_regularization"}; } +const std::string& CBoostedTreeImpl::bestHyperparametersName() { + return BEST_HYPERPARAMETERS_TAG; +} + +const std::string& CBoostedTreeImpl::bestRegularizationHyperparametersName() { + return HYPERPARAM_REGULARIZATION_TAG; +} + +CBoostedTreeImpl::TStrVec CBoostedTreeImpl::bestHyperparameterNames() { + return {HYPERPARAM_ETA_TAG, + HYPERPARAM_ETA_GROWTH_RATE_PER_TREE_TAG, + HYPERPARAM_FEATURE_BAG_FRACTION_TAG, + REGULARIZATION_DEPTH_PENALTY_MULTIPLIER_TAG, + REGULARIZATION_TREE_SIZE_PENALTY_MULTIPLIER_TAG, + REGULARIZATION_LEAF_WEIGHT_PENALTY_MULTIPLIER_TAG, + REGULARIZATION_SOFT_TREE_DEPTH_LIMIT_TAG, + REGULARIZATION_SOFT_TREE_DEPTH_TOLERANCE_TAG}; +} + template void CBoostedTreeImpl::CRegularization::acceptPersistInserter(core::CStatePersistInserter& inserter) const { core::CPersistUtils::persist(REGULARIZATION_DEPTH_PENALTY_MULTIPLIER_TAG, From c1b3ec184f1cd7f590b33bfe3cdf9af2da4b00b0 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 14:54:24 +0100 Subject: [PATCH 09/20] Typo --- include/maths/CBoostedTree.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/maths/CBoostedTree.h b/include/maths/CBoostedTree.h index 3d1374fc41..c745b6b8d3 100644 --- a/include/maths/CBoostedTree.h +++ b/include/maths/CBoostedTree.h @@ -67,7 +67,7 @@ class MATHS_EXPORT CArgMinMseImpl final : public CArgMinLossImpl { }; //! \brief Finds the value to add to a set of predicted log-odds which minimises -//! regularised the cross entropy loss w.r.t. the actual categories. +//! regularised cross entropy loss w.r.t. the actual categories. class MATHS_EXPORT CArgMinLogisticImpl final : public CArgMinLossImpl { public: CArgMinLogisticImpl(double lambda); From f01103699aba8c7e5a2ac7115863156d210602cc Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 14:56:10 +0100 Subject: [PATCH 10/20] Better variable name --- include/maths/CBoostedTree.h | 8 ++++---- lib/maths/CBoostedTree.cc | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/maths/CBoostedTree.h b/include/maths/CBoostedTree.h index c745b6b8d3..07ae20730f 100644 --- a/include/maths/CBoostedTree.h +++ b/include/maths/CBoostedTree.h @@ -84,24 +84,24 @@ class MATHS_EXPORT CArgMinLogisticImpl final : public CArgMinLossImpl { private: std::size_t bucket(double prediction) const { - double bucket{(prediction - m_MinMaxPrediction.min()) / this->bucketWidth()}; + double bucket{(prediction - m_PredictionMinMax.min()) / this->bucketWidth()}; return std::min(static_cast(bucket), m_BucketCategoryCounts.size() - 1); } double bucketCentre(std::size_t bucket) const { - return m_MinMaxPrediction.min() + + return m_PredictionMinMax.min() + (static_cast(bucket) + 0.5) * this->bucketWidth(); } double bucketWidth() const { - return m_MinMaxPrediction.range() / + return m_PredictionMinMax.range() / static_cast(m_BucketCategoryCounts.size()); } private: std::size_t m_CurrentPass = 0; - TMinMaxAccumulator m_MinMaxPrediction; + TMinMaxAccumulator m_PredictionMinMax; TSizeVector m_CategoryCounts; TSizeVectorVec m_BucketCategoryCounts; }; diff --git a/lib/maths/CBoostedTree.cc b/lib/maths/CBoostedTree.cc index c4181985b9..b6f0361992 100644 --- a/lib/maths/CBoostedTree.cc +++ b/lib/maths/CBoostedTree.cc @@ -85,7 +85,7 @@ bool CArgMinLogisticImpl::nextPass() { void CArgMinLogisticImpl::add(double prediction, double actual) { switch (m_CurrentPass) { case 0: { - m_MinMaxPrediction.add(prediction); + m_PredictionMinMax.add(prediction); ++m_CategoryCounts(static_cast(actual)); break; } @@ -104,7 +104,7 @@ void CArgMinLogisticImpl::merge(const CArgMinLossImpl& other) { if (logistic != nullptr) { switch (m_CurrentPass) { case 0: - m_MinMaxPrediction += logistic->m_MinMaxPrediction; + m_PredictionMinMax += logistic->m_PredictionMinMax; m_CategoryCounts += logistic->m_CategoryCounts; break; case 1: @@ -158,8 +158,8 @@ double CArgMinLogisticImpl::value() const { // Choose a weight interval in which all probabilites vary from close to // zero to close to one. - minWeight = -m_MinMaxPrediction.max() - 2.0; - maxWeight = -m_MinMaxPrediction.min() + 2.0; + minWeight = -m_PredictionMinMax.max() - 2.0; + maxWeight = -m_PredictionMinMax.min() + 2.0; } if (minWeight == maxWeight) { From 1b9c6a8fc1cfc34da0bf2a464b1b00de73655ee3 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 15:35:11 +0100 Subject: [PATCH 11/20] Explain change to include lambda in minimum additive weight --- lib/maths/CBoostedTree.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/maths/CBoostedTree.cc b/lib/maths/CBoostedTree.cc index b6f0361992..e9975fb553 100644 --- a/lib/maths/CBoostedTree.cc +++ b/lib/maths/CBoostedTree.cc @@ -62,6 +62,15 @@ void CArgMinMseImpl::merge(const CArgMinLossImpl& other) { } double CArgMinMseImpl::value() const { + + // We searching for the value x which minimises + // + // x^* = argmin_x{ sum_i{(a_i - (p_i + x))^2} + lambda * x^2 } + // + // This is convex so there is one minimum where derivative w.r.t. x is zero + // and x^* = 1 / (n + lambda) sum_i{ a_i - p_i }. Denoting the mean prediction + // error m = 1/n sum_i{ a_i - p_i } we have x^* = n / (n + lambda) m. + double count{CBasicStatistics::count(m_MeanError)}; return count == 0.0 ? 0.0 From dd77bb3018a63a04280113f8e88a3e98535aefba Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 15:40:35 +0100 Subject: [PATCH 12/20] Update out of date comment --- include/maths/CBoostedTree.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/maths/CBoostedTree.h b/include/maths/CBoostedTree.h index 07ae20730f..f528dd30cd 100644 --- a/include/maths/CBoostedTree.h +++ b/include/maths/CBoostedTree.h @@ -118,7 +118,9 @@ class MATHS_EXPORT CArgMinLoss { CArgMinLoss& operator=(const CArgMinLoss& other); CArgMinLoss& operator=(CArgMinLoss&& other) = default; - //! The number of passes over the data this needs. + //! Start another pass over the predictions and actuals. + //! + //! \return True if we need to perform another pass to compute value(). bool nextPass() const; //! Update with a point prediction and actual value. From 2b8835985a65ac7ae1bd5512a56c2d2b00cca1cc Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 15:45:30 +0100 Subject: [PATCH 13/20] Explain the bucket width check --- lib/maths/CBoostedTree.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/maths/CBoostedTree.cc b/lib/maths/CBoostedTree.cc index e9975fb553..f896164c4a 100644 --- a/lib/maths/CBoostedTree.cc +++ b/lib/maths/CBoostedTree.cc @@ -133,6 +133,9 @@ double CArgMinLogisticImpl::value() const { double minWeight; double maxWeight; + // This is true if and only if all the predictions were identical. In this + // case we only need one pass over the data and can compute the optimal + // value from the counts of the two categories. if (this->bucketWidth() == 0.0) { objective = [this](double weight) { double p{CTools::logisticFunction(weight)}; From 4cb6f0df3e3627907e14f833a5c143df62fe1e90 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 16:00:38 +0100 Subject: [PATCH 14/20] Explain magic minus signs --- lib/maths/CBoostedTree.cc | 9 ++++++--- lib/maths/unittest/CBoostedTreeTest.cc | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/maths/CBoostedTree.cc b/lib/maths/CBoostedTree.cc index f896164c4a..57bbfd703f 100644 --- a/lib/maths/CBoostedTree.cc +++ b/lib/maths/CBoostedTree.cc @@ -169,9 +169,12 @@ double CArgMinLogisticImpl::value() const { }; // Choose a weight interval in which all probabilites vary from close to - // zero to close to one. - minWeight = -m_PredictionMinMax.max() - 2.0; - maxWeight = -m_PredictionMinMax.min() + 2.0; + // zero to close to one. In particular, the idea is to minimize the leaf + // weight on an interval [a, b] where if we add "a" the log-odds for all + // rows <= -5, i.e. max prediction + a = -5, and if we add "b" the log-odds + // for all rows >= 5, i.e. min prediction + a = 5. + minWeight = -m_PredictionMinMax.max() - 5.0; + maxWeight = -m_PredictionMinMax.min() + 5.0; } if (minWeight == maxWeight) { diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index ecb273feb5..8299d6e715 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -908,6 +908,11 @@ void CBoostedTreeTest::testLogisticMinimizer() { max = std::max(max, weight[0]); } + // Choose a weight interval in which all probabilites vary from close to + // zero to close which we know will contain the true optimum. The idea is + // to minimize the leaf weight on an interval [a, b] where if we add "a" + // the log-odds for all rows <= 0, i.e. max prediction + a = 0, and if we + // add "b" the log-odds for all rows >= 0, i.e. min prediction + a = 0. double expected; double objectiveAtExpected; std::size_t maxIterations{20}; From 7c3dbcfa3bea3537778c64c8bbfe0cc9bc2de060 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 16:16:10 +0100 Subject: [PATCH 15/20] Extend test comment --- lib/maths/unittest/CBoostedTreeTest.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index 8299d6e715..cfeb9da7c9 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -961,8 +961,17 @@ void CBoostedTreeTest::testLogisticMinimizer() { void CBoostedTreeTest::testLogisticRegression() { - // Test we approximately minimise the cross entropy if the category labels - // are generated from log odds which are a linear function of the regressors. + // The idea of this test is to create a random linear relationship between + // the feature values and the log-odds of each class, i.e. + // + // log-odds(class_1) = sum_i{ w * x_i } + // + // where, w is some fixed weight vector and x_i denoted the i'th feature vector. + // We are try to recover this relationship in logistic regression by observing + // the actual labels. We want to test that we've roughly correctly estimated the + // log-odds. However, we target the cross-entropy so the errors in our estimates + // p_i^ should be measured in terms of cross entropy: sum_i{ p_i^ log(p_i) } + // where p_i = logistic(sum_i{ w_i * x_i}). test::CRandomNumbers rng; From 4b62cac0d0c2963c1c0b1b5a6f8fccd2b23f79eb Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 16:16:38 +0100 Subject: [PATCH 16/20] Explain loop --- lib/maths/unittest/CBoostedTreeTest.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index cfeb9da7c9..e894901b9b 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -890,6 +890,8 @@ void CBoostedTreeTest::testLogisticMinimizer() { return loss + lambda * maths::CTools::pow2(weight); }; + // This loop is fuzzing the predicted log-odds and testing we get consistently + // good estimates of the true minimizer. for (std::size_t t = 0; t < 10; ++t) { double min{std::numeric_limits::max()}; From 85aa19234abeebe7cf476b7321c5feafc7efad5c Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Tue, 8 Oct 2019 16:27:54 +0100 Subject: [PATCH 17/20] More descriptive names --- lib/maths/CBoostedTree.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/maths/CBoostedTree.cc b/lib/maths/CBoostedTree.cc index 57bbfd703f..0494478553 100644 --- a/lib/maths/CBoostedTree.cc +++ b/lib/maths/CBoostedTree.cc @@ -150,9 +150,12 @@ double CArgMinLogisticImpl::value() const { // between the logit of the empirical probability and zero. std::size_t c0{m_CategoryCounts(0) + 1}; std::size_t c1{m_CategoryCounts(1) + 1}; - double p{static_cast(c1) / static_cast(c0 + c1)}; - minWeight = p < 0.5 ? std::log(p / (1.0 - p)) : 0.0; - maxWeight = p < 0.5 ? 0.0 : std::log(p / (1.0 - p)); + double empiricalProbabilityC1{static_cast(c1) / + static_cast(c0 + c1)}; + double empiricalLogOddsC1{ + std::log(empiricalProbabilityC1 / (1.0 - empiricalProbabilityC1))}; + minWeight = empiricalProbabilityC1 < 0.5 ? empiricalLogOddsC1 : 0.0; + maxWeight = empiricalProbabilityC1 < 0.5 ? 0.0 : empiricalLogOddsC1; } else { objective = [this](double weight) { From cddab1f54d8f2900c956399c8e9c4e81502a2306 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 9 Oct 2019 09:42:20 +0100 Subject: [PATCH 18/20] Review comment --- lib/maths/unittest/CBoostedTreeTest.cc | 37 ++++++++++++-------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index e894901b9b..dc5e309761 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -984,27 +984,24 @@ void CBoostedTreeTest::testLogisticRegression() { TMeanAccumulator meanExcessCrossEntropy; for (std::size_t test = 0; test < 3; ++test) { - auto probability = [&] { - TDoubleVec weights; - rng.generateUniformSamples(-2.0, 2.0, cols - 1, weights); - TDoubleVec noise; - rng.generateNormalSamples(0.0, 1.0, rows, noise); - return [=](const TRowRef& row) { - double x{0.0}; - for (std::size_t i = 0; i < cols - 1; ++i) { - x += weights[i] * row[i]; - } - return maths::CTools::logisticFunction(x + noise[row.index()]); - }; - }(); + TDoubleVec weights; + rng.generateUniformSamples(-2.0, 2.0, cols - 1, weights); + TDoubleVec noise; + rng.generateNormalSamples(0.0, 1.0, rows, noise); + TDoubleVec uniform01; + rng.generateUniformSamples(0.0, 1.0, rows, uniform01); - auto target = [&] { - TDoubleVec uniform01; - rng.generateUniformSamples(0.0, 1.0, rows, uniform01); - return [=](const TRowRef& row) { - return uniform01[row.index()] < probability(row) ? 1.0 : 0.0; - }; - }(); + auto probability = [&](const TRowRef& row) { + double x{0.0}; + for (std::size_t i = 0; i < cols - 1; ++i) { + x += weights[i] * row[i]; + } + return maths::CTools::logisticFunction(x + noise[row.index()]); + }; + + auto target = [&](const TRowRef& row) { + return uniform01[row.index()] < probability(row) ? 1.0 : 0.0; + }; TDoubleVecVec x(cols - 1); for (std::size_t i = 0; i < cols - 1; ++i) { From 4fed5ec1f70a68fdbee871ab043e966539769df8 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 9 Oct 2019 13:17:40 +0100 Subject: [PATCH 19/20] Typo Co-Authored-By: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> --- lib/maths/unittest/CBoostedTreeTest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index dc5e309761..c1ee4801bb 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -969,7 +969,7 @@ void CBoostedTreeTest::testLogisticRegression() { // log-odds(class_1) = sum_i{ w * x_i } // // where, w is some fixed weight vector and x_i denoted the i'th feature vector. - // We are try to recover this relationship in logistic regression by observing + // We try to recover this relationship in logistic regression by observing // the actual labels. We want to test that we've roughly correctly estimated the // log-odds. However, we target the cross-entropy so the errors in our estimates // p_i^ should be measured in terms of cross entropy: sum_i{ p_i^ log(p_i) } From a09929d5b43a293c2be21eed344364a9882d4a66 Mon Sep 17 00:00:00 2001 From: Tom Veasey Date: Wed, 9 Oct 2019 13:33:51 +0100 Subject: [PATCH 20/20] Further comment tweak --- lib/maths/CBoostedTree.cc | 2 +- lib/maths/unittest/CBoostedTreeTest.cc | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/maths/CBoostedTree.cc b/lib/maths/CBoostedTree.cc index 0494478553..b1fbe2a74b 100644 --- a/lib/maths/CBoostedTree.cc +++ b/lib/maths/CBoostedTree.cc @@ -67,7 +67,7 @@ double CArgMinMseImpl::value() const { // // x^* = argmin_x{ sum_i{(a_i - (p_i + x))^2} + lambda * x^2 } // - // This is convex so there is one minimum where derivative w.r.t. x is zero + // This is convex so there is one minimum where the derivative w.r.t. x is zero // and x^* = 1 / (n + lambda) sum_i{ a_i - p_i }. Denoting the mean prediction // error m = 1/n sum_i{ a_i - p_i } we have x^* = n / (n + lambda) m. diff --git a/lib/maths/unittest/CBoostedTreeTest.cc b/lib/maths/unittest/CBoostedTreeTest.cc index dc5e309761..65185986e9 100644 --- a/lib/maths/unittest/CBoostedTreeTest.cc +++ b/lib/maths/unittest/CBoostedTreeTest.cc @@ -910,11 +910,6 @@ void CBoostedTreeTest::testLogisticMinimizer() { max = std::max(max, weight[0]); } - // Choose a weight interval in which all probabilites vary from close to - // zero to close which we know will contain the true optimum. The idea is - // to minimize the leaf weight on an interval [a, b] where if we add "a" - // the log-odds for all rows <= 0, i.e. max prediction + a = 0, and if we - // add "b" the log-odds for all rows >= 0, i.e. min prediction + a = 0. double expected; double objectiveAtExpected; std::size_t maxIterations{20};