From 4bc1285debaeb1e1e6924b5490e584215f787ef4 Mon Sep 17 00:00:00 2001 From: GitName Date: Tue, 24 Nov 2020 13:45:54 +0100 Subject: [PATCH 1/7] add macro-averaged mean absolute error --- doc/api.rst | 1 + doc/bibtex/refs.bib | 16 ++++++ doc/metrics.rst | 7 +++ imblearn/metrics/__init__.py | 2 + imblearn/metrics/_classification.py | 51 +++++++++++++++++++ imblearn/metrics/tests/test_classification.py | 16 ++++++ 6 files changed, 93 insertions(+) diff --git a/doc/api.rst b/doc/api.rst index 07ac6413c..7ee341874 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -216,6 +216,7 @@ Imbalance-learn provides some fast-prototyping tools. metrics.specificity_score metrics.geometric_mean_score metrics.make_index_balanced_accuracy + metrics.macro_averaged_mean_absolute_error .. _datasets_ref: diff --git a/doc/bibtex/refs.bib b/doc/bibtex/refs.bib index 87eb3e30c..63f8bc0d1 100644 --- a/doc/bibtex/refs.bib +++ b/doc/bibtex/refs.bib @@ -207,4 +207,20 @@ @article{torelli2014rose issn = {1573-756X}, url = {https://doi.org/10.1007/s10618-012-0295-5}, doi = {10.1007/s10618-012-0295-5} +} + +@article{esuli2009ordinal, + author = {A. Esuli and S. Baccianella and F. Sebastiani}, + title = {Evaluation Measures for Ordinal Regression}, + journal = {Intelligent Systems Design and Applications, International Conference on}, + year = {2009}, + volume = {1}, + issn = {}, + pages = {283-287}, + keywords = {ordinal regression;ordinal classification;evaluation measures;class imbalance;product reviews}, + doi = {10.1109/ISDA.2009.230}, + url = {https://doi.ieeecomputersociety.org/10.1109/ISDA.2009.230}, + publisher = {IEEE Computer Society}, + address = {Los Alamitos, CA, USA}, + month = {dec} } \ No newline at end of file diff --git a/doc/metrics.rst b/doc/metrics.rst index 8b9474ca3..f513dba74 100644 --- a/doc/metrics.rst +++ b/doc/metrics.rst @@ -54,3 +54,10 @@ The :func:`classification_report_imbalanced` will compute a set of metrics per class and summarize it in a table. The parameter `output_dict` allows to get a string or a Python dictionary. This dictionary can be reused to create a Pandas dataframe for instance. + +.. _macro_averaged_mean_absolute_error: + +Macro-Averaged Mean Absolute Error (MA-MAE) +------------------------------------------- + +The :func:`macro_averaged_mean_absolute_error` :cite:`esuli2009ordinal` is used for imbalanced ordinal classification. We compute each MAE for each class and average them, giving an equal weight to each class. diff --git a/imblearn/metrics/__init__.py b/imblearn/metrics/__init__.py index 3097a292c..ffc730f82 100644 --- a/imblearn/metrics/__init__.py +++ b/imblearn/metrics/__init__.py @@ -9,6 +9,7 @@ from ._classification import geometric_mean_score from ._classification import make_index_balanced_accuracy from ._classification import classification_report_imbalanced +from ._classification import macro_averaged_mean_absolute_error __all__ = [ "sensitivity_specificity_support", @@ -17,4 +18,5 @@ "geometric_mean_score", "make_index_balanced_accuracy", "classification_report_imbalanced", + "macro_averaged_mean_absolute_error" ] diff --git a/imblearn/metrics/_classification.py b/imblearn/metrics/_classification.py index 5ab8c3f23..7e574b0e6 100644 --- a/imblearn/metrics/_classification.py +++ b/imblearn/metrics/_classification.py @@ -18,6 +18,7 @@ import numpy as np import scipy as sp +from sklearn.metrics import mean_absolute_error from sklearn.metrics import precision_recall_fscore_support from sklearn.metrics._classification import _check_targets from sklearn.metrics._classification import _prf_divide @@ -997,3 +998,53 @@ class 2 1.00 0.67 1.00 0.80 0.82 0.64\ if output_dict: return report_dict return report + + +@_deprecate_positional_args +def macro_averaged_mean_absolute_error(y_true, y_pred): + """Compute Macro-Averaged Mean Absolute Error (MA-MAE) for imbalanced ordinal classification. + + This function computes each MAE for each class and average them, giving an equal weight to each class. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + y_true : 1d array-like, or label indicator array / sparse matrix + Ground truth (correct) target values. + + y_pred : 1d array-like, or label indicator array / sparse matrix + Estimated targets as returned by a classifier. + + Returns + ------- + loss : float or ndarray of floats + Macro-Avaeraged MAE output is non-negative floating point. The best value is 0.0. + + Examples + -------- + >>> import numpy as np + >>> from sklearn.metrics import mean_absolute_error + >>> from imblearn.metrics import macro_averaged_mean_absolute_error + >>> y_true_balanced = [1, 1, 1, 2, 2, 2] + >>> y_true_imbalanced = [1, 1, 1, 1, 1, 2] + >>> y_pred = [1, 2, 1, 2, 1, 2] + >>> mean_absolute_error(y_true_balanced, y_pred) + 0.3333333333333333 + >>> mean_absolute_error(y_true_imbalanced, y_pred) + 0.3333333333333333 + >>> macro_averaged_mean_absolute_error(y_true_balanced, y_pred) + 0.3333333333333333 + >>> macro_averaged_mean_absolute_error(y_true_imbalanced, y_pred) + 0.2 + + """ + all_mae = [] + y_true = np.array(y_true) + y_pred = np.array(y_pred) + for class_to_predict in np.unique(y_true): + index_class_to_predict = np.where(y_true == class_to_predict)[0] + mae_class = mean_absolute_error(y_true[index_class_to_predict], y_pred[index_class_to_predict]) + all_mae.append(mae_class) + ma_mae = sum(all_mae) / len(all_mae) + return ma_mae diff --git a/imblearn/metrics/tests/test_classification.py b/imblearn/metrics/tests/test_classification.py index b6db641ce..a40f95692 100644 --- a/imblearn/metrics/tests/test_classification.py +++ b/imblearn/metrics/tests/test_classification.py @@ -29,6 +29,7 @@ from imblearn.metrics import geometric_mean_score from imblearn.metrics import make_index_balanced_accuracy from imblearn.metrics import classification_report_imbalanced +from imblearn.metrics import macro_averaged_mean_absolute_error from imblearn.utils.testing import warns @@ -498,3 +499,18 @@ def test_classification_report_imbalanced_dict(): assert outer_keys == expected_outer_keys assert inner_keys == expected_inner_keys + + +@pytest.mark.parametrize( + "y_true, y_pred, expected_ma_mae", + [ + ([1, 1, 1, 2, 2, 2], [1, 2, 1, 2, 1, 2], 0.333), + ([1, 1, 1, 1, 1, 2], [1, 2, 1, 2, 1, 2], 0.2), + ([1, 1, 1, 2, 2, 2, 3, 3, 3], [1, 3, 1, 2, 1, 1, 2, 3, 3], 0.555), + ([1, 1, 1, 1, 1, 1, 2, 3, 3], [1, 3, 1, 2, 1, 1, 2, 3, 3], 0.166), + + ], +) +def test_macro_averaged_mean_absolute_error(y_true, y_pred, expected_ma_mae): + ma_mae = macro_averaged_mean_absolute_error(y_true, y_pred) + assert ma_mae == pytest.approx(expected_ma_mae, rel=R_TOL) From 601f6bf50d750c8a30db004cf6ef3b3b883f4306 Mon Sep 17 00:00:00 2001 From: GitName Date: Tue, 24 Nov 2020 14:12:18 +0100 Subject: [PATCH 2/7] fix linting --- imblearn/metrics/_classification.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/imblearn/metrics/_classification.py b/imblearn/metrics/_classification.py index 7e574b0e6..e675a0377 100644 --- a/imblearn/metrics/_classification.py +++ b/imblearn/metrics/_classification.py @@ -1002,9 +1002,11 @@ class 2 1.00 0.67 1.00 0.80 0.82 0.64\ @_deprecate_positional_args def macro_averaged_mean_absolute_error(y_true, y_pred): - """Compute Macro-Averaged Mean Absolute Error (MA-MAE) for imbalanced ordinal classification. + """Compute Macro-Averaged Mean Absolute Error (MA-MAE) + for imbalanced ordinal classification. - This function computes each MAE for each class and average them, giving an equal weight to each class. + This function computes each MAE for each class and average them, + giving an equal weight to each class. Read more in the :ref:`User Guide `. @@ -1019,7 +1021,8 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): Returns ------- loss : float or ndarray of floats - Macro-Avaeraged MAE output is non-negative floating point. The best value is 0.0. + Macro-Averaged MAE output is non-negative floating point. + The best value is 0.0. Examples -------- @@ -1044,7 +1047,8 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): y_pred = np.array(y_pred) for class_to_predict in np.unique(y_true): index_class_to_predict = np.where(y_true == class_to_predict)[0] - mae_class = mean_absolute_error(y_true[index_class_to_predict], y_pred[index_class_to_predict]) + mae_class = mean_absolute_error(y_true[index_class_to_predict], + y_pred[index_class_to_predict]) all_mae.append(mae_class) ma_mae = sum(all_mae) / len(all_mae) return ma_mae From 2db5794d249e7ac4041909cb773b8a53f3085524 Mon Sep 17 00:00:00 2001 From: GitName Date: Tue, 24 Nov 2020 14:26:56 +0100 Subject: [PATCH 3/7] fix doctest --- imblearn/metrics/_classification.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/imblearn/metrics/_classification.py b/imblearn/metrics/_classification.py index e675a0377..c85c38159 100644 --- a/imblearn/metrics/_classification.py +++ b/imblearn/metrics/_classification.py @@ -1032,13 +1032,15 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): >>> y_true_balanced = [1, 1, 1, 2, 2, 2] >>> y_true_imbalanced = [1, 1, 1, 1, 1, 2] >>> y_pred = [1, 2, 1, 2, 1, 2] - >>> mean_absolute_error(y_true_balanced, y_pred) - 0.3333333333333333 - >>> mean_absolute_error(y_true_imbalanced, y_pred) - 0.3333333333333333 - >>> macro_averaged_mean_absolute_error(y_true_balanced, y_pred) - 0.3333333333333333 - >>> macro_averaged_mean_absolute_error(y_true_imbalanced, y_pred) + >>> np.round(mean_absolute_error(y_true_balanced, y_pred), 4) + 0.3333 + >>> np.round(mean_absolute_error(y_true_imbalanced, y_pred), 4) + 0.3333 + >>> np.round(macro_averaged_mean_absolute_error(y_true_balanced, y_pred), + 4) + 0.3333 + >>> np.round(macro_averaged_mean_absolute_error(y_true_imbalanced, y_pred, + 4) 0.2 """ From 05da8dada8e601b73fd06f416e938e5a53c5e7b5 Mon Sep 17 00:00:00 2001 From: GitName Date: Tue, 24 Nov 2020 14:46:57 +0100 Subject: [PATCH 4/7] fix docstring --- imblearn/metrics/_classification.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/imblearn/metrics/_classification.py b/imblearn/metrics/_classification.py index c85c38159..0775dabbd 100644 --- a/imblearn/metrics/_classification.py +++ b/imblearn/metrics/_classification.py @@ -1029,19 +1029,18 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): >>> import numpy as np >>> from sklearn.metrics import mean_absolute_error >>> from imblearn.metrics import macro_averaged_mean_absolute_error - >>> y_true_balanced = [1, 1, 1, 2, 2, 2] - >>> y_true_imbalanced = [1, 1, 1, 1, 1, 2] - >>> y_pred = [1, 2, 1, 2, 1, 2] - >>> np.round(mean_absolute_error(y_true_balanced, y_pred), 4) - 0.3333 - >>> np.round(mean_absolute_error(y_true_imbalanced, y_pred), 4) - 0.3333 - >>> np.round(macro_averaged_mean_absolute_error(y_true_balanced, y_pred), - 4) - 0.3333 + >>> y_true_balanced = [1, 1, 2, 2] + >>> y_true_imbalanced = [1, 2, 2, 2] + >>> y_pred = [1, 2, 1, 2] + >>> mean_absolute_error(y_true_balanced, y_pred) + 0.5 + >>> mean_absolute_error(y_true_imbalanced, y_pred) + 0.25 + >>> macro_averaged_mean_absolute_error(y_true_balanced, y_pred) + 0.5 >>> np.round(macro_averaged_mean_absolute_error(y_true_imbalanced, y_pred, 4) - 0.2 + 0.1667 """ all_mae = [] From de54ace7d6f82b5631454ecaeffe34b3da2cb9d5 Mon Sep 17 00:00:00 2001 From: GitName Date: Tue, 24 Nov 2020 14:55:21 +0100 Subject: [PATCH 5/7] fix float approximation in docstring --- imblearn/metrics/_classification.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/imblearn/metrics/_classification.py b/imblearn/metrics/_classification.py index 0775dabbd..e113ec1d2 100644 --- a/imblearn/metrics/_classification.py +++ b/imblearn/metrics/_classification.py @@ -1038,9 +1038,8 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): 0.25 >>> macro_averaged_mean_absolute_error(y_true_balanced, y_pred) 0.5 - >>> np.round(macro_averaged_mean_absolute_error(y_true_imbalanced, y_pred, - 4) - 0.1667 + >>> macro_averaged_mean_absolute_error(y_true_imbalanced, y_pred) + 0.16666666666666666 """ all_mae = [] From 26eeabe6fd559a1759273fcdace0158da6884e51 Mon Sep 17 00:00:00 2001 From: GitName Date: Tue, 24 Nov 2020 16:00:23 +0100 Subject: [PATCH 6/7] review --- doc/metrics.rst | 19 ++++++++++++------- imblearn/metrics/_classification.py | 17 ++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/doc/metrics.rst b/doc/metrics.rst index f513dba74..98a748b8c 100644 --- a/doc/metrics.rst +++ b/doc/metrics.rst @@ -45,6 +45,18 @@ The :func:`make_index_balanced_accuracy` :cite:`garcia2012effectiveness` can wrap any metric and give more importance to a specific class using the parameter ``alpha``. +.. _macro_averaged_mean_absolute_error: + +Macro-Averaged Mean Absolute Error (MA-MAE) +------------------------------------------- + +Ordinal classification is used when there is a rank among classes, for example +levels of functionality or movie ratings. + +The :func:`macro_averaged_mean_absolute_error` :cite:`esuli2009ordinal` is used +for imbalanced ordinal classification. We compute each MAE for each class and +average them, giving an equal weight to each class. + .. _classification_report: Summary of important metrics @@ -54,10 +66,3 @@ The :func:`classification_report_imbalanced` will compute a set of metrics per class and summarize it in a table. The parameter `output_dict` allows to get a string or a Python dictionary. This dictionary can be reused to create a Pandas dataframe for instance. - -.. _macro_averaged_mean_absolute_error: - -Macro-Averaged Mean Absolute Error (MA-MAE) -------------------------------------------- - -The :func:`macro_averaged_mean_absolute_error` :cite:`esuli2009ordinal` is used for imbalanced ordinal classification. We compute each MAE for each class and average them, giving an equal weight to each class. diff --git a/imblearn/metrics/_classification.py b/imblearn/metrics/_classification.py index e113ec1d2..94629842e 100644 --- a/imblearn/metrics/_classification.py +++ b/imblearn/metrics/_classification.py @@ -1012,10 +1012,10 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): Parameters ---------- - y_true : 1d array-like, or label indicator array / sparse matrix + y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) Ground truth (correct) target values. - y_pred : 1d array-like, or label indicator array / sparse matrix + y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) Estimated targets as returned by a classifier. Returns @@ -1033,20 +1033,19 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): >>> y_true_imbalanced = [1, 2, 2, 2] >>> y_pred = [1, 2, 1, 2] >>> mean_absolute_error(y_true_balanced, y_pred) - 0.5 + 0.5 >>> mean_absolute_error(y_true_imbalanced, y_pred) - 0.25 + 0.25 >>> macro_averaged_mean_absolute_error(y_true_balanced, y_pred) - 0.5 + 0.5 >>> macro_averaged_mean_absolute_error(y_true_imbalanced, y_pred) - 0.16666666666666666 + 0.16666666666666666 """ all_mae = [] - y_true = np.array(y_true) - y_pred = np.array(y_pred) + y_true, y_pred = np.asarray(y_true), np.asarray(y_pred) for class_to_predict in np.unique(y_true): - index_class_to_predict = np.where(y_true == class_to_predict)[0] + index_class_to_predict = np.flatnonzero(y_true == class_to_predict) mae_class = mean_absolute_error(y_true[index_class_to_predict], y_pred[index_class_to_predict]) all_mae.append(mae_class) From d04635edb5fbd756ad95a25260b67dafe62ac870 Mon Sep 17 00:00:00 2001 From: Guillaume Lemaitre Date: Tue, 9 Feb 2021 00:14:47 +0100 Subject: [PATCH 7/7] revie --- doc/api.rst | 2 +- doc/metrics.rst | 4 +- doc/whats_new/v0.7.rst | 6 +++ imblearn/metrics/__init__.py | 2 +- imblearn/metrics/_classification.py | 41 +++++++++++++------ imblearn/metrics/tests/test_classification.py | 14 +++++++ 6 files changed, 52 insertions(+), 17 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 7ee341874..bdf85e0bb 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -215,8 +215,8 @@ Imbalance-learn provides some fast-prototyping tools. metrics.sensitivity_score metrics.specificity_score metrics.geometric_mean_score - metrics.make_index_balanced_accuracy metrics.macro_averaged_mean_absolute_error + metrics.make_index_balanced_accuracy .. _datasets_ref: diff --git a/doc/metrics.rst b/doc/metrics.rst index 98a748b8c..98368a650 100644 --- a/doc/metrics.rst +++ b/doc/metrics.rst @@ -54,8 +54,8 @@ Ordinal classification is used when there is a rank among classes, for example levels of functionality or movie ratings. The :func:`macro_averaged_mean_absolute_error` :cite:`esuli2009ordinal` is used -for imbalanced ordinal classification. We compute each MAE for each class and -average them, giving an equal weight to each class. +for imbalanced ordinal classification. The mean absolute error is computed for +each class and averaged over classes, giving an equal weight to each class. .. _classification_report: diff --git a/doc/whats_new/v0.7.rst b/doc/whats_new/v0.7.rst index 6cb266854..bca4a680d 100644 --- a/doc/whats_new/v0.7.rst +++ b/doc/whats_new/v0.7.rst @@ -76,6 +76,12 @@ Enhancements dictionary instead of a string. :pr:`770` by :user:`Guillaume Lemaitre `. +- Add the the function + :func:`imblearn.metrics.macro_averaged_mean_absolute_error` returning the + average across class of the MAE. This metric is used in ordinal + classification. + :pr:`780` by :user:`Aurélien Massiot `. + Deprecation ........... diff --git a/imblearn/metrics/__init__.py b/imblearn/metrics/__init__.py index ffc730f82..ffab645a1 100644 --- a/imblearn/metrics/__init__.py +++ b/imblearn/metrics/__init__.py @@ -18,5 +18,5 @@ "geometric_mean_score", "make_index_balanced_accuracy", "classification_report_imbalanced", - "macro_averaged_mean_absolute_error" + "macro_averaged_mean_absolute_error", ] diff --git a/imblearn/metrics/_classification.py b/imblearn/metrics/_classification.py index 94629842e..90ac6d51a 100644 --- a/imblearn/metrics/_classification.py +++ b/imblearn/metrics/_classification.py @@ -22,9 +22,12 @@ from sklearn.metrics import precision_recall_fscore_support from sklearn.metrics._classification import _check_targets from sklearn.metrics._classification import _prf_divide - from sklearn.preprocessing import LabelEncoder from sklearn.utils.multiclass import unique_labels +from sklearn.utils.validation import ( + check_consistent_length, + column_or_1d, +) try: from inspect import signature @@ -1000,8 +1003,7 @@ class 2 1.00 0.67 1.00 0.80 0.82 0.64\ return report -@_deprecate_positional_args -def macro_averaged_mean_absolute_error(y_true, y_pred): +def macro_averaged_mean_absolute_error(y_true, y_pred, *, sample_weight=None): """Compute Macro-Averaged Mean Absolute Error (MA-MAE) for imbalanced ordinal classification. @@ -1018,6 +1020,9 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) Estimated targets as returned by a classifier. + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + Returns ------- loss : float or ndarray of floats @@ -1040,14 +1045,24 @@ def macro_averaged_mean_absolute_error(y_true, y_pred): 0.5 >>> macro_averaged_mean_absolute_error(y_true_imbalanced, y_pred) 0.16666666666666666 - """ - all_mae = [] - y_true, y_pred = np.asarray(y_true), np.asarray(y_pred) - for class_to_predict in np.unique(y_true): - index_class_to_predict = np.flatnonzero(y_true == class_to_predict) - mae_class = mean_absolute_error(y_true[index_class_to_predict], - y_pred[index_class_to_predict]) - all_mae.append(mae_class) - ma_mae = sum(all_mae) / len(all_mae) - return ma_mae + _, y_true, y_pred = _check_targets(y_true, y_pred) + if sample_weight is not None: + sample_weight = column_or_1d(sample_weight) + else: + sample_weight = np.ones(y_true.shape) + check_consistent_length(y_true, y_pred, sample_weight) + labels = unique_labels(y_true, y_pred) + mae = [] + for possible_class in labels: + indices = np.flatnonzero(y_true == possible_class) + + mae.append( + mean_absolute_error( + y_true[indices], + y_pred[indices], + sample_weight=sample_weight[indices], + ) + ) + + return np.sum(mae) / len(mae) diff --git a/imblearn/metrics/tests/test_classification.py b/imblearn/metrics/tests/test_classification.py index a40f95692..3bab9b044 100644 --- a/imblearn/metrics/tests/test_classification.py +++ b/imblearn/metrics/tests/test_classification.py @@ -514,3 +514,17 @@ def test_classification_report_imbalanced_dict(): def test_macro_averaged_mean_absolute_error(y_true, y_pred, expected_ma_mae): ma_mae = macro_averaged_mean_absolute_error(y_true, y_pred) assert ma_mae == pytest.approx(expected_ma_mae, rel=R_TOL) + + +def test_macro_averaged_mean_absolute_error_sample_weight(): + y_true = [1, 1, 1, 2, 2, 2] + y_pred = [1, 2, 1, 2, 1, 2] + + ma_mae_no_weights = macro_averaged_mean_absolute_error(y_true, y_pred) + + sample_weight = [1, 1, 1, 1, 1, 1] + ma_mae_unit_weights = macro_averaged_mean_absolute_error( + y_true, y_pred, sample_weight=sample_weight, + ) + + assert ma_mae_unit_weights == pytest.approx(ma_mae_no_weights)