diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index 7f6f5c910a3..f4ae4ffbf89 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -995,6 +995,8 @@ details. metrics.median_absolute_error metrics.mean_absolute_percentage_error metrics.r2_score + metrics.root_mean_squared_log_error + metrics.root_mean_squared_error metrics.mean_poisson_deviance metrics.mean_gamma_deviance metrics.mean_tweedie_deviance diff --git a/doc/modules/model_evaluation.rst b/doc/modules/model_evaluation.rst index c989f49c0fc..eac1cde5675 100644 --- a/doc/modules/model_evaluation.rst +++ b/doc/modules/model_evaluation.rst @@ -94,8 +94,9 @@ Scoring Function 'max_error' :func:`metrics.max_error` 'neg_mean_absolute_error' :func:`metrics.mean_absolute_error` 'neg_mean_squared_error' :func:`metrics.mean_squared_error` -'neg_root_mean_squared_error' :func:`metrics.mean_squared_error` +'neg_root_mean_squared_error' :func:`metrics.root_mean_squared_error` 'neg_mean_squared_log_error' :func:`metrics.mean_squared_log_error` +'neg_root_mean_squared_log_error' :func:`metrics.root_mean_squared_log_error` 'neg_median_absolute_error' :func:`metrics.median_absolute_error` 'r2' :func:`metrics.r2_score` 'neg_mean_poisson_deviance' :func:`metrics.mean_poisson_deviance` @@ -2310,6 +2311,10 @@ function:: for an example of mean squared error usage to evaluate gradient boosting regression. +Taking the square root of the MSE, called the root mean squared error (RMSE), is another +common metric that provides a measure in the same units as the target variable. RSME is +available through the :func:`root_mean_squared_error` function. + .. _mean_squared_log_error: Mean squared logarithmic error @@ -2347,6 +2352,9 @@ function:: >>> mean_squared_log_error(y_true, y_pred) 0.044... +The root mean squared logarithmic error (RMSLE) is available through the +:func:`root_mean_squared_log_error` function. + .. _mean_absolute_percentage_error: Mean absolute percentage error diff --git a/doc/whats_new/v1.4.rst b/doc/whats_new/v1.4.rst index 367315dc3c6..8dbd867b0c9 100644 --- a/doc/whats_new/v1.4.rst +++ b/doc/whats_new/v1.4.rst @@ -238,6 +238,18 @@ Changelog - |Enhancement| :class:`preprocessing.TargetEncoder` now supports `target_type` 'multiclass'. :pr:`26674` by :user:`Lucy Liu `. +:mod:`sklearn.metrics` +...................... + +- |API| The `squared` parameter of :func:`metrics.mean_squared_error` and + :func:`metrics.mean_squared_log_error` is deprecated and will be removed in 1.6. + Use the new functions :func:`metrics.root_mean_squared_error` and + :func:`root_mean_squared_log_error` instead. + :pr:`26734` by :user:`Alejandro Martin Gil <101AlexMartin>`. + +- |Enhancement| Added `neg_root_mean_squared_log_error_scorer` as scorer + :pr:`26734` by :user:`Alejandro Martin Gil <101AlexMartin>`. + :mod:`sklearn.model_selection` .............................. diff --git a/sklearn/metrics/__init__.py b/sklearn/metrics/__init__.py index 488c776ae9a..713c5fe651d 100644 --- a/sklearn/metrics/__init__.py +++ b/sklearn/metrics/__init__.py @@ -62,6 +62,8 @@ from ._regression import ( mean_tweedie_deviance, median_absolute_error, r2_score, + root_mean_squared_error, + root_mean_squared_log_error, ) from ._scorer import check_scoring, get_scorer, get_scorer_names, make_scorer from .cluster import ( @@ -166,6 +168,8 @@ __all__ = [ "RocCurveDisplay", "roc_auc_score", "roc_curve", + "root_mean_squared_log_error", + "root_mean_squared_error", "get_scorer_names", "silhouette_samples", "silhouette_score", diff --git a/sklearn/metrics/_regression.py b/sklearn/metrics/_regression.py index a6dfacf30d3..77a93f4c175 100644 --- a/sklearn/metrics/_regression.py +++ b/sklearn/metrics/_regression.py @@ -24,6 +24,7 @@ the lower the better. # Uttam kumar # Sylvain Marie # Ohad Michel +# Alejandro Martin Gil # License: BSD 3 clause import warnings @@ -33,7 +34,7 @@ import numpy as np from scipy.special import xlogy from ..exceptions import UndefinedMetricWarning -from ..utils._param_validation import Interval, StrOptions, validate_params +from ..utils._param_validation import Hidden, Interval, StrOptions, validate_params from ..utils.stats import _weighted_percentile from ..utils.validation import ( _check_sample_weight, @@ -52,6 +53,8 @@ __ALL__ = [ "mean_absolute_percentage_error", "mean_pinball_loss", "r2_score", + "root_mean_squared_log_error", + "root_mean_squared_error", "explained_variance_score", "mean_tweedie_deviance", "mean_poisson_deviance", @@ -407,12 +410,17 @@ def mean_absolute_percentage_error( "y_pred": ["array-like"], "sample_weight": ["array-like", None], "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"], - "squared": ["boolean"], + "squared": [Hidden(StrOptions({"deprecated"})), "boolean"], }, prefer_skip_nested_validation=True, ) def mean_squared_error( - y_true, y_pred, *, sample_weight=None, multioutput="uniform_average", squared=True + y_true, + y_pred, + *, + sample_weight=None, + multioutput="uniform_average", + squared="deprecated", ): """Mean squared error regression loss. @@ -443,6 +451,11 @@ def mean_squared_error( squared : bool, default=True If True returns MSE value, if False returns RMSE value. + .. deprecated:: 1.4 + `squared` is deprecated in 1.4 and will be removed in 1.6. + Use :func:`~sklearn.metrics.root_mean_squared_error` + instead to calculate the root mean squared error. + Returns ------- loss : float or ndarray of floats @@ -456,30 +469,37 @@ def mean_squared_error( >>> y_pred = [2.5, 0.0, 2, 8] >>> mean_squared_error(y_true, y_pred) 0.375 - >>> y_true = [3, -0.5, 2, 7] - >>> y_pred = [2.5, 0.0, 2, 8] - >>> mean_squared_error(y_true, y_pred, squared=False) - 0.612... >>> y_true = [[0.5, 1],[-1, 1],[7, -6]] >>> y_pred = [[0, 2],[-1, 2],[8, -5]] >>> mean_squared_error(y_true, y_pred) 0.708... - >>> mean_squared_error(y_true, y_pred, squared=False) - 0.822... >>> mean_squared_error(y_true, y_pred, multioutput='raw_values') array([0.41666667, 1. ]) >>> mean_squared_error(y_true, y_pred, multioutput=[0.3, 0.7]) 0.825... """ + # TODO(1.6): remove + if squared != "deprecated": + warnings.warn( + ( + "'squared' is deprecated in version 1.4 and " + "will be removed in 1.6. To calculate the " + "root mean squared error, use the function" + "'root_mean_squared_error'." + ), + FutureWarning, + ) + if not squared: + return root_mean_squared_error( + y_true, y_pred, sample_weight=sample_weight, multioutput=multioutput + ) + y_type, y_true, y_pred, multioutput = _check_reg_targets( y_true, y_pred, multioutput ) check_consistent_length(y_true, y_pred, sample_weight) output_errors = np.average((y_true - y_pred) ** 2, axis=0, weights=sample_weight) - if not squared: - output_errors = np.sqrt(output_errors) - if isinstance(multioutput, str): if multioutput == "raw_values": return output_errors @@ -496,12 +516,91 @@ def mean_squared_error( "y_pred": ["array-like"], "sample_weight": ["array-like", None], "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"], - "squared": ["boolean"], + }, + prefer_skip_nested_validation=True, +) +def root_mean_squared_error( + y_true, y_pred, *, sample_weight=None, multioutput="uniform_average" +): + """Root mean squared error regression loss. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + + Parameters + ---------- + y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + + y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + multioutput : {'raw_values', 'uniform_average'} or array-like of shape \ + (n_outputs,), default='uniform_average' + Defines aggregating of multiple output values. + Array-like value defines weights used to average errors. + + 'raw_values' : + Returns a full set of errors in case of multioutput input. + + 'uniform_average' : + Errors of all outputs are averaged with uniform weight. + + Returns + ------- + loss : float or ndarray of floats + A non-negative floating point value (the best value is 0.0), or an + array of floating point values, one for each individual target. + + Examples + -------- + >>> from sklearn.metrics import root_mean_squared_error + >>> y_true = [3, -0.5, 2, 7] + >>> y_pred = [2.5, 0.0, 2, 8] + >>> root_mean_squared_error(y_true, y_pred) + 0.612... + >>> y_true = [[0.5, 1],[-1, 1],[7, -6]] + >>> y_pred = [[0, 2],[-1, 2],[8, -5]] + >>> root_mean_squared_error(y_true, y_pred) + 0.822... + """ + output_errors = np.sqrt( + mean_squared_error( + y_true, y_pred, sample_weight=sample_weight, multioutput="raw_values" + ) + ) + + if isinstance(multioutput, str): + if multioutput == "raw_values": + return output_errors + elif multioutput == "uniform_average": + # pass None as weights to np.average: uniform mean + multioutput = None + + return np.average(output_errors, weights=multioutput) + + +@validate_params( + { + "y_true": ["array-like"], + "y_pred": ["array-like"], + "sample_weight": ["array-like", None], + "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"], + "squared": [Hidden(StrOptions({"deprecated"})), "boolean"], }, prefer_skip_nested_validation=True, ) def mean_squared_log_error( - y_true, y_pred, *, sample_weight=None, multioutput="uniform_average", squared=True + y_true, + y_pred, + *, + sample_weight=None, + multioutput="uniform_average", + squared="deprecated", ): """Mean squared logarithmic error regression loss. @@ -530,10 +629,16 @@ def mean_squared_log_error( 'uniform_average' : Errors of all outputs are averaged with uniform weight. + squared : bool, default=True If True returns MSLE (mean squared log error) value. If False returns RMSLE (root mean squared log error) value. + .. deprecated:: 1.4 + `squared` is deprecated in 1.4 and will be removed in 1.6. + Use :func:`~sklearn.metrics.root_mean_squared_log_error` + instead to calculate the root mean squared logarithmic error. + Returns ------- loss : float or ndarray of floats @@ -547,8 +652,6 @@ def mean_squared_log_error( >>> y_pred = [2.5, 5, 4, 8] >>> mean_squared_log_error(y_true, y_pred) 0.039... - >>> mean_squared_log_error(y_true, y_pred, squared=False) - 0.199... >>> y_true = [[0.5, 1], [1, 2], [7, 6]] >>> y_pred = [[0.5, 2], [1, 2.5], [8, 8]] >>> mean_squared_log_error(y_true, y_pred) @@ -558,6 +661,22 @@ def mean_squared_log_error( >>> mean_squared_log_error(y_true, y_pred, multioutput=[0.3, 0.7]) 0.060... """ + # TODO(1.6): remove + if squared != "deprecated": + warnings.warn( + ( + "'squared' is deprecated in version 1.4 and " + "will be removed in 1.6. To calculate the " + "root mean squared logarithmic error, use the function" + "'root_mean_squared_log_error'." + ), + FutureWarning, + ) + if not squared: + return root_mean_squared_log_error( + y_true, y_pred, sample_weight=sample_weight, multioutput=multioutput + ) + y_type, y_true, y_pred, multioutput = _check_reg_targets( y_true, y_pred, multioutput ) @@ -574,7 +693,79 @@ def mean_squared_log_error( np.log1p(y_pred), sample_weight=sample_weight, multioutput=multioutput, - squared=squared, + ) + + +@validate_params( + { + "y_true": ["array-like"], + "y_pred": ["array-like"], + "sample_weight": ["array-like", None], + "multioutput": [StrOptions({"raw_values", "uniform_average"}), "array-like"], + }, + prefer_skip_nested_validation=True, +) +def root_mean_squared_log_error( + y_true, y_pred, *, sample_weight=None, multioutput="uniform_average" +): + """Root mean squared logarithmic error regression loss. + + Read more in the :ref:`User Guide `. + + .. versionadded:: 1.4 + + Parameters + ---------- + y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + + y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + multioutput : {'raw_values', 'uniform_average'} or array-like of shape \ + (n_outputs,), default='uniform_average' + + Defines aggregating of multiple output values. + Array-like value defines weights used to average errors. + + 'raw_values' : + Returns a full set of errors when the input is of multioutput + format. + + 'uniform_average' : + Errors of all outputs are averaged with uniform weight. + + Returns + ------- + loss : float or ndarray of floats + A non-negative floating point value (the best value is 0.0), or an + array of floating point values, one for each individual target. + + Examples + -------- + >>> from sklearn.metrics import root_mean_squared_log_error + >>> y_true = [3, 5, 2.5, 7] + >>> y_pred = [2.5, 5, 4, 8] + >>> root_mean_squared_log_error(y_true, y_pred) + 0.199... + """ + _, y_true, y_pred, multioutput = _check_reg_targets(y_true, y_pred, multioutput) + check_consistent_length(y_true, y_pred, sample_weight) + + if (y_true < 0).any() or (y_pred < 0).any(): + raise ValueError( + "Root Mean Squared Logarithmic Error cannot be used when " + "targets contain negative values." + ) + + return root_mean_squared_error( + np.log1p(y_true), + np.log1p(y_pred), + sample_weight=sample_weight, + multioutput=multioutput, ) diff --git a/sklearn/metrics/_scorer.py b/sklearn/metrics/_scorer.py index 302831366aa..50880080dd3 100644 --- a/sklearn/metrics/_scorer.py +++ b/sklearn/metrics/_scorer.py @@ -64,6 +64,8 @@ from . import ( r2_score, recall_score, roc_auc_score, + root_mean_squared_error, + root_mean_squared_log_error, top_k_accuracy_score, ) from .cluster import ( @@ -762,7 +764,10 @@ neg_median_absolute_error_scorer = make_scorer( median_absolute_error, greater_is_better=False ) neg_root_mean_squared_error_scorer = make_scorer( - mean_squared_error, greater_is_better=False, squared=False + root_mean_squared_error, greater_is_better=False +) +neg_root_mean_squared_log_error_scorer = make_scorer( + root_mean_squared_log_error, greater_is_better=False ) neg_mean_poisson_deviance_scorer = make_scorer( mean_poisson_deviance, greater_is_better=False @@ -837,10 +842,11 @@ _SCORERS = dict( matthews_corrcoef=matthews_corrcoef_scorer, neg_median_absolute_error=neg_median_absolute_error_scorer, neg_mean_absolute_error=neg_mean_absolute_error_scorer, - neg_mean_absolute_percentage_error=neg_mean_absolute_percentage_error_scorer, # noqa + neg_mean_absolute_percentage_error=neg_mean_absolute_percentage_error_scorer, neg_mean_squared_error=neg_mean_squared_error_scorer, neg_mean_squared_log_error=neg_mean_squared_log_error_scorer, neg_root_mean_squared_error=neg_root_mean_squared_error_scorer, + neg_root_mean_squared_log_error=neg_root_mean_squared_log_error_scorer, neg_mean_poisson_deviance=neg_mean_poisson_deviance_scorer, neg_mean_gamma_deviance=neg_mean_gamma_deviance_scorer, accuracy=accuracy_scorer, diff --git a/sklearn/metrics/tests/test_regression.py b/sklearn/metrics/tests/test_regression.py index f0486d1e942..29afac5cbc8 100644 --- a/sklearn/metrics/tests/test_regression.py +++ b/sklearn/metrics/tests/test_regression.py @@ -23,6 +23,8 @@ from sklearn.metrics import ( mean_tweedie_deviance, median_absolute_error, r2_score, + root_mean_squared_error, + root_mean_squared_log_error, ) from sklearn.metrics._regression import _check_reg_targets from sklearn.model_selection import GridSearchCV @@ -123,12 +125,12 @@ def test_regression_metrics(n_samples=50): ) -def test_mean_squared_error_multioutput_raw_value_squared(): +def test_root_mean_squared_error_multioutput_raw_value(): # non-regression test for # https://github.com/scikit-learn/scikit-learn/pull/16323 - mse1 = mean_squared_error([[1]], [[10]], multioutput="raw_values", squared=True) - mse2 = mean_squared_error([[1]], [[10]], multioutput="raw_values", squared=False) - assert np.sqrt(mse1) == pytest.approx(mse2) + mse = mean_squared_error([[1]], [[10]], multioutput="raw_values") + rmse = root_mean_squared_error([[1]], [[10]], multioutput="raw_values") + assert np.sqrt(mse) == pytest.approx(rmse) def test_multioutput_regression(): @@ -138,12 +140,15 @@ def test_multioutput_regression(): error = mean_squared_error(y_true, y_pred) assert_almost_equal(error, (1.0 / 3 + 2.0 / 3 + 2.0 / 3) / 4.0) - error = mean_squared_error(y_true, y_pred, squared=False) + error = root_mean_squared_error(y_true, y_pred) assert_almost_equal(error, 0.454, decimal=2) error = mean_squared_log_error(y_true, y_pred) assert_almost_equal(error, 0.200, decimal=2) + error = root_mean_squared_log_error(y_true, y_pred) + assert_almost_equal(error, 0.315, decimal=2) + # mean_absolute_error and mean_squared_error are equal because # it is a binary problem. error = mean_absolute_error(y_true, y_pred) @@ -219,7 +224,7 @@ def test_regression_metrics_at_limits(): # Single-sample case # Note: for r2 and d2_tweedie see also test_regression_single_sample assert_almost_equal(mean_squared_error([0.0], [0.0]), 0.0) - assert_almost_equal(mean_squared_error([0.0], [0.0], squared=False), 0.0) + assert_almost_equal(root_mean_squared_error([0.0], [0.0]), 0.0) assert_almost_equal(mean_squared_log_error([0.0], [0.0]), 0.0) assert_almost_equal(mean_absolute_error([0.0], [0.0]), 0.0) assert_almost_equal(mean_pinball_loss([0.0], [0.0]), 0.0) @@ -257,6 +262,12 @@ def test_regression_metrics_at_limits(): ) with pytest.raises(ValueError, match=msg): mean_squared_log_error([1.0, -2.0, 3.0], [1.0, 2.0, 3.0]) + msg = ( + "Root Mean Squared Logarithmic Error cannot be used when targets " + "contain negative values." + ) + with pytest.raises(ValueError, match=msg): + root_mean_squared_log_error([1.0, -2.0, 3.0], [1.0, 2.0, 3.0]) # Tweedie deviance error power = -1.2 @@ -438,7 +449,7 @@ def test_regression_custom_weights(): y_pred = [[1, 1], [2, -1], [5, 4], [5, 6.5]] msew = mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6]) - rmsew = mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6], squared=False) + rmsew = root_mean_squared_error(y_true, y_pred, multioutput=[0.4, 0.6]) maew = mean_absolute_error(y_true, y_pred, multioutput=[0.4, 0.6]) mapew = mean_absolute_percentage_error(y_true, y_pred, multioutput=[0.4, 0.6]) rw = r2_score(y_true, y_pred, multioutput=[0.4, 0.6]) @@ -611,3 +622,50 @@ def test_pinball_loss_relation_with_mae(): mean_absolute_error(y_true, y_pred) == mean_pinball_loss(y_true, y_pred, alpha=0.5) * 2 ) + + +# TODO(1.6): remove this test +@pytest.mark.parametrize("metric", [mean_squared_error, mean_squared_log_error]) +def test_mean_squared_deprecation_squared(metric): + """Check the deprecation warning of the squared parameter""" + depr_msg = "'squared' is deprecated in version 1.4 and will be removed in 1.6." + y_true, y_pred = np.arange(10), np.arange(1, 11) + with pytest.warns(FutureWarning, match=depr_msg): + metric(y_true, y_pred, squared=False) + + +# TODO(1.6): remove this test +@pytest.mark.filterwarnings("ignore:'squared' is deprecated") +@pytest.mark.parametrize( + "old_func, new_func", + [ + (mean_squared_error, root_mean_squared_error), + (mean_squared_log_error, root_mean_squared_log_error), + ], +) +def test_rmse_rmsle_parameter(old_func, new_func): + # Check that the new rmse/rmsle function is equivalent to + # the old mse/msle + squared=False function. + y_true = np.array([[1, 0, 0, 1], [0, 1, 1, 1], [1, 1, 0, 1]]) + y_pred = np.array([[0, 0, 0, 1], [1, 0, 1, 1], [0, 0, 0, 1]]) + y_true = np.array([[0.5, 1], [1, 2], [7, 6]]) + y_pred = np.array([[0.5, 2], [1, 2.5], [8, 8]]) + sw = np.arange(len(y_true)) + + expected = old_func(y_true, y_pred, squared=False) + actual = new_func(y_true, y_pred) + assert_allclose(expected, actual) + + expected = old_func(y_true, y_pred, sample_weight=sw, squared=False) + actual = new_func(y_true, y_pred, sample_weight=sw) + assert_allclose(expected, actual) + + expected = old_func(y_true, y_pred, multioutput="raw_values", squared=False) + actual = new_func(y_true, y_pred, multioutput="raw_values") + assert_allclose(expected, actual) + + expected = old_func( + y_true, y_pred, sample_weight=sw, multioutput="raw_values", squared=False + ) + actual = new_func(y_true, y_pred, sample_weight=sw, multioutput="raw_values") + assert_allclose(expected, actual) diff --git a/sklearn/metrics/tests/test_score_objects.py b/sklearn/metrics/tests/test_score_objects.py index 10d991c477f..0bd1def6a50 100644 --- a/sklearn/metrics/tests/test_score_objects.py +++ b/sklearn/metrics/tests/test_score_objects.py @@ -73,6 +73,7 @@ REGRESSION_SCORERS = [ "neg_mean_squared_log_error", "neg_median_absolute_error", "neg_root_mean_squared_error", + "neg_root_mean_squared_log_error", "mean_absolute_error", "mean_absolute_percentage_error", "mean_squared_error", diff --git a/sklearn/tests/test_public_functions.py b/sklearn/tests/test_public_functions.py index 20ac2e1e161..22e70b39e1c 100644 --- a/sklearn/tests/test_public_functions.py +++ b/sklearn/tests/test_public_functions.py @@ -295,6 +295,8 @@ PARAM_VALIDATION_FUNCTION_LIST = [ "sklearn.metrics.recall_score", "sklearn.metrics.roc_auc_score", "sklearn.metrics.roc_curve", + "sklearn.metrics.root_mean_squared_error", + "sklearn.metrics.root_mean_squared_log_error", "sklearn.metrics.top_k_accuracy_score", "sklearn.metrics.v_measure_score", "sklearn.metrics.zero_one_loss",