diff --git a/feature_engine/datasets/titanic.py b/feature_engine/datasets/titanic.py index 5a7c3c3f9..e46214aa7 100644 --- a/feature_engine/datasets/titanic.py +++ b/feature_engine/datasets/titanic.py @@ -2,7 +2,7 @@ import pandas as pd from sklearn.pipeline import Pipeline -from feature_engine.imputation import CategoricalImputer, MeanMedianImputer +from feature_engine.imputation import CategoricalImputer, MeanImputer # TODO: loading the dataset from the internet is not the best, we need to store it @@ -95,7 +95,7 @@ def load_titanic( "categorical_imputer", CategoricalImputer(imputation_method="missing"), ), - ("mean_median_imputer", MeanMedianImputer(imputation_method="mean")), + ("mean_imputer", MeanImputer(imputation_method="mean")), ] ) diff --git a/feature_engine/imputation/__init__.py b/feature_engine/imputation/__init__.py index beb4c41f8..f9c9aa730 100644 --- a/feature_engine/imputation/__init__.py +++ b/feature_engine/imputation/__init__.py @@ -2,19 +2,22 @@ The module imputation includes classes to perform missing data imputation """ -from .arbitrary_number import ArbitraryNumberImputer +from .arbitrary_number import ArbitraryImputer, ArbitraryNumberImputer from .categorical import CategoricalImputer from .drop_missing_data import DropMissingData from .end_tail import EndTailImputer -from .mean_median import MeanMedianImputer -from .missing_indicator import AddMissingIndicator +from .mean_median import MeanImputer, MeanMedianImputer +from .missing_indicator import AddMissingIndicator, MissingIndicator from .random_sample import RandomSampleImputer __all__ = [ + "MeanImputer", "MeanMedianImputer", + "ArbitraryImputer", "ArbitraryNumberImputer", "CategoricalImputer", "EndTailImputer", + "MissingIndicator", "AddMissingIndicator", "RandomSampleImputer", "DropMissingData", diff --git a/feature_engine/imputation/arbitrary_number.py b/feature_engine/imputation/arbitrary_number.py index 69d7c624b..6103c1eb5 100644 --- a/feature_engine/imputation/arbitrary_number.py +++ b/feature_engine/imputation/arbitrary_number.py @@ -1,6 +1,7 @@ # Authors: Soledad Galli # License: BSD 3 clause +import warnings from typing import List, Optional, Union import pandas as pd @@ -47,9 +48,9 @@ transform=_transform_imputers_docstring, fit_transform=_fit_transform_docstring, ) -class ArbitraryNumberImputer(BaseImputer): +class ArbitraryImputer(BaseImputer): """ - The ArbitraryNumberImputer() replaces missing data by an arbitrary + The ArbitraryImputer() replaces missing data by an arbitrary value determined by the user. It works only with numerical variables. You can impute all variables with the same number by defining @@ -104,12 +105,12 @@ class ArbitraryNumberImputer(BaseImputer): >>> import pandas as pd >>> import numpy as np - >>> from feature_engine.imputation import ArbitraryNumberImputer + >>> from feature_engine.imputation import ArbitraryImputer >>> X = pd.DataFrame(dict( >>> x1 = [np.nan,1,1,0,np.nan], >>> x2 = ["a", np.nan, "b", np.nan, "a"], >>> )) - >>> ani = ArbitraryNumberImputer(arbitrary_number=-999) + >>> ani = ArbitraryImputer(arbitrary_number=-999) >>> ani.fit(X) >>> ani.transform(X) x1 x2 @@ -175,3 +176,27 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self._get_feature_names_in(X) return self + + +# TODO: remove in version 2.1.0 +class ArbitraryNumberImputer(ArbitraryImputer): + def __init__( + self, + arbitrary_number: Union[int, float] = 999, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + imputer_dict: Optional[dict] = None, + ) -> None: + warnings.warn( + "ArbitraryNumberImputer was deprecated in favour of ArbitraryImputer in " + "version 2.0.0 and will be removed in version 2.1.0. To silence this " + "warning, use ArbitraryImputer instead.", + FutureWarning, + stacklevel=2, + ) + super().__init__( + arbitrary_number=arbitrary_number, + variables=variables, + return_empty=return_empty, + imputer_dict=imputer_dict, + ) diff --git a/feature_engine/imputation/mean_median.py b/feature_engine/imputation/mean_median.py index 997ec2813..97a6a3994 100644 --- a/feature_engine/imputation/mean_median.py +++ b/feature_engine/imputation/mean_median.py @@ -1,6 +1,7 @@ # Authors: Soledad Galli # License: BSD 3 clause +import warnings from typing import List, Optional, Union import pandas as pd @@ -43,13 +44,13 @@ transform=_transform_imputers_docstring, fit_transform=_fit_transform_docstring, ) -class MeanMedianImputer(BaseImputer): +class MeanImputer(BaseImputer): """ - The MeanMedianImputer() replaces missing data by the mean or median value of the + The MeanImputer() replaces missing data by the mean or median value of the variable. It works only with numerical variables. You can pass a list of variables to impute. Alternatively, the - MeanMedianImputer() will automatically select all variables of type numeric in the + MeanImputer() will automatically select all variables of type numeric in the training set. More details in the :ref:`User Guide `. @@ -87,12 +88,12 @@ class MeanMedianImputer(BaseImputer): >>> import pandas as pd >>> import numpy as np - >>> from feature_engine.imputation import MeanMedianImputer + >>> from feature_engine.imputation import MeanImputer >>> X = pd.DataFrame(dict( >>> x1 = [np.nan,1,1,0,np.nan], >>> x2 = ["a", np.nan, "b", np.nan, "a"], >>> )) - >>> mmi = MeanMedianImputer(imputation_method='median') + >>> mmi = MeanImputer(imputation_method='median') >>> mmi.fit(X) >>> mmi.transform(X) x1 x2 @@ -151,3 +152,25 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self._get_feature_names_in(X) return self + + +# TODO: remove in version 2.1.0 +class MeanMedianImputer(MeanImputer): + def __init__( + self, + imputation_method: str = "median", + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + ) -> None: + warnings.warn( + "MeanMedianImputer was deprecated in favour of MeanImputer in version " + "2.0.0 and will be removed in version 2.1.0. To silence this warning, " + "use MeanImputer instead.", + FutureWarning, + stacklevel=2, + ) + super().__init__( + imputation_method=imputation_method, + variables=variables, + return_empty=return_empty, + ) diff --git a/feature_engine/imputation/missing_indicator.py b/feature_engine/imputation/missing_indicator.py index 153572ad2..209614922 100644 --- a/feature_engine/imputation/missing_indicator.py +++ b/feature_engine/imputation/missing_indicator.py @@ -1,6 +1,7 @@ # Authors: Soledad Galli # License: BSD 3 clause +import warnings from typing import List, Optional, Union import pandas as pd @@ -32,13 +33,13 @@ n_features_in_=_n_features_in_docstring, fit_transform=_fit_transform_docstring, ) -class AddMissingIndicator(BaseImputer): +class MissingIndicator(BaseImputer): """ - The AddMissingIndicator() adds binary variables that indicate if data is + The MissingIndicator() adds binary variables that indicate if data is missing (one indicator per variable). The added variables (missing indicators) are named with the original variable name plus '_na'. - The AddMissingIndicator() works for both numerical and categorical variables. You + The MissingIndicator() works for both numerical and categorical variables. You can pass a list with the variables for which the missing indicators should be added. Alternatively, the imputer will select and add missing indicators to all variables in the training set. @@ -91,12 +92,12 @@ class AddMissingIndicator(BaseImputer): >>> import pandas as pd >>> import numpy as np - >>> from feature_engine.imputation import AddMissingIndicator + >>> from feature_engine.imputation import MissingIndicator >>> X = pd.DataFrame(dict( >>> x1 = [np.nan,1,1,0,np.nan], >>> x2 = ["a", np.nan, "b", np.nan, "a"], >>> )) - >>> ami = AddMissingIndicator() + >>> ami = MissingIndicator() >>> ami.fit(X) >>> ami.transform(X) x1 x2 x1_na x2_na @@ -200,3 +201,25 @@ def __sklearn_tags__(self): tags = super().__sklearn_tags__() tags.input_tags.allow_nan = True return tags + + +# TODO: remove in version 2.1.0 +class AddMissingIndicator(MissingIndicator): + def __init__( + self, + missing_only: bool = True, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + ) -> None: + warnings.warn( + "AddMissingIndicator was deprecated in favour of MissingIndicator in " + "version 2.0.0 and will be removed in version 2.1.0. To silence this " + "warning, use MissingIndicator instead.", + FutureWarning, + stacklevel=2, + ) + super().__init__( + missing_only=missing_only, + variables=variables, + return_empty=return_empty, + ) diff --git a/tests/check_estimators_with_parametrize_tests.py b/tests/check_estimators_with_parametrize_tests.py index 4962ded00..efdbf2151 100644 --- a/tests/check_estimators_with_parametrize_tests.py +++ b/tests/check_estimators_with_parametrize_tests.py @@ -25,12 +25,12 @@ WoEEncoder, ) from feature_engine.imputation import ( - AddMissingIndicator, - ArbitraryNumberImputer, + ArbitraryImputer, CategoricalImputer, DropMissingData, EndTailImputer, - MeanMedianImputer, + MeanImputer, + MissingIndicator, RandomSampleImputer, ) from feature_engine.outliers import ArbitraryOutlierCapper, OutlierTrimmer, Winsoriser @@ -87,11 +87,11 @@ def test_sklearn_compatible_creator(estimator, check): # imputation @parametrize_with_checks( [ - MeanMedianImputer(), - ArbitraryNumberImputer(), + MeanImputer(), + ArbitraryImputer(), CategoricalImputer(fill_value=0, ignore_format=True), EndTailImputer(), - AddMissingIndicator(), + MissingIndicator(), RandomSampleImputer(), DropMissingData(), ] diff --git a/tests/test_imputation/test_arbitrary_number_imputer.py b/tests/test_imputation/test_arbitrary_number_imputer.py index dd83dea68..8ce4c1297 100644 --- a/tests/test_imputation/test_arbitrary_number_imputer.py +++ b/tests/test_imputation/test_arbitrary_number_imputer.py @@ -1,12 +1,40 @@ +import re + import pandas as pd import pytest -from feature_engine.imputation import ArbitraryNumberImputer +from feature_engine.imputation import ArbitraryImputer, ArbitraryNumberImputer + +DEPRECATION_WARNING = ( + "ArbitraryNumberImputer was deprecated in favour of ArbitraryImputer in " + "version 2.0.0 and will be removed in version 2.1.0. To silence this " + "warning, use ArbitraryImputer instead." +) + + +@pytest.fixture( + params=[ArbitraryImputer, ArbitraryNumberImputer], + ids=["ArbitraryImputer", "ArbitraryNumberImputer"], +) +def imputer_class(request): + return request.param + + +def make_imputer(imputer_class, **kwargs): + if imputer_class is ArbitraryNumberImputer: + with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): + return imputer_class(**kwargs) + return imputer_class(**kwargs) + + +def test_arbitrary_number_imputer_raises_future_warning(): + with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): + ArbitraryNumberImputer() -def test_impute_with_99_and_automatically_select_variables(df_na): +def test_impute_with_99_and_automatically_select_variables(df_na, imputer_class): # set up the transformer - imputer = ArbitraryNumberImputer(arbitrary_number=99, variables=None) + imputer = make_imputer(imputer_class, arbitrary_number=99, variables=None) X_transformed = imputer.fit_transform(df_na) # set up output reference @@ -31,9 +59,9 @@ def test_impute_with_99_and_automatically_select_variables(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_impute_with_1_and_single_variable_entered_by_user(df_na): +def test_impute_with_1_and_single_variable_entered_by_user(df_na, imputer_class): # set up transformer - imputer = ArbitraryNumberImputer(arbitrary_number=-1, variables=["Age"]) + imputer = make_imputer(imputer_class, arbitrary_number=-1, variables=["Age"]) X_transformed = imputer.fit_transform(df_na) # set up output reference @@ -54,14 +82,14 @@ def test_impute_with_1_and_single_variable_entered_by_user(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_error_when_arbitrary_number_is_string(): +def test_error_when_arbitrary_number_is_string(imputer_class): with pytest.raises(ValueError): - ArbitraryNumberImputer(arbitrary_number="arbitrary") + make_imputer(imputer_class, arbitrary_number="arbitrary") -def test_dictionary_of_imputation_values(df_na): +def test_dictionary_of_imputation_values(df_na, imputer_class): # set up transformer - imputer = ArbitraryNumberImputer(imputer_dict={"Age": -42, "Marks": -999}) + imputer = make_imputer(imputer_class, imputer_dict={"Age": -42, "Marks": -999}) X_transformed = imputer.fit_transform(df_na) # set up expected output @@ -79,6 +107,6 @@ def test_dictionary_of_imputation_values(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def imputer_error_when_dictionary_value_is_string(): +def test_error_when_dictionary_value_is_string(imputer_class): with pytest.raises(ValueError): - ArbitraryNumberImputer(imputer_dict={"Age": "arbitrary_number"}) + make_imputer(imputer_class, imputer_dict={"Age": "arbitrary_number"}) diff --git a/tests/test_imputation/test_check_estimator_imputers.py b/tests/test_imputation/test_check_estimator_imputers.py index 0091c7bf7..f40efb323 100644 --- a/tests/test_imputation/test_check_estimator_imputers.py +++ b/tests/test_imputation/test_check_estimator_imputers.py @@ -6,22 +6,22 @@ from sklearn.utils.fixes import parse_version from feature_engine.imputation import ( - AddMissingIndicator, - ArbitraryNumberImputer, + ArbitraryImputer, CategoricalImputer, DropMissingData, EndTailImputer, - MeanMedianImputer, + MeanImputer, + MissingIndicator, RandomSampleImputer, ) from tests.estimator_checks.estimator_checks import check_feature_engine_estimator _estimators = [ - MeanMedianImputer(), - ArbitraryNumberImputer(), + MeanImputer(), + ArbitraryImputer(), CategoricalImputer(fill_value=0, ignore_format=True), EndTailImputer(), - AddMissingIndicator(), + MissingIndicator(), RandomSampleImputer(), DropMissingData(), ] @@ -48,7 +48,7 @@ def test_check_estimator_from_sklearn(estimator): def test_check_estimator_from_feature_engine(estimator): if estimator.__class__.__name__ == "CategoricalImputer": estimator.set_params(ignore_format=False) - if estimator.__class__.__name__ in ["DropMissingData", "AddMissingIndicator"]: + if estimator.__class__.__name__ in ["DropMissingData", "MissingIndicator"]: estimator.set_params(missing_only=False) return check_feature_engine_estimator(estimator) @@ -57,7 +57,7 @@ def test_check_estimator_from_feature_engine(estimator): def test_transformers_in_pipeline_with_set_output_pandas(transformer): if transformer.__class__.__name__ == "CategoricalImputer": transformer.set_params(ignore_format=True) - if transformer.__class__.__name__ in ["DropMissingData", "AddMissingIndicator"]: + if transformer.__class__.__name__ in ["DropMissingData", "MissingIndicator"]: transformer.set_params(missing_only=False) X = pd.DataFrame({"feature_1": [1, 2, 3, 4, 5], "feature_2": [6, 7, 8, 9, 10]}) diff --git a/tests/test_imputation/test_mean_median_imputer.py b/tests/test_imputation/test_mean_median_imputer.py index b065c2190..c3603ecf7 100644 --- a/tests/test_imputation/test_mean_median_imputer.py +++ b/tests/test_imputation/test_mean_median_imputer.py @@ -1,12 +1,40 @@ +import re + import pandas as pd import pytest -from feature_engine.imputation import MeanMedianImputer +from feature_engine.imputation import MeanImputer, MeanMedianImputer + +DEPRECATION_WARNING = ( + "MeanMedianImputer was deprecated in favour of MeanImputer in version " + "2.0.0 and will be removed in version 2.1.0. To silence this warning, " + "use MeanImputer instead." +) + + +@pytest.fixture( + params=[MeanImputer, MeanMedianImputer], + ids=["MeanImputer", "MeanMedianImputer"], +) +def imputer_class(request): + return request.param + + +def make_imputer(imputer_class, **kwargs): + if imputer_class is MeanMedianImputer: + with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): + return imputer_class(**kwargs) + return imputer_class(**kwargs) + + +def test_mean_median_imputer_raises_future_warning(): + with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): + MeanMedianImputer() -def test_mean_imputation_and_automatically_select_variables(df_na): +def test_mean_imputation_and_automatically_select_variables(df_na, imputer_class): # set up transformer - imputer = MeanMedianImputer(imputation_method="mean", variables=None) + imputer = make_imputer(imputer_class, imputation_method="mean", variables=None) X_transformed = imputer.fit_transform(df_na) # set up reference result @@ -37,9 +65,9 @@ def test_mean_imputation_and_automatically_select_variables(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_median_imputation_when_user_enters_single_variables(df_na): +def test_median_imputation_when_user_enters_single_variables(df_na, imputer_class): # set up trasnformer - imputer = MeanMedianImputer(imputation_method="median", variables=["Age"]) + imputer = make_imputer(imputer_class, imputation_method="median", variables=["Age"]) X_transformed = imputer.fit_transform(df_na) # set up reference output @@ -59,6 +87,6 @@ def test_median_imputation_when_user_enters_single_variables(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_error_with_wrong_imputation_method(): +def test_error_with_wrong_imputation_method(imputer_class): with pytest.raises(ValueError): - MeanMedianImputer(imputation_method="arbitrary") + make_imputer(imputer_class, imputation_method="arbitrary") diff --git a/tests/test_imputation/test_missing_indicator.py b/tests/test_imputation/test_missing_indicator.py index a7f6e9f7c..68f384ca1 100644 --- a/tests/test_imputation/test_missing_indicator.py +++ b/tests/test_imputation/test_missing_indicator.py @@ -1,16 +1,46 @@ +import re import warnings + import numpy as np import pandas as pd import pytest from sklearn.pipeline import Pipeline -from feature_engine.imputation import AddMissingIndicator +from feature_engine.imputation import AddMissingIndicator, MissingIndicator + +DEPRECATION_WARNING = ( + "AddMissingIndicator was deprecated in favour of MissingIndicator in " + "version 2.0.0 and will be removed in version 2.1.0. To silence this " + "warning, use MissingIndicator instead." +) + + +@pytest.fixture( + params=[MissingIndicator, AddMissingIndicator], + ids=["MissingIndicator", "AddMissingIndicator"], +) +def indicator_class(request): + return request.param + +def make_indicator(indicator_class, **kwargs): + if indicator_class is AddMissingIndicator: + with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): + return indicator_class(**kwargs) + return indicator_class(**kwargs) -def test_detect_variables_with_missing_data_when_variables_is_none(df_na): + +def test_add_missing_indicator_raises_future_warning(): + with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): + AddMissingIndicator() + + +def test_detect_variables_with_missing_data_when_variables_is_none( + df_na, indicator_class +): # test case 1: automatically detect variables with missing data - imputer = AddMissingIndicator(missing_only=True, variables=None) + imputer = make_indicator(indicator_class, missing_only=True, variables=None) X_transformed = imputer.fit_transform(df_na) # init params @@ -25,8 +55,8 @@ def test_detect_variables_with_missing_data_when_variables_is_none(df_na): assert X_transformed["Name_na"].sum() == 2 -def test_add_indicators_to_all_variables_when_variables_is_none(df_na): - imputer = AddMissingIndicator(missing_only=False, variables=None) +def test_add_indicators_to_all_variables_when_variables_is_none(df_na, indicator_class): + imputer = make_indicator(indicator_class, missing_only=False, variables=None) X_transformed = imputer.fit_transform(df_na) assert imputer.variables_ == ["Name", "City", "Studies", "Age", "Marks", "dob"] assert X_transformed.shape == (8, 12) @@ -34,8 +64,8 @@ def test_add_indicators_to_all_variables_when_variables_is_none(df_na): assert X_transformed["dob_na"].sum() == 0 -def test_add_indicators_to_one_variable(df_na): - imputer = AddMissingIndicator(variables="Name") +def test_add_indicators_to_one_variable(df_na, indicator_class): + imputer = make_indicator(indicator_class, variables="Name") X_transformed = imputer.fit_transform(df_na) assert imputer.variables_ == ["Name"] assert X_transformed.shape == (8, 7) @@ -43,9 +73,13 @@ def test_add_indicators_to_one_variable(df_na): assert X_transformed["Name_na"].sum() == 2 -def test_detect_variables_with_missing_data_in_variables_entered_by_user(df_na): - imputer = AddMissingIndicator( - missing_only=True, variables=["City", "Studies", "Age", "dob"] +def test_detect_variables_with_missing_data_in_variables_entered_by_user( + df_na, indicator_class +): + imputer = make_indicator( + indicator_class, + missing_only=True, + variables=["City", "Studies", "Age", "dob"], ) X_transformed = imputer.fit_transform(df_na) assert imputer.variables == ["City", "Studies", "Age", "dob"] @@ -56,15 +90,15 @@ def test_detect_variables_with_missing_data_in_variables_entered_by_user(df_na): assert X_transformed["City_na"].sum() == 2 -def test_error_when_missing_only_not_bool(): +def test_error_when_missing_only_not_bool(indicator_class): with pytest.raises(ValueError): - AddMissingIndicator(missing_only="missing_only") + make_indicator(indicator_class, missing_only="missing_only") -def test_get_feature_names_out(df_na): +def test_get_feature_names_out(df_na, indicator_class): original_features = df_na.columns.to_list() - tr = AddMissingIndicator(missing_only=False) + tr = make_indicator(indicator_class, missing_only=False) tr.fit(df_na) out = [f + "_na" for f in original_features] @@ -73,7 +107,7 @@ def test_get_feature_names_out(df_na): assert tr.get_feature_names_out(input_features=None) == feat_out assert tr.get_feature_names_out(input_features=original_features) == feat_out - tr = AddMissingIndicator(missing_only=True) + tr = make_indicator(indicator_class, missing_only=True) tr.fit(df_na) out = [f + "_na" for f in original_features[0:-1]] @@ -89,10 +123,12 @@ def test_get_feature_names_out(df_na): tr.get_feature_names_out(["Name", "hola"]) -def test_get_feature_names_out_from_pipeline(df_na): +def test_get_feature_names_out_from_pipeline(df_na, indicator_class): original_features = df_na.columns.to_list() - tr = Pipeline([("transformer", AddMissingIndicator(missing_only=False))]) + tr = Pipeline( + [("transformer", make_indicator(indicator_class, missing_only=False))] + ) tr.fit(df_na) out = [f + "_na" for f in original_features] @@ -102,7 +138,7 @@ def test_get_feature_names_out_from_pipeline(df_na): assert tr.get_feature_names_out(input_features=original_features) == feat_out -def test_no_performance_warning_with_many_variables(): +def test_no_performance_warning_with_many_variables(indicator_class): n_cols = 101 df = pd.DataFrame( np.random.randn(10, n_cols), @@ -112,7 +148,7 @@ def test_no_performance_warning_with_many_variables(): # Introduce missing values df.iloc[0, :] = np.nan - ami = AddMissingIndicator(missing_only=False) + ami = make_indicator(indicator_class, missing_only=False) ami.fit(df) with warnings.catch_warnings(record=True) as captured: