diff --git a/feature_engine/imputation/__init__.py b/feature_engine/imputation/__init__.py index beb4c41f8..5218ac1cd 100644 --- a/feature_engine/imputation/__init__.py +++ b/feature_engine/imputation/__init__.py @@ -2,20 +2,23 @@ The module imputation includes classes to perform missing data imputation """ -from .arbitrary_number import ArbitraryNumberImputer +from .arbitrary_number import ArbitraryImputer, ArbitraryNumberImputer from .categorical import CategoricalImputer from .drop_missing_data import DropMissingData from .end_tail import EndTailImputer -from .mean_median import MeanMedianImputer -from .missing_indicator import AddMissingIndicator +from .mean_median import MeanImputer, MeanMedianImputer +from .missing_indicator import AddMissingIndicator, MissingIndicator from .random_sample import RandomSampleImputer __all__ = [ + "MeanImputer", "MeanMedianImputer", + "ArbitraryImputer", "ArbitraryNumberImputer", + "MissingIndicator", + "AddMissingIndicator", "CategoricalImputer", "EndTailImputer", - "AddMissingIndicator", "RandomSampleImputer", "DropMissingData", ] diff --git a/feature_engine/imputation/arbitrary_number.py b/feature_engine/imputation/arbitrary_number.py index 69d7c624b..f9e949904 100644 --- a/feature_engine/imputation/arbitrary_number.py +++ b/feature_engine/imputation/arbitrary_number.py @@ -2,7 +2,7 @@ # License: BSD 3 clause from typing import List, Optional, Union - +import warnings import pandas as pd from feature_engine._check_init_parameters.check_input_dictionary import ( @@ -47,9 +47,9 @@ transform=_transform_imputers_docstring, fit_transform=_fit_transform_docstring, ) -class ArbitraryNumberImputer(BaseImputer): +class ArbitraryImputer(BaseImputer): """ - The ArbitraryNumberImputer() replaces missing data by an arbitrary + The ArbitraryImputer() replaces missing data by an arbitrary value determined by the user. It works only with numerical variables. You can impute all variables with the same number by defining @@ -104,12 +104,12 @@ class ArbitraryNumberImputer(BaseImputer): >>> import pandas as pd >>> import numpy as np - >>> from feature_engine.imputation import ArbitraryNumberImputer + >>> from feature_engine.imputation import ArbitraryImputer >>> X = pd.DataFrame(dict( >>> x1 = [np.nan,1,1,0,np.nan], >>> x2 = ["a", np.nan, "b", np.nan, "a"], >>> )) - >>> ani = ArbitraryNumberImputer(arbitrary_number=-999) + >>> ani = ArbitraryImputer(arbitrary_number=-999) >>> ani.fit(X) >>> ani.transform(X) x1 x2 @@ -175,3 +175,35 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self._get_feature_names_in(X) return self + + +# TODO remove in version 2.1.0 + + +class ArbitraryNumberImputer(ArbitraryImputer): + """Deprecated alias for ArbitraryImputer.""" + + def __init__( + self, + arbitrary_number: Union[int, float] = 999, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + imputer_dict: Optional[dict] = None, + ) -> None: + + warnings.warn( + ( + "ArbitraryNumberImputer was deprecated in version 2.0.0 " + "in favour of ArbitraryImputer and will be removed in version 2.1.0. " + "Use ArbitraryImputer instead." + ), + FutureWarning, + stacklevel=2, + ) + + super().__init__( + arbitrary_number=arbitrary_number, + variables=variables, + return_empty=return_empty, + imputer_dict=imputer_dict, + ) diff --git a/feature_engine/imputation/mean_median.py b/feature_engine/imputation/mean_median.py index 997ec2813..0fadc3006 100644 --- a/feature_engine/imputation/mean_median.py +++ b/feature_engine/imputation/mean_median.py @@ -2,8 +2,8 @@ # License: BSD 3 clause from typing import List, Optional, Union - import pandas as pd +import warnings from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, @@ -43,13 +43,13 @@ transform=_transform_imputers_docstring, fit_transform=_fit_transform_docstring, ) -class MeanMedianImputer(BaseImputer): +class MeanImputer(BaseImputer): """ - The MeanMedianImputer() replaces missing data by the mean or median value of the + The MeanImputer() replaces missing data by the mean or median value of the variable. It works only with numerical variables. You can pass a list of variables to impute. Alternatively, the - MeanMedianImputer() will automatically select all variables of type numeric in the + MeanImputer() will automatically select all variables of type numeric in the training set. More details in the :ref:`User Guide `. @@ -87,12 +87,12 @@ class MeanMedianImputer(BaseImputer): >>> import pandas as pd >>> import numpy as np - >>> from feature_engine.imputation import MeanMedianImputer + >>> from feature_engine.imputation import MeanImputer >>> X = pd.DataFrame(dict( >>> x1 = [np.nan,1,1,0,np.nan], >>> x2 = ["a", np.nan, "b", np.nan, "a"], >>> )) - >>> mmi = MeanMedianImputer(imputation_method='median') + >>> mmi = MeanImputer(imputation_method='median') >>> mmi.fit(X) >>> mmi.transform(X) x1 x2 @@ -151,3 +151,31 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self._get_feature_names_in(X) return self + + +# TODO remove in version 2.1.0 + + +class MeanMedianImputer(MeanImputer): + def __init__( + self, + imputation_method: str = "median", + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + ) -> None: + + warnings.warn( + ( + "MeanMedianImputer was deprecated in version 2.0.0 " + "and will be removed in version 2.1.0. " + "Use MeanImputer instead." + ), + FutureWarning, + stacklevel=2, + ) + + super().__init__( + imputation_method=imputation_method, + variables=variables, + return_empty=return_empty, + ) diff --git a/feature_engine/imputation/missing_indicator.py b/feature_engine/imputation/missing_indicator.py index 153572ad2..dba8d3ef6 100644 --- a/feature_engine/imputation/missing_indicator.py +++ b/feature_engine/imputation/missing_indicator.py @@ -2,7 +2,7 @@ # License: BSD 3 clause from typing import List, Optional, Union - +import warnings import pandas as pd from feature_engine._check_init_parameters.check_variables import ( @@ -32,13 +32,13 @@ n_features_in_=_n_features_in_docstring, fit_transform=_fit_transform_docstring, ) -class AddMissingIndicator(BaseImputer): +class MissingIndicator(BaseImputer): """ - The AddMissingIndicator() adds binary variables that indicate if data is + The MissingIndicator() adds binary variables that indicate if data is missing (one indicator per variable). The added variables (missing indicators) are named with the original variable name plus '_na'. - The AddMissingIndicator() works for both numerical and categorical variables. You + The MissingIndicator() works for both numerical and categorical variables. You can pass a list with the variables for which the missing indicators should be added. Alternatively, the imputer will select and add missing indicators to all variables in the training set. @@ -91,12 +91,12 @@ class AddMissingIndicator(BaseImputer): >>> import pandas as pd >>> import numpy as np - >>> from feature_engine.imputation import AddMissingIndicator + >>> from feature_engine.imputation import MissingIndicator >>> X = pd.DataFrame(dict( >>> x1 = [np.nan,1,1,0,np.nan], >>> x2 = ["a", np.nan, "b", np.nan, "a"], >>> )) - >>> ami = AddMissingIndicator() + >>> ami = MissingIndicator() >>> ami.fit(X) >>> ami.transform(X) x1 x2 x1_na x2_na @@ -200,3 +200,36 @@ def __sklearn_tags__(self): tags = super().__sklearn_tags__() tags.input_tags.allow_nan = True return tags + + +# TODO remove in version 2.1.0 + +class AddMissingIndicator(MissingIndicator): + """ + Deprecated alias for MissingIndicator. + + Use MissingIndicator instead. + """ + + def __init__( + self, + missing_only: bool = True, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, + ) -> None: + + warnings.warn( + ( + "AddMissingIndicator was deprecated in version 2.0.0 " + "in favour of MissingIndicator and will be removed in " + "version 2.1.0. Use MissingIndicator instead." + ), + FutureWarning, + stacklevel=2, + ) + + super().__init__( + missing_only=missing_only, + variables=variables, + return_empty=return_empty, + ) diff --git a/tests/test_imputation/test_arbitrary_number_imputer.py b/tests/test_imputation/test_arbitrary_number_imputer.py index dd83dea68..1b6cee0eb 100644 --- a/tests/test_imputation/test_arbitrary_number_imputer.py +++ b/tests/test_imputation/test_arbitrary_number_imputer.py @@ -1,12 +1,19 @@ import pandas as pd import pytest -from feature_engine.imputation import ArbitraryNumberImputer +from feature_engine.imputation import ( + ArbitraryImputer, + ArbitraryNumberImputer, +) -def test_impute_with_99_and_automatically_select_variables(df_na): +@pytest.mark.parametrize( + "imputer_cls", + [ArbitraryImputer, ArbitraryNumberImputer], +) +def test_impute_with_99_and_automatically_select_variables(df_na, imputer_cls): # set up the transformer - imputer = ArbitraryNumberImputer(arbitrary_number=99, variables=None) + imputer = imputer_cls(arbitrary_number=99, variables=None) X_transformed = imputer.fit_transform(df_na) # set up output reference @@ -24,16 +31,18 @@ def test_impute_with_99_and_automatically_select_variables(df_na): assert imputer.imputer_dict_ == {"Age": 99, "Marks": 99} # test transform output - # selected variables should not contain NA - # non selected variables should still contain NA assert X_transformed[["Age", "Marks"]].isnull().sum().sum() == 0 assert X_transformed[["Name", "City"]].isnull().sum().sum() > 0 pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_impute_with_1_and_single_variable_entered_by_user(df_na): +@pytest.mark.parametrize( + "imputer_cls", + [ArbitraryImputer, ArbitraryNumberImputer], +) +def test_impute_with_1_and_single_variable_entered_by_user(df_na, imputer_cls): # set up transformer - imputer = ArbitraryNumberImputer(arbitrary_number=-1, variables=["Age"]) + imputer = imputer_cls(arbitrary_number=-1, variables=["Age"]) X_transformed = imputer.fit_transform(df_na) # set up output reference @@ -54,14 +63,25 @@ def test_impute_with_1_and_single_variable_entered_by_user(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_error_when_arbitrary_number_is_string(): +@pytest.mark.parametrize( + "imputer_cls", + [ArbitraryImputer, ArbitraryNumberImputer], +) +def test_error_when_arbitrary_number_is_string(imputer_cls): with pytest.raises(ValueError): - ArbitraryNumberImputer(arbitrary_number="arbitrary") + imputer_cls(arbitrary_number="arbitrary") -def test_dictionary_of_imputation_values(df_na): +@pytest.mark.parametrize( + "imputer_cls", + [ArbitraryImputer, ArbitraryNumberImputer], +) +def test_dictionary_of_imputation_values(df_na, imputer_cls): # set up transformer - imputer = ArbitraryNumberImputer(imputer_dict={"Age": -42, "Marks": -999}) + imputer = imputer_cls( + imputer_dict={"Age": -42, "Marks": -999} + ) + X_transformed = imputer.fit_transform(df_na) # set up expected output @@ -71,7 +91,10 @@ def test_dictionary_of_imputation_values(df_na): # test fit params assert imputer.n_features_in_ == 6 - assert imputer.imputer_dict_ == {"Age": -42, "Marks": -999} + assert imputer.imputer_dict_ == { + "Age": -42, + "Marks": -999, + } # test transform params assert X_transformed[["Age", "Marks"]].isnull().sum().sum() == 0 @@ -79,6 +102,18 @@ def test_dictionary_of_imputation_values(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def imputer_error_when_dictionary_value_is_string(): +@pytest.mark.parametrize( + "imputer_cls", + [ArbitraryImputer, ArbitraryNumberImputer], +) +def test_error_when_dictionary_value_is_string(imputer_cls): with pytest.raises(ValueError): - ArbitraryNumberImputer(imputer_dict={"Age": "arbitrary_number"}) + imputer_cls(imputer_dict={"Age": "arbitrary_number"}) + + +def test_arbitrary_number_imputer_deprecation_warning(): + with pytest.warns( + FutureWarning, + match="Use ArbitraryImputer instead", + ): + ArbitraryNumberImputer() diff --git a/tests/test_imputation/test_mean_median_imputer.py b/tests/test_imputation/test_mean_median_imputer.py index b065c2190..b798085f6 100644 --- a/tests/test_imputation/test_mean_median_imputer.py +++ b/tests/test_imputation/test_mean_median_imputer.py @@ -1,12 +1,13 @@ import pandas as pd import pytest -from feature_engine.imputation import MeanMedianImputer +from feature_engine.imputation import MeanImputer, MeanMedianImputer -def test_mean_imputation_and_automatically_select_variables(df_na): +@pytest.mark.parametrize("imputer_cls", [MeanImputer, MeanMedianImputer]) +def test_mean_imputation_and_automatically_select_variables(df_na, imputer_cls): # set up transformer - imputer = MeanMedianImputer(imputation_method="mean", variables=None) + imputer = imputer_cls(imputation_method="mean", variables=None) X_transformed = imputer.fit_transform(df_na) # set up reference result @@ -29,17 +30,16 @@ def test_mean_imputation_and_automatically_select_variables(df_na): } assert imputer.n_features_in_ == 6 - # test transform output: - # selected variables should have no NA - # not selected variables should still have NA + # test transform output assert X_transformed[["Age", "Marks"]].isnull().sum().sum() == 0 assert X_transformed[["Name", "City"]].isnull().sum().sum() > 0 pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_median_imputation_when_user_enters_single_variables(df_na): - # set up trasnformer - imputer = MeanMedianImputer(imputation_method="median", variables=["Age"]) +@pytest.mark.parametrize("imputer_cls", [MeanImputer, MeanMedianImputer]) +def test_median_imputation_when_user_enters_single_variables(df_na, imputer_cls): + # set up transformer + imputer = imputer_cls(imputation_method="median", variables=["Age"]) X_transformed = imputer.fit_transform(df_na) # set up reference output @@ -59,6 +59,15 @@ def test_median_imputation_when_user_enters_single_variables(df_na): pd.testing.assert_frame_equal(X_transformed, X_reference) -def test_error_with_wrong_imputation_method(): +@pytest.mark.parametrize("imputer_cls", [MeanImputer, MeanMedianImputer]) +def test_error_with_wrong_imputation_method(imputer_cls): with pytest.raises(ValueError): - MeanMedianImputer(imputation_method="arbitrary") + imputer_cls(imputation_method="arbitrary") + + +def test_mean_median_imputer_deprecation_warning(): + with pytest.warns( + FutureWarning, + match="Use MeanImputer instead", + ): + MeanMedianImputer() diff --git a/tests/test_imputation/test_missing_indicator.py b/tests/test_imputation/test_missing_indicator.py index a7f6e9f7c..386d3b61e 100644 --- a/tests/test_imputation/test_missing_indicator.py +++ b/tests/test_imputation/test_missing_indicator.py @@ -1,53 +1,92 @@ import warnings + import numpy as np import pandas as pd import pytest from sklearn.pipeline import Pipeline -from feature_engine.imputation import AddMissingIndicator +from feature_engine.imputation import MissingIndicator, AddMissingIndicator -def test_detect_variables_with_missing_data_when_variables_is_none(df_na): +@pytest.mark.parametrize( + "indicator_cls", + [MissingIndicator, AddMissingIndicator], +) +def test_detect_variables_with_missing_data_when_variables_is_none( + df_na, indicator_cls +): # test case 1: automatically detect variables with missing data - imputer = AddMissingIndicator(missing_only=True, variables=None) + imputer = indicator_cls(missing_only=True, variables=None) X_transformed = imputer.fit_transform(df_na) # init params assert imputer.missing_only is True assert imputer.variables is None + # fit params assert imputer.variables_ == ["Name", "City", "Studies", "Age", "Marks"] assert imputer.n_features_in_ == 6 + # transform outputs assert X_transformed.shape == (8, 11) assert "Name_na" in X_transformed.columns assert X_transformed["Name_na"].sum() == 2 -def test_add_indicators_to_all_variables_when_variables_is_none(df_na): - imputer = AddMissingIndicator(missing_only=False, variables=None) +@pytest.mark.parametrize( + "indicator_cls", + [MissingIndicator, AddMissingIndicator], +) +def test_add_indicators_to_all_variables_when_variables_is_none( + df_na, indicator_cls +): + imputer = indicator_cls(missing_only=False, variables=None) + X_transformed = imputer.fit_transform(df_na) - assert imputer.variables_ == ["Name", "City", "Studies", "Age", "Marks", "dob"] + + assert imputer.variables_ == [ + "Name", + "City", + "Studies", + "Age", + "Marks", + "dob", + ] assert X_transformed.shape == (8, 12) assert "dob_na" in X_transformed.columns assert X_transformed["dob_na"].sum() == 0 -def test_add_indicators_to_one_variable(df_na): - imputer = AddMissingIndicator(variables="Name") +@pytest.mark.parametrize( + "indicator_cls", + [MissingIndicator, AddMissingIndicator], +) +def test_add_indicators_to_one_variable(df_na, indicator_cls): + imputer = indicator_cls(variables="Name") + X_transformed = imputer.fit_transform(df_na) + assert imputer.variables_ == ["Name"] assert X_transformed.shape == (8, 7) assert "Name_na" in X_transformed.columns assert X_transformed["Name_na"].sum() == 2 -def test_detect_variables_with_missing_data_in_variables_entered_by_user(df_na): - imputer = AddMissingIndicator( - missing_only=True, variables=["City", "Studies", "Age", "dob"] +@pytest.mark.parametrize( + "indicator_cls", + [MissingIndicator, AddMissingIndicator], +) +def test_detect_variables_with_missing_data_in_variables_entered_by_user( + df_na, indicator_cls +): + imputer = indicator_cls( + missing_only=True, + variables=["City", "Studies", "Age", "dob"], ) + X_transformed = imputer.fit_transform(df_na) + assert imputer.variables == ["City", "Studies", "Age", "dob"] assert imputer.variables_ == ["City", "Studies", "Age"] assert X_transformed.shape == (8, 9) @@ -56,15 +95,23 @@ def test_detect_variables_with_missing_data_in_variables_entered_by_user(df_na): assert X_transformed["City_na"].sum() == 2 -def test_error_when_missing_only_not_bool(): +@pytest.mark.parametrize( + "indicator_cls", + [MissingIndicator, AddMissingIndicator], +) +def test_error_when_missing_only_not_bool(indicator_cls): with pytest.raises(ValueError): - AddMissingIndicator(missing_only="missing_only") + indicator_cls(missing_only="missing_only") -def test_get_feature_names_out(df_na): +@pytest.mark.parametrize( + "indicator_cls", + [MissingIndicator, AddMissingIndicator], +) +def test_get_feature_names_out(df_na, indicator_cls): original_features = df_na.columns.to_list() - tr = AddMissingIndicator(missing_only=False) + tr = indicator_cls(missing_only=False) tr.fit(df_na) out = [f + "_na" for f in original_features] @@ -73,7 +120,7 @@ def test_get_feature_names_out(df_na): assert tr.get_feature_names_out(input_features=None) == feat_out assert tr.get_feature_names_out(input_features=original_features) == feat_out - tr = AddMissingIndicator(missing_only=True) + tr = indicator_cls(missing_only=True) tr.fit(df_na) out = [f + "_na" for f in original_features[0:-1]] @@ -89,10 +136,17 @@ def test_get_feature_names_out(df_na): tr.get_feature_names_out(["Name", "hola"]) -def test_get_feature_names_out_from_pipeline(df_na): +@pytest.mark.parametrize( + "indicator_cls", + [MissingIndicator, AddMissingIndicator], +) +def test_get_feature_names_out_from_pipeline(df_na, indicator_cls): original_features = df_na.columns.to_list() - tr = Pipeline([("transformer", AddMissingIndicator(missing_only=False))]) + tr = Pipeline( + [("transformer", indicator_cls(missing_only=False))] + ) + tr.fit(df_na) out = [f + "_na" for f in original_features] @@ -102,8 +156,13 @@ def test_get_feature_names_out_from_pipeline(df_na): assert tr.get_feature_names_out(input_features=original_features) == feat_out -def test_no_performance_warning_with_many_variables(): +@pytest.mark.parametrize( + "indicator_cls", + [MissingIndicator, AddMissingIndicator], +) +def test_no_performance_warning_with_many_variables(indicator_cls): n_cols = 101 + df = pd.DataFrame( np.random.randn(10, n_cols), columns=[f"col_{i}" for i in range(n_cols)], @@ -112,7 +171,7 @@ def test_no_performance_warning_with_many_variables(): # Introduce missing values df.iloc[0, :] = np.nan - ami = AddMissingIndicator(missing_only=False) + ami = indicator_cls(missing_only=False) ami.fit(df) with warnings.catch_warnings(record=True) as captured: @@ -123,3 +182,11 @@ def test_no_performance_warning_with_many_variables(): issubclass(w.category, pd.errors.PerformanceWarning) for w in captured ), "PerformanceWarning was raised during transform" + + +def test_add_missing_indicator_deprecation_warning(): + with pytest.warns( + FutureWarning, + match="Use MissingIndicator instead", + ): + AddMissingIndicator()