Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions feature_engine/imputation/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,23 @@
The module imputation includes classes to perform missing data imputation
"""

from .arbitrary_number import ArbitraryNumberImputer
from .arbitrary_number import ArbitraryImputer, ArbitraryNumberImputer
from .categorical import CategoricalImputer
from .drop_missing_data import DropMissingData
from .end_tail import EndTailImputer
from .mean_median import MeanMedianImputer
from .missing_indicator import AddMissingIndicator
from .mean_median import MeanImputer, MeanMedianImputer
from .missing_indicator import AddMissingIndicator, MissingIndicator
from .random_sample import RandomSampleImputer

__all__ = [
"MeanImputer",
"MeanMedianImputer",
"ArbitraryImputer",
"ArbitraryNumberImputer",
"MissingIndicator",
"AddMissingIndicator",
"CategoricalImputer",
"EndTailImputer",
"AddMissingIndicator",
"RandomSampleImputer",
"DropMissingData",
]
42 changes: 37 additions & 5 deletions feature_engine/imputation/arbitrary_number.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# License: BSD 3 clause

from typing import List, Optional, Union

import warnings
import pandas as pd

from feature_engine._check_init_parameters.check_input_dictionary import (
Expand Down Expand Up @@ -47,9 +47,9 @@
transform=_transform_imputers_docstring,
fit_transform=_fit_transform_docstring,
)
class ArbitraryNumberImputer(BaseImputer):
class ArbitraryImputer(BaseImputer):
"""
The ArbitraryNumberImputer() replaces missing data by an arbitrary
The ArbitraryImputer() replaces missing data by an arbitrary
value determined by the user. It works only with numerical variables.

You can impute all variables with the same number by defining
Expand Down Expand Up @@ -104,12 +104,12 @@ class ArbitraryNumberImputer(BaseImputer):

>>> import pandas as pd
>>> import numpy as np
>>> from feature_engine.imputation import ArbitraryNumberImputer
>>> from feature_engine.imputation import ArbitraryImputer
>>> X = pd.DataFrame(dict(
>>> x1 = [np.nan,1,1,0,np.nan],
>>> x2 = ["a", np.nan, "b", np.nan, "a"],
>>> ))
>>> ani = ArbitraryNumberImputer(arbitrary_number=-999)
>>> ani = ArbitraryImputer(arbitrary_number=-999)
>>> ani.fit(X)
>>> ani.transform(X)
x1 x2
Expand Down Expand Up @@ -175,3 +175,35 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
self._get_feature_names_in(X)

return self


# TODO remove in version 2.1.0


class ArbitraryNumberImputer(ArbitraryImputer):
"""Deprecated alias for ArbitraryImputer."""

def __init__(
self,
arbitrary_number: Union[int, float] = 999,
variables: Union[None, int, str, List[Union[str, int]]] = None,
return_empty: bool = False,
imputer_dict: Optional[dict] = None,
) -> None:

warnings.warn(
(
"ArbitraryNumberImputer was deprecated in version 2.0.0 "
"in favour of ArbitraryImputer and will be removed in version 2.1.0. "
"Use ArbitraryImputer instead."
),
FutureWarning,
stacklevel=2,
)

super().__init__(
arbitrary_number=arbitrary_number,
variables=variables,
return_empty=return_empty,
imputer_dict=imputer_dict,
)
40 changes: 34 additions & 6 deletions feature_engine/imputation/mean_median.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
# License: BSD 3 clause

from typing import List, Optional, Union

import pandas as pd
import warnings

from feature_engine._check_init_parameters.check_variables import (
_check_variables_input_value,
Expand Down Expand Up @@ -43,13 +43,13 @@
transform=_transform_imputers_docstring,
fit_transform=_fit_transform_docstring,
)
class MeanMedianImputer(BaseImputer):
class MeanImputer(BaseImputer):
"""
The MeanMedianImputer() replaces missing data by the mean or median value of the
The MeanImputer() replaces missing data by the mean or median value of the
variable. It works only with numerical variables.
You can pass a list of variables to impute. Alternatively, the
MeanMedianImputer() will automatically select all variables of type numeric in the
MeanImputer() will automatically select all variables of type numeric in the
training set.
More details in the :ref:`User Guide <mean_median_imputer>`.
Expand Down Expand Up @@ -87,12 +87,12 @@ class MeanMedianImputer(BaseImputer):
>>> import pandas as pd
>>> import numpy as np
>>> from feature_engine.imputation import MeanMedianImputer
>>> from feature_engine.imputation import MeanImputer
>>> X = pd.DataFrame(dict(
>>> x1 = [np.nan,1,1,0,np.nan],
>>> x2 = ["a", np.nan, "b", np.nan, "a"],
>>> ))
>>> mmi = MeanMedianImputer(imputation_method='median')
>>> mmi = MeanImputer(imputation_method='median')
>>> mmi.fit(X)
>>> mmi.transform(X)
x1 x2
Expand Down Expand Up @@ -151,3 +151,31 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
self._get_feature_names_in(X)

return self


# TODO remove in version 2.1.0


class MeanMedianImputer(MeanImputer):
def __init__(
self,
imputation_method: str = "median",
variables: Union[None, int, str, List[Union[str, int]]] = None,
return_empty: bool = False,
) -> None:

warnings.warn(
(
"MeanMedianImputer was deprecated in version 2.0.0 "
"and will be removed in version 2.1.0. "
"Use MeanImputer instead."
),
FutureWarning,
stacklevel=2,
)

super().__init__(
imputation_method=imputation_method,
variables=variables,
return_empty=return_empty,
)
45 changes: 39 additions & 6 deletions feature_engine/imputation/missing_indicator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# License: BSD 3 clause

from typing import List, Optional, Union

import warnings
import pandas as pd

from feature_engine._check_init_parameters.check_variables import (
Expand Down Expand Up @@ -32,13 +32,13 @@
n_features_in_=_n_features_in_docstring,
fit_transform=_fit_transform_docstring,
)
class AddMissingIndicator(BaseImputer):
class MissingIndicator(BaseImputer):
"""
The AddMissingIndicator() adds binary variables that indicate if data is
The MissingIndicator() adds binary variables that indicate if data is
missing (one indicator per variable). The added variables (missing indicators) are
named with the original variable name plus '_na'.

The AddMissingIndicator() works for both numerical and categorical variables. You
The MissingIndicator() works for both numerical and categorical variables. You
can pass a list with the variables for which the missing indicators should be
added. Alternatively, the imputer will select and add missing indicators to all
variables in the training set.
Expand Down Expand Up @@ -91,12 +91,12 @@ class AddMissingIndicator(BaseImputer):

>>> import pandas as pd
>>> import numpy as np
>>> from feature_engine.imputation import AddMissingIndicator
>>> from feature_engine.imputation import MissingIndicator
>>> X = pd.DataFrame(dict(
>>> x1 = [np.nan,1,1,0,np.nan],
>>> x2 = ["a", np.nan, "b", np.nan, "a"],
>>> ))
>>> ami = AddMissingIndicator()
>>> ami = MissingIndicator()
>>> ami.fit(X)
>>> ami.transform(X)
x1 x2 x1_na x2_na
Expand Down Expand Up @@ -200,3 +200,36 @@ def __sklearn_tags__(self):
tags = super().__sklearn_tags__()
tags.input_tags.allow_nan = True
return tags


# TODO remove in version 2.1.0

class AddMissingIndicator(MissingIndicator):
"""
Deprecated alias for MissingIndicator.

Use MissingIndicator instead.
"""

def __init__(
self,
missing_only: bool = True,
variables: Union[None, int, str, List[Union[str, int]]] = None,
return_empty: bool = False,
) -> None:

warnings.warn(
(
"AddMissingIndicator was deprecated in version 2.0.0 "
"in favour of MissingIndicator and will be removed in "
"version 2.1.0. Use MissingIndicator instead."
),
FutureWarning,
stacklevel=2,
)

super().__init__(
missing_only=missing_only,
variables=variables,
return_empty=return_empty,
)
63 changes: 49 additions & 14 deletions tests/test_imputation/test_arbitrary_number_imputer.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,19 @@
import pandas as pd
import pytest

from feature_engine.imputation import ArbitraryNumberImputer
from feature_engine.imputation import (
ArbitraryImputer,
ArbitraryNumberImputer,
)


def test_impute_with_99_and_automatically_select_variables(df_na):
@pytest.mark.parametrize(
"imputer_cls",
[ArbitraryImputer, ArbitraryNumberImputer],
)
def test_impute_with_99_and_automatically_select_variables(df_na, imputer_cls):
# set up the transformer
imputer = ArbitraryNumberImputer(arbitrary_number=99, variables=None)
imputer = imputer_cls(arbitrary_number=99, variables=None)
X_transformed = imputer.fit_transform(df_na)

# set up output reference
Expand All @@ -24,16 +31,18 @@ def test_impute_with_99_and_automatically_select_variables(df_na):
assert imputer.imputer_dict_ == {"Age": 99, "Marks": 99}

# test transform output
# selected variables should not contain NA
# non selected variables should still contain NA
assert X_transformed[["Age", "Marks"]].isnull().sum().sum() == 0
assert X_transformed[["Name", "City"]].isnull().sum().sum() > 0
pd.testing.assert_frame_equal(X_transformed, X_reference)


def test_impute_with_1_and_single_variable_entered_by_user(df_na):
@pytest.mark.parametrize(
"imputer_cls",
[ArbitraryImputer, ArbitraryNumberImputer],
)
def test_impute_with_1_and_single_variable_entered_by_user(df_na, imputer_cls):
# set up transformer
imputer = ArbitraryNumberImputer(arbitrary_number=-1, variables=["Age"])
imputer = imputer_cls(arbitrary_number=-1, variables=["Age"])
X_transformed = imputer.fit_transform(df_na)

# set up output reference
Expand All @@ -54,14 +63,25 @@ def test_impute_with_1_and_single_variable_entered_by_user(df_na):
pd.testing.assert_frame_equal(X_transformed, X_reference)


def test_error_when_arbitrary_number_is_string():
@pytest.mark.parametrize(
"imputer_cls",
[ArbitraryImputer, ArbitraryNumberImputer],
)
def test_error_when_arbitrary_number_is_string(imputer_cls):
with pytest.raises(ValueError):
ArbitraryNumberImputer(arbitrary_number="arbitrary")
imputer_cls(arbitrary_number="arbitrary")


def test_dictionary_of_imputation_values(df_na):
@pytest.mark.parametrize(
"imputer_cls",
[ArbitraryImputer, ArbitraryNumberImputer],
)
def test_dictionary_of_imputation_values(df_na, imputer_cls):
# set up transformer
imputer = ArbitraryNumberImputer(imputer_dict={"Age": -42, "Marks": -999})
imputer = imputer_cls(
imputer_dict={"Age": -42, "Marks": -999}
)

X_transformed = imputer.fit_transform(df_na)

# set up expected output
Expand All @@ -71,14 +91,29 @@ def test_dictionary_of_imputation_values(df_na):

# test fit params
assert imputer.n_features_in_ == 6
assert imputer.imputer_dict_ == {"Age": -42, "Marks": -999}
assert imputer.imputer_dict_ == {
"Age": -42,
"Marks": -999,
}

# test transform params
assert X_transformed[["Age", "Marks"]].isnull().sum().sum() == 0
assert X_transformed[["Name", "City"]].isnull().sum().sum() > 0
pd.testing.assert_frame_equal(X_transformed, X_reference)


def imputer_error_when_dictionary_value_is_string():
@pytest.mark.parametrize(
"imputer_cls",
[ArbitraryImputer, ArbitraryNumberImputer],
)
def test_error_when_dictionary_value_is_string(imputer_cls):
with pytest.raises(ValueError):
ArbitraryNumberImputer(imputer_dict={"Age": "arbitrary_number"})
imputer_cls(imputer_dict={"Age": "arbitrary_number"})


def test_arbitrary_number_imputer_deprecation_warning():
with pytest.warns(
FutureWarning,
match="Use ArbitraryImputer instead",
):
ArbitraryNumberImputer()
Loading