Merge pull request #183 from DoubleML/s-extend-default-tests

Extend Default Setting Unit Tests and check propensities

Author: SvenKlaassen

.gitignore

@@ -27,3 +27,4 @@ share/python-wheels/
 *.egg
 MANIFEST
 *.idea
+*.vscode

doubleml/_utils.py

@@ -1,4 +1,5 @@
 import numpy as np
+import warnings
 
 from sklearn.model_selection import cross_val_predict
 from sklearn.base import clone
@@ -214,3 +215,11 @@ def _check_finite_predictions(preds, learner, learner_name, smpls):
     if not np.all(np.isfinite(preds[test_indices])):
         raise ValueError(f'Predictions from learner {str(learner)} for {learner_name} are not finite.')
     return
+
+
+def _check_is_propensity(preds, learner, learner_name, smpls, eps=1e-12):
+    test_indices = np.concatenate([test_index for _, test_index in smpls])
+    if any((preds[test_indices] < eps) | (preds[test_indices] > 1 - eps)):
+        warnings.warn(f'Propensity predictions from learner {str(learner)} for'
+                      f' {learner_name} are close to zero or one (eps={eps}).')
+    return

doubleml/double_ml.py

@@ -114,6 +114,7 @@ def __init__(self,
 
         # also initialize bootstrap arrays with the default number of bootstrap replications
         self._n_rep_boot, self._boot_coef, self._boot_t_stat = self._initialize_boot_arrays(n_rep_boot=500)
+        self._boot_method = None
 
         # initialize instance attributes which are later used for iterating
         self._i_rep = None
@@ -181,6 +182,13 @@ def n_rep_boot(self):
         """
         return self._n_rep_boot
 
+    @property
+    def boot_method(self):
+        """
+        The method to construct the bootstrap replications.
+        """
+        return self._boot_method
+
     @property
     def score(self):
         """
@@ -567,6 +575,7 @@ def bootstrap(self, method='normal', n_rep_boot=500):
                 self._boot_coef[self._i_treat, i_start:i_end], self._boot_t_stat[self._i_treat, i_start:i_end] =\
                     self._compute_bootstrap(weights)
 
+        self._boot_method = method
         return self
 
     def confint(self, joint=False, level=0.95):

doubleml/double_ml_iivm.py

@@ -5,7 +5,7 @@
 from .double_ml import DoubleML
 from .double_ml_data import DoubleMLData
 from .double_ml_score_mixins import LinearScoreMixin
-from ._utils import _dml_cv_predict, _get_cond_smpls, _dml_tune, _check_finite_predictions
+from ._utils import _dml_cv_predict, _get_cond_smpls, _dml_tune, _check_finite_predictions, _check_is_propensity
 
 
 class DoubleMLIIVM(LinearScoreMixin, DoubleML):
@@ -251,6 +251,8 @@ def _nuisance_est(self, smpls, n_jobs_cv, return_models=False):
                                  'observed to be binary with values 0 and 1. Make sure that for classifiers '
                                  'probabilities and not labels are predicted.')
 
+            _check_is_propensity(g_hat0['preds'], self._learner['ml_g'], 'ml_g', smpls, eps=1e-12)
+
         g_hat1 = _dml_cv_predict(self._learner['ml_g'], x, y, smpls=smpls_z1, n_jobs=n_jobs_cv,
                                  est_params=self._get_params('ml_g1'), method=self._predict_method['ml_g'],
                                  return_models=return_models)
@@ -265,11 +267,14 @@ def _nuisance_est(self, smpls, n_jobs_cv, return_models=False):
                                  'observed to be binary with values 0 and 1. Make sure that for classifiers '
                                  'probabilities and not labels are predicted.')
 
+            _check_is_propensity(g_hat1['preds'], self._learner['ml_g'], 'ml_g', smpls, eps=1e-12)
+
         # nuisance m
         m_hat = _dml_cv_predict(self._learner['ml_m'], x, z, smpls=smpls, n_jobs=n_jobs_cv,
                                 est_params=self._get_params('ml_m'), method=self._predict_method['ml_m'],
                                 return_models=return_models)
         _check_finite_predictions(m_hat['preds'], self._learner['ml_m'], 'ml_m', smpls)
+        _check_is_propensity(m_hat['preds'], self._learner['ml_m'], 'ml_m', smpls, eps=1e-12)
 
         # nuisance r
         if self.subgroups['always_takers']:

doubleml/double_ml_irm.py

@@ -10,7 +10,7 @@
 from .double_ml_data import DoubleMLData
 from .double_ml_score_mixins import LinearScoreMixin
 
-from ._utils import _dml_cv_predict, _get_cond_smpls, _dml_tune, _check_finite_predictions
+from ._utils import _dml_cv_predict, _get_cond_smpls, _dml_tune, _check_finite_predictions, _check_is_propensity
 
 
 class DoubleMLIRM(LinearScoreMixin, DoubleML):
@@ -227,6 +227,7 @@ def _nuisance_est(self, smpls, n_jobs_cv, return_models=False):
                                 est_params=self._get_params('ml_m'), method=self._predict_method['ml_m'],
                                 return_models=return_models)
         _check_finite_predictions(m_hat['preds'], self._learner['ml_m'], 'ml_m', smpls)
+        _check_is_propensity(m_hat['preds'], self._learner['ml_m'], 'ml_m', smpls, eps=1e-12)
 
         psi_a, psi_b = self._score_elements(y, d,
                                             g_hat0['preds'], g_hat1['preds'], m_hat['preds'],

doubleml/double_ml_plr.py

@@ -9,7 +9,7 @@
 from .double_ml import DoubleML
 from .double_ml_data import DoubleMLData
 from .double_ml_score_mixins import LinearScoreMixin
-from ._utils import _dml_cv_predict, _dml_tune, _check_finite_predictions
+from ._utils import _dml_cv_predict, _dml_tune, _check_finite_predictions, _check_is_propensity
 
 
 # To be removed in version 0.6.0
@@ -219,6 +219,8 @@ def _nuisance_est(self, smpls, n_jobs_cv, return_models=False):
                                 est_params=self._get_params('ml_m'), method=self._predict_method['ml_m'],
                                 return_models=return_models)
         _check_finite_predictions(m_hat['preds'], self._learner['ml_m'], 'ml_m', smpls)
+        if self._check_learner(self._learner['ml_m'], 'ml_m', regressor=True, classifier=True):
+            _check_is_propensity(m_hat['preds'], self._learner['ml_m'], 'ml_m', smpls, eps=1e-12)
 
         if self._dml_data.binary_treats[self._dml_data.d_cols[self._i_treat]]:
             binary_preds = (type_of_target(m_hat['preds']) == 'binary')

doubleml/tests/test_blp.py

@@ -89,4 +89,4 @@ def test_dml_blp_ci_2(dml_blp_fixture):
 
 def test_dml_blp_return_types(dml_blp_fixture):
     assert isinstance(dml_blp_fixture['blp_model'].__str__(), str)
-    assert isinstance(dml_blp_fixture['blp_model'].summary, pd.DataFrame)
+    assert isinstance(dml_blp_fixture['blp_model'].summary, pd.DataFrame)

doubleml/tests/test_doubleml_model_defaults.py

@@ -17,13 +17,37 @@
 dml_irm = DoubleMLIRM(dml_data_irm, Lasso(), LogisticRegression())
 dml_iivm = DoubleMLIIVM(dml_data_iivm, Lasso(), LogisticRegression(), LogisticRegression())
 
+dml_plr.fit()
+dml_pliv.fit()
+dml_irm.fit()
+dml_iivm.fit()
+
+dml_plr.bootstrap()
+dml_pliv.bootstrap()
+dml_irm.bootstrap()
+dml_iivm.bootstrap()
+
 
 def _assert_resampling_default_settings(dml_obj):
     assert dml_obj.n_folds == 5
     assert dml_obj.n_rep == 1
     assert dml_obj.draw_sample_splitting
     assert dml_obj.apply_cross_fitting
 
+    # fit method
+    assert dml_obj.predictions is None
+    assert dml_obj.models is None
+
+    # bootstrap method
+    assert dml_obj.boot_method == 'normal'
+    assert dml_obj.n_rep_boot == 500
+
+    # confint method
+    assert dml_obj.confint().equals(dml_obj.confint(joint=False, level=0.95))
+
+    # p_adjust method
+    assert dml_obj.p_adjust().equals(dml_obj.p_adjust(method='romano-wolf'))
+
 
 @pytest.mark.ci
 def test_plr_defaults():