add evaluate_learner()

Author: SvenKlaassen

doubleml/double_ml.py

@@ -3,6 +3,7 @@
 import warnings
 
 from sklearn.base import is_regressor, is_classifier
+from sklearn.metrics import mean_squared_error
 
 from scipy.stats import norm
 
@@ -1038,6 +1039,71 @@ def _store_models(self, models):
         for learner in self.params_names:
             self._models[learner][self._dml_data.d_cols[self._i_treat]][self._i_rep] = models[learner]
 
+    def evaluate_learners(self, learners=None, metric=mean_squared_error):
+        """
+       Evaluate fitted learners for DoubleML models on crossvalidated predicitons.
+
+        Parameters
+        ----------
+        learners : list
+            A list of strings which correspond to the nuisance functions of the model.
+
+        metric : callable
+            A callable function with inputs ``y_pred`` and ``y_true``.
+            Default is the euclidean distance.
+
+        Returns
+        -------
+        dist : dict
+            A dictionary containing the evaluated metric for each learner.
+
+        Examples
+        --------
+        >>> import numpy as np
+        >>> import doubleml as dml
+        >>> from sklearn.metrics import mean_absolute_error
+        >>> from doubleml.datasets import make_irm_data
+        >>> from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
+        >>> np.random.seed(3141)
+        >>> ml_g = RandomForestRegressor(n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2)
+        >>> ml_m = RandomForestClassifier(n_estimators=100, max_features=20, max_depth=5, min_samples_leaf=2)
+        >>> data = make_irm_data(theta=0.5, n_obs=500, dim_x=20, return_type='DataFrame')
+        >>> obj_dml_data = dml.DoubleMLData(data, 'y', 'd')
+        >>> dml_irm_obj = dml.DoubleMLIRM(obj_dml_data, ml_g, ml_m)
+        >>> dml_irm_obj.fit()
+        >>> dml_irm_obj.evaluate_learners(metric=mean_absolute_error)
+        {'ml_g0': array([[1.13318973]]),
+         'ml_g1': array([[0.91659939]]),
+         'ml_m': array([[0.36350912]])}
+        """
+        # if no learners are provided try to evaluate all learners
+        if learners is None:
+            learners = self.params_names
+
+        # check metric
+        if not callable(metric):
+            raise TypeError('metric should be either a callable. '
+                            '%r was passed.' % metric)
+
+        if all(learner in self.params_names for learner in learners):
+            if self.nuisance_targets is None:
+                raise ValueError('Apply fit() before evaluate_learners().')
+            else:
+                dist = {learner: np.full((self.n_rep, self._dml_data.n_coefs), np.nan)
+                        for learner in learners}
+            for learner in learners:
+                for rep in range(self.n_rep):
+                    for coef_idx in range(self._dml_data.n_coefs):
+                        res = metric(y_pred=self.predictions[learner][:, rep, coef_idx].reshape(1, -1),
+                                     y_true=self.nuisance_targets[learner][:, rep, coef_idx].reshape(1, -1))
+                        if not np.isfinite(res):
+                            raise ValueError(f'Evaluation from learner {str(learner)} is not finite.')
+                        dist[learner][rep, coef_idx] = res
+            return dist
+        else:
+            raise ValueError(f'The learners have to be a subset of {str(self.params_names)}. '
+                             f'Learners {str(learners)} provided.')
+
     def draw_sample_splitting(self):
         """
         Draw sample splitting for DoubleML models.

doubleml/tests/test_doubleml_evaluate_learner.py

@@ -0,0 +1,72 @@
+import pytest
+import numpy as np
+import doubleml as dml
+from sklearn.metrics import mean_absolute_error, mean_squared_error
+from doubleml.datasets import make_irm_data
+from sklearn.base import clone
+
+from sklearn.linear_model import LogisticRegression, LinearRegression
+from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
+
+
+np.random.seed(3141)
+data = make_irm_data(theta=0.5, n_obs=200, dim_x=5, return_type='DataFrame')
+obj_dml_data = dml.DoubleMLData(data, 'y', 'd')
+
+
+@pytest.fixture(scope='module',
+                params=[[LinearRegression(),
+                         LogisticRegression(solver='lbfgs', max_iter=250)],
+                        [RandomForestRegressor(max_depth=2, n_estimators=10),
+                         RandomForestClassifier(max_depth=2, n_estimators=10)]])
+def learner(request):
+    return request.param
+
+
+@pytest.fixture(scope='module',
+                params=[1, 5])
+def n_rep(request):
+    return request.param
+
+
+@pytest.fixture(scope='module',
+                params=['dml1', 'dml2'])
+def dml_procedure(request):
+    return request.param
+
+
+@pytest.fixture(scope='module',
+                params=[mean_absolute_error, mean_squared_error])
+def metric(request):
+    return request.param
+
+
+@pytest.fixture(scope='module',
+                params=[0.01, 0.05])
+def trimming_threshold(request):
+    return request.param
+
+
+@pytest.fixture(scope='module')
+def dml_irm_eval_learner_fixture(metric, learner, dml_procedure, trimming_threshold, n_rep):
+    # Set machine learning methods for m & g
+    ml_g = clone(learner[0])
+    ml_m = clone(learner[1])
+
+    np.random.seed(3141)
+    dml_irm_obj = dml.DoubleMLIRM(obj_dml_data,
+                                  ml_g, ml_m,
+                                  n_folds=2,
+                                  n_rep=n_rep,
+                                  dml_procedure=dml_procedure,
+                                  trimming_threshold=trimming_threshold)
+    dml_irm_obj.fit()
+    res = dml_irm_obj.evaluate_learners(metric=metric)
+    return res
+
+
+@pytest.mark.ci
+def test_dml_irm_eval_learner(dml_irm_eval_learner_fixture, n_rep):
+    assert dml_irm_eval_learner_fixture['ml_g0'].shape == (n_rep, 1)
+    assert dml_irm_eval_learner_fixture['ml_g1'].shape == (n_rep, 1)
+    assert dml_irm_eval_learner_fixture['ml_m'].shape == (n_rep, 1)

doubleml/tests/test_doubleml_exceptions.py

@@ -791,3 +791,35 @@ def test_doubleml_exception_cate():
     msg = 'Only implemented for one repetition. Number of repetitions is 2.'
     with pytest.raises(NotImplementedError, match=msg):
         dml_irm_obj.cate(basis=2)
+
+
+@pytest.mark.ci
+def test_double_ml_exception_evaluate_learner():
+    dml_irm_obj = DoubleMLIRM(dml_data_irm,
+                              ml_g=Lasso(),
+                              ml_m=LogisticRegression(),
+                              trimming_threshold=0.05,
+                              n_folds=5,
+                              score='ATTE')
+
+    msg = r'Apply fit\(\) before evaluate_learners\(\).'
+    with pytest.raises(ValueError, match=msg):
+        dml_irm_obj.evaluate_learners()
+
+    dml_irm_obj.fit()
+
+    msg = "metric should be either a callable. 'mse' was passed."
+    with pytest.raises(TypeError, match=msg):
+        dml_irm_obj.evaluate_learners(metric="mse")
+
+    msg = (r"The learners have to be a subset of \['ml_g0', 'ml_g1', 'ml_m'\]. "
+           r"Learners \['ml_g', 'ml_m'\] provided.")
+    with pytest.raises(ValueError, match=msg):
+        dml_irm_obj.evaluate_learners(learners=['ml_g', 'ml_m'])
+
+    msg = 'Evaluation from learner ml_g0 is not finite.'
+
+    def eval_fct(y_pred, y_true):
+        return np.nan
+    with pytest.raises(ValueError, match=msg):
+        dml_irm_obj.evaluate_learners(metric=eval_fct)