Merge branch 'transformer_pipeline'. Fixes #46.

Author: dukebody

README.rst

@@ -190,6 +190,12 @@ Sklearn-pandas' ``cross_val_score`` function provides exactly the same interface
 Changelog
 ---------
 
+1.1.0 (development)
+*******************
+
+* Use custom ``TransformerPipeline`` class to allow transformation steps accepting only a X argument. Fixes #46.
+
+
 1.0.0 (2015-11-28)
 *******************
 

pipeline.py

@@ -0,0 +1,64 @@
+import six
+from sklearn.pipeline import _name_estimators, Pipeline
+from sklearn.utils import tosequence
+
+
+class TransformerPipeline(Pipeline):
+    """
+    Pipeline that expects all steps to be transformers taking a single argument
+    and having fit and transform methods.
+
+    Code is copied from sklearn's Pipeline, leaving out the `y=None` argument.
+    """
+    def __init__(self, steps):
+        names, estimators = zip(*steps)
+        if len(dict(steps)) != len(steps):
+            raise ValueError("Provided step names are not unique: %s" % (names,))
+
+        # shallow copy of steps
+        self.steps = tosequence(steps)
+        estimator = estimators[-1]
+
+        for e in estimators:
+            if (not (hasattr(e, "fit") or hasattr(e, "fit_transform")) or not
+                    hasattr(e, "transform")):
+                raise TypeError("All steps of the chain should "
+                                "be transforms and implement fit and transform"
+                                " '%s' (type %s) doesn't)" % (e, type(e)))
+
+        if not hasattr(estimator, "fit"):
+            raise TypeError("Last step of chain should implement fit "
+                            "'%s' (type %s) doesn't)"
+                            % (estimator, type(estimator)))
+
+    def _pre_transform(self, X, **fit_params):
+        fit_params_steps = dict((step, {}) for step, _ in self.steps)
+        for pname, pval in six.iteritems(fit_params):
+            step, param = pname.split('__', 1)
+            fit_params_steps[step][param] = pval
+        Xt = X
+        for name, transform in self.steps[:-1]:
+            if hasattr(transform, "fit_transform"):
+                Xt = transform.fit_transform(Xt, **fit_params_steps[name])
+            else:
+                Xt = transform.fit(Xt, **fit_params_steps[name]) \
+                              .transform(Xt)
+        return Xt, fit_params_steps[self.steps[-1][0]]
+
+    def fit(self, X, **fit_params):
+        Xt, fit_params = self._pre_transform(X, **fit_params)
+        self.steps[-1][-1].fit(Xt, **fit_params)
+        return self
+
+    def fit_transform(self, X, **fit_params):
+        Xt, fit_params = self._pre_transform(X, **fit_params)
+        if hasattr(self.steps[-1][-1], 'fit_transform'):
+            return self.steps[-1][-1].fit_transform(Xt, **fit_params)
+        else:
+            return self.steps[-1][-1].fit(Xt, **fit_params).transform(Xt)
+
+
+def make_transformer_pipeline(*steps):
+    """Construct a TransformerPipeline from the given estimators.
+    """
+    return TransformerPipeline(_name_estimators(steps))

sklearn_pandas/__init__.py

@@ -1,13 +1,14 @@
 __version__ = '1.0.0'
 
+
+import sys
 import numpy as np
 import pandas as pd
 from scipy import sparse
 from sklearn.base import BaseEstimator, TransformerMixin
-from sklearn.pipeline import make_pipeline
 from sklearn import cross_validation
 from sklearn import grid_search
-import sys
+from pipeline import make_transformer_pipeline
 
 # load in the correct stringtype: str for py3, basestring for py2
 string_types = str if sys.version_info >= (3, 0) else basestring
@@ -68,7 +69,7 @@ def _handle_feature(fea):
 
 def _build_transformer(transformers):
     if isinstance(transformers, list):
-        transformers = make_pipeline(*transformers)
+        transformers = make_transformer_pipeline(*transformers)
     return transformers
 
 

test_pipeline.py

@@ -0,0 +1,27 @@
+import pytest
+from pipeline import TransformerPipeline
+
+
+class NoTransformT(object):
+    """Transformer without transform method.
+    """
+    def fit(self, x):
+        return self
+
+
+class NoFitT(object):
+    """Transformer without fit method.
+    """
+    def transform(self, x):
+        return self
+
+
+def test_all_steps_fit_transform():
+    """
+    All steps must implement fit and transform. Otherwise, raise TypeError.
+    """
+    with pytest.raises(TypeError):
+        TransformerPipeline([('svc', NoTransformT())])
+
+    with pytest.raises(TypeError):
+        TransformerPipeline([('svc', NoFitT())])

tests/test_dataframe_mapper.py

@@ -28,6 +28,17 @@
 )
 
 
+class MockXTransformer(object):
+    """
+    Mock transformer that accepts no y argument.
+    """
+    def fit(self, X):
+        return self
+
+    def transform(self, X):
+        return X
+
+
 class MockTClassifier(object):
     """
     Mock transformer/classifier.
@@ -148,6 +159,18 @@ def test_build_transformers():
         assert pipeline.steps[ix][1] == transformer
 
 
+def test_list_transformers_single_arg(simple_dataframe):
+    """
+    Multiple transformers can be specified in a list even if some of them
+    only accept one X argument instead of two (X, y).
+    """
+    mapper = DataFrameMapper([
+        ('a', [MockXTransformer()])
+    ])
+    # doesn't fail
+    mapper.fit_transform(simple_dataframe)
+
+
 def test_list_transformers():
     """
     Specifying a list of transformers applies them sequentially to the