delete DataSampler

Author: ferrine

pymc3/data.py

@@ -12,7 +12,6 @@
 __all__ = [
     'get_data',
     'GeneratorAdapter',
-    'DataSampler',
     'Minibatch'
 ]
 
@@ -92,64 +91,6 @@ def __hash__(self):
         return hash(id(self))
 
 
-class DataSampler(object):
-    """
-    Convenient picklable data sampler for minibatch inference.
-
-    This generator can be used for passing to pm.generator
-    creating picklable theano computational grapf
-
-    Parameters
-    ----------
-    data : array like
-    batchsize : sample size over zero axis
-    random_seed : int for numpy random generator
-    dtype : str representing dtype
-
-    Usage
-    -----
-    >>> import pickle
-    >>> from functools import partial
-    >>> np.random.seed(42) # reproducibility
-    >>> pm.set_tt_rng(42)
-    >>> data = np.random.normal(size=(1000,)) + 10
-    >>> minibatches = DataSampler(data, batchsize=50)
-    >>> with pm.Model():
-    ...     sd = pm.Uniform('sd', 0, 10)
-    ...     mu = pm.Normal('mu', sd=10)
-    ...     obs_norm = pm.Normal('obs_norm', mu=mu, sd=sd,
-    ...                          observed=minibatches,
-    ...                          total_size=data.shape[0])
-    ...     adam = partial(pm.adam, learning_rate=.8) # easy problem
-    ...     approx = pm.fit(10000, method='advi', obj_optimizer=adam)
-    >>> new = pickle.loads(pickle.dumps(approx))
-    >>> new #doctest: +ELLIPSIS
-    <pymc3.variational.approximations.MeanField object at 0x...>
-    >>> new.sample(draws=1000)['mu'].mean()
-    10.08339999101371
-    >>> new.sample(draws=1000)['sd'].mean()
-    1.2178044136104513
-    """
-    def __init__(self, data, batchsize=50, random_seed=42, dtype='floatX'):
-        self.dtype = theano.config.floatX if dtype == 'floatX' else dtype
-        self.rng = np.random.RandomState(random_seed)
-        self.data = data
-        self.n = batchsize
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        idx = (self.rng
-               .uniform(size=self.n,
-                        low=0.0,
-                        high=self.data.shape[0] - 1e-16)
-               .astype('int64'))
-        return np.asarray(self.data[idx], self.dtype)
-
-    next = __next__
-
-
 class Minibatch(tt.TensorVariable):
     """Multidimensional minibatch
 

pymc3/tests/conftest.py

@@ -1,7 +1,32 @@
 import theano
+import numpy as np
 import pytest
 
 
+class DataSampler(object):
+    """
+    Not for users
+    """
+    def __init__(self, data, batchsize=50, random_seed=42, dtype='floatX'):
+        self.dtype = theano.config.floatX if dtype == 'floatX' else dtype
+        self.rng = np.random.RandomState(random_seed)
+        self.data = data
+        self.n = batchsize
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        idx = (self.rng
+               .uniform(size=self.n,
+                        low=0.0,
+                        high=self.data.shape[0] - 1e-16)
+               .astype('int64'))
+        return np.asarray(self.data[idx], self.dtype)
+
+    next = __next__
+
+
 @pytest.fixture(scope="session", autouse=True)
 def theano_config():
     config = theano.configparser.change_flags(compute_test_value='raise')
@@ -16,3 +41,10 @@ def strict_float32():
         floatX='float32')
     with config:
         yield
+
+
+@pytest.fixture('session', params=[
+    np.random.uniform(size=(1000, 10))
+])
+def datagen(request):
+    return DataSampler(request.param)

pymc3/tests/test_minibatches.py

@@ -8,7 +8,7 @@
 import theano
 
 import pymc3 as pm
-from pymc3 import floatX, GeneratorAdapter, generator, DataSampler, tt_rng, Normal
+from pymc3 import floatX, GeneratorAdapter, generator, tt_rng, Normal
 from pymc3.tests.helpers import select_by_precision
 from pymc3.theanof import GeneratorOp
 
@@ -29,6 +29,7 @@ def integers_ndim(ndim):
 
 @pytest.mark.usefixtures('strict_float32')
 class TestGenerator(object):
+
     def test_basic(self):
         generator = GeneratorAdapter(integers())
         gop = GeneratorOp(generator)()
@@ -86,24 +87,20 @@ def gen():
         np.testing.assert_equal(np.ones((10, 10)) * 0, f())
         np.testing.assert_equal(np.ones((10, 10)) * 1, f())
 
-    def test_pickling(self):
-        data = np.random.uniform(size=(1000, 10))
-        minibatches = DataSampler(data, batchsize=50)
-        gen = generator(minibatches)
+    def test_pickling(self, datagen):
+        gen = generator(datagen)
         pickle.loads(pickle.dumps(gen))
         bad_gen = generator(integers())
         with pytest.raises(Exception):
             pickle.dumps(bad_gen)
 
-    def test_gen_cloning_with_shape_change(self):
-        data = floatX(np.random.uniform(size=(1000, 10)))
-        minibatches = DataSampler(data, batchsize=50)
-        gen = generator(minibatches)
+    def test_gen_cloning_with_shape_change(self, datagen):
+        gen = generator(datagen)
         gen_r = tt_rng().normal(size=gen.shape).T
         X = gen.dot(gen_r)
         res, _ = theano.scan(lambda x: x.sum(), X, n_steps=X.shape[0])
         assert res.eval().shape == (50,)
-        shared = theano.shared(data)
+        shared = theano.shared(datagen.data.astype(gen.dtype))
         res2 = theano.clone(res, {gen: shared**2})
         assert res2.eval().shape == (1000,)