Only pin theano-pymc in requirements.txt

.pre-commit-config.yaml

@@ -11,6 +11,7 @@ repos:
     -   id: no-commit-to-branch
         args: [--branch, master]
     -   id: requirements-txt-fixer
+        exclude: ^requirements-dev\.txt$
     -   id: trailing-whitespace
 - repo: https://github.com/nbQA-dev/nbQA
   rev: 0.5.4
@@ -61,6 +62,27 @@ repos:
       language: python
       name: Check no tests are ignored
       pass_filenames: false
+    - id: conda-env-sort
+      additional_dependencies: [pyyaml]
+      entry: python scripts/sort_conda_envs.py
+      files: ^conda-envs/
+      language: python
+      name: Sort dependencies in conda envs
+      types: [yaml]
+    - id: pin-theano-pymc
+      additional_dependencies: [pyyaml]
+      entry: python scripts/pin_theano.py
+      files: ^conda-envs/
+      language: python
+      name: Check theano-pymc versions match the one in requirements.txt
+      types: [yaml]
+    - id: pip-to-conda
+      additional_dependencies: [pyyaml]
+      entry: python scripts/generate_pip_deps_from_conda.py
+      files: ^(conda-envs/environment-dev-py38\.yml|requirements-dev\.txt)$
+      language: python
+      name: Generate pip dependency from conda
+      pass_filenames: false
     - id: no-relative-imports
       name: No relative imports
       entry: from \.[\.\w]* import

conda-envs/environment-dev-py36.yml

@@ -1,31 +1,31 @@
 name: pymc3-dev-py36
 channels:
-  - conda-forge
-  - defaults
+- conda-forge
+- defaults
 dependencies:
-  - python=3.6
-  - arviz>=0.9
-  - theano-pymc==1.0.12
-  - numpy>=1.13
-  - scipy>=0.18
-  - pandas>=0.18
-  - patsy>=0.5
-  - fastprogress>=0.2
-  - h5py>=2.7
-  - typing-extensions>=3.7
-  - python-graphviz
-  - ipython>=7.16
-  - nbsphinx>=0.4
-  - numpydoc>=0.9
-  - pre-commit>=2.8.0
-  - pytest-cov>=2.5
-  - pytest>=3.0
-  - recommonmark>=0.4
-  - sphinx-autobuild>=0.7
-  - sphinx>=1.5
-  - watermark
-  - dataclasses  # python_version < 3.7
-  - contextvars  # python_version < 3.7
-  - mkl-service
-  - dill
-  - libblas=*=*mkl
+- arviz>=0.9
+- contextvars
+- dataclasses
+- dill
+- fastprogress>=0.2
+- h5py>=2.7
+- ipython>=7.16
+- libblas=*=*mkl
+- mkl-service
+- nbsphinx>=0.4
+- numpy>=1.13
+- numpydoc>=0.9
+- pandas>=0.18
+- patsy>=0.5
+- pre-commit>=2.8.0
+- pytest-cov>=2.5
+- pytest>=3.0
+- python-graphviz
+- python=3.6
+- recommonmark>=0.4
+- scipy>=0.18
+- sphinx-autobuild>=0.7
+- sphinx>=1.5
+- theano-pymc=1.0.12
+- typing-extensions>=3.7
+- watermark

conda-envs/environment-dev-py37.yml

@@ -1,29 +1,29 @@
 name: pymc3-dev-py37
 channels:
-  - conda-forge
-  - defaults
+- conda-forge
+- defaults
 dependencies:
-  - python=3.7
-  - arviz>=0.9
-  - theano-pymc==1.0.12
-  - numpy>=1.13
-  - scipy>=0.18
-  - pandas>=0.18
-  - patsy>=0.5
-  - fastprogress>=0.2
-  - h5py>=2.7
-  - typing-extensions>=3.7
-  - python-graphviz
-  - ipython>=7.16
-  - nbsphinx>=0.4
-  - numpydoc>=0.9
-  - pre-commit>=2.8.0
-  - pytest-cov>=2.5
-  - pytest>=3.0
-  - recommonmark>=0.4
-  - sphinx-autobuild>=0.7
-  - sphinx>=1.5
-  - watermark
-  - mkl-service
-  - dill
-  - libblas=*=*mkl
+- arviz>=0.9
+- dill
+- fastprogress>=0.2
+- h5py>=2.7
+- ipython>=7.16
+- libblas=*=*mkl
+- mkl-service
+- nbsphinx>=0.4
+- numpy>=1.13
+- numpydoc>=0.9
+- pandas>=0.18
+- patsy>=0.5
+- pre-commit>=2.8.0
+- pytest-cov>=2.5
+- pytest>=3.0
+- python-graphviz
+- python=3.7
+- recommonmark>=0.4
+- scipy>=0.18
+- sphinx-autobuild>=0.7
+- sphinx>=1.5
+- theano-pymc=1.0.12
+- typing-extensions>=3.7
+- watermark

conda-envs/environment-dev-py38.yml

@@ -1,29 +1,29 @@
 name: pymc3-dev-py38
 channels:
-  - conda-forge
-  - defaults
+- conda-forge
+- defaults
 dependencies:
-  - python=3.8
-  - arviz>=0.9
-  - theano-pymc==1.0.12
-  - numpy>=1.13
-  - scipy>=0.18
-  - pandas>=0.18
-  - patsy>=0.5
-  - fastprogress>=0.2
-  - h5py>=2.7
-  - typing-extensions>=3.7
-  - python-graphviz
-  - ipython>=7.16
-  - nbsphinx>=0.4
-  - numpydoc>=0.9
-  - pre-commit>=2.8.0
-  - pytest-cov>=2.5
-  - pytest>=3.0
-  - recommonmark>=0.4
-  - sphinx-autobuild>=0.7
-  - sphinx>=1.5
-  - watermark
-  - mkl-service
-  - dill
-  - libblas=*=*mkl
+- arviz>=0.9
+- dill
+- fastprogress>=0.2
+- h5py>=2.7
+- ipython>=7.16
+- libblas=*=*mkl
+- mkl-service
+- nbsphinx>=0.4
+- numpy>=1.13
+- numpydoc>=0.9
+- pandas>=0.18
+- patsy>=0.5
+- pre-commit>=2.8.0
+- pytest-cov>=2.5
+- pytest>=3.0
+- python-graphviz
+- python=3.8
+- recommonmark>=0.4
+- scipy>=0.18
+- sphinx-autobuild>=0.7
+- sphinx>=1.5
+- theano-pymc=1.0.12
+- typing-extensions>=3.7
+- watermark

requirements-dev.txt

@@ -1,22 +1,23 @@
-bokeh>=0.12.13
-coverage>=5.1
+# This file is auto-generated from environment-dev-py38.yml, do not modify.
+# See that file for comments about the need/usage of each dependency.
+
+arviz>=0.9
 dill
-graphviz>=0.8.3
-ipython
-nbsphinx>=0.4.2
-nose>=1.3.7
-nose-parameterized==0.6.0
-numpydoc>=0.9.1
-parameterized
+fastprogress>=0.2
+h5py>=2.7
+ipython>=7.16
+nbsphinx>=0.4
+numpy>=1.13
+numpydoc>=0.9
+pandas>=0.18
+patsy>=0.5
 pre-commit>=2.8.0
-pycodestyle>=2.3.1
-pyflakes>=1.5.0
-pylint>=1.7.4
-pytest>=3.0.7
-pytest-cov>=2.5.1
-pytest-xdist
-recommonmark>=0.4.0
-seaborn>=0.8.1
-sphinx>=1.5.5
-sphinx-autobuild==0.7.1
+pytest-cov>=2.5
+pytest>=3.0
+recommonmark>=0.4
+scipy>=0.18
+sphinx-autobuild>=0.7
+sphinx>=1.5
+theano-pymc==1.0.12
+typing-extensions>=3.7
 watermark

scripts/generate_pip_deps_from_conda.py

@@ -0,0 +1,131 @@
+# BSD 3-Clause License
+
+# Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
+# All rights reserved.
+
+# Copyright (c) 2011-2020, Open source contributors.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Check requirements-dev.txt has been generated from conda-envs/environment-dev-py38.yml
+
+This is intended to be used as a pre-commit hook, see `.pre-commit-config.yaml`.
+You can run it manually with `pre-commit run pip-to-conda --all`.
+"""
+import os
+import re
+
+import yaml
+
+EXCLUDE = {"python", "libblas", "mkl-service", "python-graphviz"}
+RENAME = {}
+
+
+def conda_package_to_pip(package):
+    """
+    Convert a conda package to its pip equivalent.
+
+    In most cases they are the same, those are the exceptions:
+    - Packages that should be excluded (in `EXCLUDE`)
+    - Packages that should be renamed (in `RENAME`)
+    - A package requiring a specific version, in conda is defined with a single
+      equal (e.g. ``pandas=1.0``) and in pip with two (e.g. ``pandas==1.0``)
+    """
+    package = re.sub("(?<=[^<>])=", "==", package).strip()
+
+    for compare in ("<=", ">=", "=="):
+        if compare not in package:
+            continue
+        pkg, version = package.split(compare, maxsplit=1)
+        if pkg in EXCLUDE:
+            return
+
+        if pkg in RENAME:
+            return "".join((RENAME[pkg], compare, version))
+
+        break
+
+    if package in EXCLUDE:
+        return
+
+    if package in RENAME:
+        return RENAME[package]
+
+    return package
+
+
+def main(conda_fname, pip_fname):
+    """
+    Generate the pip dependencies file from the conda file, or compare that
+    they are synchronized (``compare=True``).
+
+    Parameters
+    ----------
+    conda_fname : str
+        Path to the conda file with dependencies (e.g. `environment.yml`).
+    pip_fname : str
+        Path to the pip file with dependencies (e.g. `requirements-dev.txt`).
+    compare : bool, default False
+        Whether to generate the pip file (``False``) or to compare if the
+        pip file has been generated with this script and the last version
+        of the conda file (``True``).
+
+    Returns
+    -------
+    bool
+        True if the comparison fails, False otherwise
+    """
+    with open(conda_fname) as conda_fd:
+        deps = yaml.safe_load(conda_fd)["dependencies"]
+
+    pip_deps = []
+    for dep in deps:
+        if isinstance(dep, str):
+            conda_dep = conda_package_to_pip(dep)
+            if conda_dep:
+                pip_deps.append(conda_dep)
+        elif isinstance(dep, dict) and len(dep) == 1 and "pip" in dep:
+            pip_deps += dep["pip"]
+        else:
+            raise ValueError(f"Unexpected dependency {dep}")
+
+    fname = os.path.split(conda_fname)[1]
+    header = (
+        f"# This file is auto-generated from {fname}, do not modify.\n"
+        "# See that file for comments about the need/usage of each dependency.\n\n"
+    )
+    pip_content = header + "\n".join(pip_deps) + "\n"
+
+    with open(pip_fname, "w") as pip_fd:
+        pip_fd.write(pip_content)
+
+
+if __name__ == "__main__":
+    main(
+        os.path.join("conda-envs", "environment-dev-py38.yml"),
+        "requirements-dev.txt",
+    )