change install options to use env variables (#323)

Summary: Pull Request resolved: #323

Test Plan:
Imported from OSS
Should build in CI

Reviewed By: malfet, huydhn

Differential Revision: D46771967

Pulled By: PaliC

fbshipit-source-id: 02ddaf4e7a950b45d25adca996a430b2cbbcadaf

Author: PaliC

.github/workflows/build_test_release.yaml

@@ -61,7 +61,7 @@ jobs:
         echo "::endgroup::"
 
         echo "::group::Install"
-        python -m pip install -e . --install-option="--cudatests"
+        BUILD_CUDA_TESTS=1 python -m pip install -e .
         echo "::endgroup::"
 
         echo "::group::Generate examples"

.github/workflows/install_test.yaml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Set Python Version
         run: |
-          echo "python-version=3.${{ matrix.python3-minor-version }}" >> $GITHUB_ENV
+          echo "python-version=3.${{ matrix.python3-minor-version }}" >> "$GITHUB_ENV"
       - name: Setup Python ${{ env.python-version }}
         uses: actions/setup-python@v2
         with:
@@ -66,38 +66,38 @@ jobs:
       - name: Install dependencies
         run: |
           set -eux
-          source ~/venvs/multipy/bin/activate
-          pip install -r dev-requirements.txt
+          source "$HOME/venvs/multipy/bin/activate"
+          python -m pip install -r dev-requirements.txt
           pip3 install --pre torch
           deactivate
 
       - name: Run lightweight pip install within virtualenv
         run: |
           set -eux
-          source ~/venvs/multipy/bin/activate
-          pip install -e . --install-option="--cmakeoff"
+          source "$HOME/venvs/multipy/bin/activate"
+          INSTALL_PYTHON_ONLY=1 python -m pip install -e .
           deactivate
 
       - name: Run pip editable install within virtualenv
         run: |
           set -eux
-          source ~/venvs/multipy/bin/activate
+          source "$HOME/venvs/multipy/bin/activate"
           rm -rf multipy/runtime/build*
           pip install -e .
           deactivate
 
       - name: Run setup.py install within virtualenv
         run: |
           set -eux
-          source ~/venvs/multipy/bin/activate
+          source "$HOME/venvs/multipy/bin/activate"
           python setup.py install
           test -e ~/venvs/multipy/lib/python3.${{ matrix.python3-minor-version }}/site-packages/multipy/runtime/build/libtorch_deploy.a
           deactivate
 
       - name: Install sdist
         run: |
           set -eux
-          source ~/venvs/multipy/bin/activate
+          source "$HOME/venvs/multipy/bin/activate"
           python setup.py sdist
           pip install dist/*.tar.gz --force-reinstall
           test -e ~/venvs/multipy/lib/python3.${{ matrix.python3-minor-version }}/site-packages/multipy/runtime/build/libtorch_deploy.a
@@ -106,18 +106,18 @@ jobs:
       - name: Install bdist_wheel
         run: |
           set -eux
-          source ~/venvs/multipy/bin/activate
+          source "$HOME/venvs/multipy/bin/activate"
           python setup.py bdist_wheel
           pip install dist/*.whl --force-reinstall
           test -e ~/venvs/multipy/lib/python3.${{ matrix.python3-minor-version }}/site-packages/multipy/runtime/build/libtorch_deploy.a
           deactivate
 
-      - name: Install bdist_wheel with cudatests
+      - name: Install bdist_wheel with cuda tests
         run: |
           set -eux
-          source ~/venvs/multipy/bin/activate
+          source "$HOME/venvs/multipy/bin/activate"
           python setup.py bdist_wheel
-          pip install dist/*.whl --force-reinstall --install-option="--cudatests"
+          BUILD_CUDA_TESTS=1 pip install dist/*.whl "--force-reinstall"
           test -e ~/venvs/multipy/lib/python3.${{ matrix.python3-minor-version }}/site-packages/multipy/runtime/build/libtorch_deploy.a
           deactivate
 

Dockerfile

@@ -102,7 +102,7 @@ RUN ls && pwd && rm -rf multipy/runtime/build && \
     source ~/venvs/multipy/bin/activate; \
     fi && \
     if [[ ${BUILD_CUDA_TESTS} -eq 1 ]]; then \
-    python -m pip install -e . --install-option="--cudatests"; \
+    BUILD_CUDA_TESTS=1 python -m pip install -e .; \
     else \
     python -m pip install -e .; \
     fi && \

README.md

@@ -140,7 +140,7 @@ The C++ binaries should be available in `/opt/dist`.
 
 Alternatively, one can install only the python modules without invoking `cmake` as follows:
 ```shell
-pip install  -e . --install-option="--cmakeoff"
+INSTALL_PYTHON_ONLY=1 pip install  -e .
 ```
 
 > **NOTE** As of 10/11/2022 the linking of prebuilt static fPIC versions of python downloaded from `conda-forge` can be problematic on certain systems (for example Centos 8), with linker errors like `libpython_multipy.a: error adding symbols: File format not recognized`. This seems to be an issue with `binutils`, and the steps in https://wiki.gentoo.org/wiki/Project:Toolchain/Binutils_2.32_upgrade_notes/elfutils_0.175:_unable_to_initialize_decompress_status_for_section_.debug_info can help. Alternatively, the user can go with the `virtualenv`/`pyenv` flow above.
@@ -164,7 +164,7 @@ cd multipy
 git submodule sync && git submodule update --init --recursive
 
 # install python parts of `torch::deploy` in multipy/multipy/utils
-pip install -e . --install-option="--cmakeoff"
+INSTALL_PYTHON_ONLY=1 pip install -e .
 
 cd multipy/runtime
 

docs/source/setup.rst

@@ -132,7 +132,7 @@ Alternatively, one can install only the python modules without invoking
 .. code:: shell
 
    # from base multipy directory
-   pip install  -e . --install-option="--cmakeoff"
+   INSTALL_PYTHON_ONLY=1 pip install  -e .
 
 .. warning::
    As of 10/11/2022 the linking of prebuilt static ``-fPIC``
@@ -164,7 +164,7 @@ Building
 
    cd multipy
    # install python parts of `torch::deploy` in multipy/multipy/utils
-   pip install -e . --install-option="--cmakeoff"
+   INSTALL_PYTHON_ONLY=1 pip install -e .
 
    cd multipy/runtime
 

multipy/runtime/interpreter/interpreter_impl.cpp

@@ -21,6 +21,7 @@
 #include <torch/csrc/DynamicTypes.h>
 #include <torch/csrc/autograd/generated/variable_factories.h>
 #include <torch/csrc/jit/frontend/tracer.h>
+#include <torch/csrc/utils/pybind.h>
 #include <cassert>
 #include <cstdio>
 #include <iostream>

setup.py

@@ -5,6 +5,16 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+
+# Environment variables you are probably interested in:
+#
+# BUILD_CUDA_TESTS=0
+#     Build cuda tests
+#
+# INSTALL_PYTHON_ONLY=0
+#     install only the python parts of multipy
+#
+
 import os
 import os.path
 import re
@@ -16,8 +26,6 @@
 
 from setuptools import Extension, find_packages, setup
 from setuptools.command.build_ext import build_ext
-from setuptools.command.develop import develop
-from setuptools.command.install import install
 
 
 class MultipyRuntimeExtension(Extension):
@@ -30,34 +38,9 @@ def get_cmake_version():
     return output.splitlines()[0].split()[2]
 
 
-class MultipyRuntimeCmake(object):
-    user_options = [
-        ("cmakeoff", None, None),
-        ("cudatests", None, None),
-    ]
-
-
-class MultipyRuntimeDevelop(MultipyRuntimeCmake, develop):
-    user_options = develop.user_options + MultipyRuntimeCmake.user_options
-
-    def initialize_options(self):
-        develop.initialize_options(self)
-        self.cmakeoff = None
-
-        self.cudatests = None
-
-    def finalize_options(self):
-        develop.finalize_options(self)
-        if self.cmakeoff is not None:
-            self.distribution.get_command_obj("build_ext").cmake_off = True
-        if self.cudatests is not None:
-            self.distribution.get_command_obj("build_ext").cuda_tests_flag = "ON"
-
-
-class MultipyRuntimeBuild(MultipyRuntimeCmake, build_ext):
-    user_options = build_ext.user_options + MultipyRuntimeCmake.user_options
-    cmake_off = False
-    cuda_tests_flag = "OFF"
+class MultipyRuntimeBuild(build_ext):
+    cmake_off = os.environ.get("INSTALL_PYTHON_ONLY", 0)
+    cuda_tests_flag = os.environ.get("BUILD_CUDA_TESTS", 0)
 
     def run(self):
         if self.cmake_off:
@@ -145,44 +128,6 @@ def run(self):
         super().run()
 
 
-class MultipyRuntimeInstall(MultipyRuntimeCmake, install):
-    user_options = install.user_options + MultipyRuntimeCmake.user_options
-
-    def initialize_options(self):
-        install.initialize_options(self)
-        self.cmakeoff = None
-        self.cudatests = None
-
-    def finalize_options(self):
-        install.finalize_options(self)
-        if self.cmakeoff is not None:
-            self.distribution.get_command_obj("build_ext").cmake_off = True
-        if self.cudatests is not None:
-            self.distribution.get_command_obj("build_ext").cuda_tests_flag = "ON"
-
-    def run(self):
-        # Setuptools/setup.py on docker image has some interesting behavior, in that the
-        # optional "--cmakeoff" flag gets applied to dependencies specified in
-        # requirements.txt as well (installed using "install-requires" argument of setup()).
-        # Since we obviously don't want things like "pip install numpy --install-option=--cmakeoff",
-        # we install these deps directly in this overridden install command without
-        # spurious options, instead of using "install-requires".
-        base_dir = os.path.abspath(os.path.dirname(__file__))
-        try:
-            reqs_filename = "requirements.txt"
-            subprocess.run(
-                [f"pip install -r {reqs_filename}"],
-                cwd=base_dir,
-                shell=True,
-                check=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-            )
-        except subprocess.CalledProcessError as e:
-            raise RuntimeError(e.output.decode("utf-8")) from None
-        install.run(self)
-
-
 ext_modules = [
     MultipyRuntimeExtension("multipy.so"),
 ]
@@ -206,15 +151,13 @@ def get_nightly_version():
 if __name__ == "__main__":
     if sys.version_info < (3, 7):
         sys.exit("python >= 3.7 required for multipy")
-
     name = "torchdeploy"
     NAME_ARG = "--override-name"
     if NAME_ARG in sys.argv:
         idx = sys.argv.index(NAME_ARG)
         name = sys.argv.pop(idx + 1)
         sys.argv.pop(idx)
     is_nightly = "nightly" in name
-
     with open("README.md", encoding="utf8") as f:
         readme = f.read()
 
@@ -230,7 +173,7 @@ def get_nightly_version():
         version=version,
         author="MultiPy Devs",
         # TODO: @sahanp create email for MultiPy
-        author_email="sahanp@fb.com",
+        author_email="sahanp@meta.com",
         description="torch::deploy (multipy) is a C++ library that makes it easier to run eager PyTorch models in production by using independent python interpreters to avoid the GIL.",
         long_description=readme,
         long_description_content_type="text/markdown",
@@ -251,8 +194,6 @@ def get_nightly_version():
         ext_modules=ext_modules,
         cmdclass={
             "build_ext": MultipyRuntimeBuild,
-            "develop": MultipyRuntimeDevelop,
-            "install": MultipyRuntimeInstall,
             "clean": MultipyRuntimeClean,
         },
         package_data={