Merge pull request #204 from man-group/bugfix/server-fixtures

Bugfix/server fixtures

Author: eeaston

.circleci/config.yml

@@ -3,6 +3,7 @@ test-tmpl: &test-tmpl
   command: |
     . ../venv/bin/activate
     export DEBUG=1
+    export SERVER_FIXTURES_HOSTNAME=127.0.0.1
     export SERVER_FIXTURES_JENKINS_WAR=
     cat *.egg-info/top_level.txt  | xargs -Ipysrc coverage run -p --source=pysrc setup.py test -sv -ra || touch ../FAILED-$(basename $PWD)
 
@@ -132,11 +133,6 @@ job-tmpl: &job-tmpl
 
 version: 2
 jobs:
-  py35:
-    <<: *job-tmpl
-    environment:
-      PYTHON: "python3.5"
-
   py36:
     <<: *job-tmpl
     environment:

Makefile

@@ -8,7 +8,7 @@ EXTRA_DEPS = setuptools-git \
              python-jenkins \
              redis          \
              pymongo        \
-             psycopg2       \
+             psycopg2-binary\
              boto3          \
              rethinkdb      \
              docker         \
@@ -17,18 +17,21 @@ EXTRA_DEPS = setuptools-git \
 
 COPY_FILES = VERSION CHANGES.md common_setup.py MANIFEST.in LICENSE
 UPLOAD_OPTS =
+PYPI_INDEX =
+
+PIP_INSTALL_ARGS := $(shell [ ! -z "${PYPI_INDEX}" ] && echo --index ${PYPI_INDEX} )
 
 # removed from PHONY:  circleci_sip circleci_pyqt
 .PHONY: extras copyfiles wheels eggs sdists install develop test upload clean
 
 extras:
-	pip install $(EXTRA_DEPS)
+	pip install ${PIP_INSTALL_ARGS} ${EXTRA_DEPS}
 
 copyfiles:
-	./foreach.sh 'for file in $(COPY_FILES); do cp ../$$file .; done'
+	./foreach.sh 'for file in ${COPY_FILES}; do cp ../$$file .; done'
 
 wheels: copyfiles
-	pip install -U wheel
+	pip install ${PIP_INSTALL_ARGS} -U wheel
 	./foreach.sh --changed 'python setup.py bdist_wheel'
 
 eggs: copyfiles
@@ -38,12 +41,12 @@ sdists: copyfiles
 	./foreach.sh --changed 'python setup.py sdist'
 
 install: copyfiles
-	pip install -U wheel
+	pip install ${PIP_INSTALL_ARGS} -U wheel
 	./foreach.sh 'python setup.py bdist_wheel'
-	./foreach.sh 'pip install dist/*.whl'
+	./foreach.sh 'pip install ${PIP_INSTALL_ARGS} dist/*.whl'
 
 develop: copyfiles extras
-	./foreach.sh 'pip install -e.[tests]'
+	./foreach.sh 'pip install ${PIP_INSTALL_ARGS} -e.[tests]'
 
 test:
 	rm -f FAILED-*

README.md

@@ -51,14 +51,21 @@ To run all the tests:
     make test
 ```
 
-To setup test environment in Vagrant (requires virtualbox):
+## Vagrant
+
+Some of the plugins have complex dependencies, particularly `pytest-server-fixtures`.
+To make it easier to develop, there is a `Vagrantfile` which will setup a virtual machine
+with all the dependencies installed to run the tests.
+
+To set up the environment in Vagrant (requires virtualbox) and run the tests:
 
 ```bash
     $ vagrant up
     $ vagrant ssh
 
     # ..... inside vagrant ....
     . venv/bin/activate
+    cd src
     make develop
     make test
 ```
@@ -72,3 +79,20 @@ This example will build all the wheel distributions:
     ./foreach.sh python setup.py bdist_wheel
 ```
 
+### Only-Changed mode
+
+To run a command only on packages that have changed since the last tagged release, use `--changed`.
+This example will only upload packages that need releasing:
+
+```bash
+    ./foreach.sh python setup.py bdist_wheel upload
+```
+
+### Quiet mode
+
+To run a command with no extra output other than from what you run, use `--quiet`
+```bash
+    ./foreach.sh --quiet grep PY3
+```
+
+

install.sh

@@ -32,11 +32,14 @@ function install_python_packaging {
 function install_python {
   local py=$1
   apt-get install -y $py $py-dev
-  curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | $py
+  if [ "$py" = "python3.6" ]; then
+    curl --silent --show-error --retry 5  https://bootstrap.pypa.io/pip/3.6/get-pip.py | $py
+  else
+    curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | $py
+  fi
   install_python_packaging $py
 }
 
-
 function choco_install {
   local args=$*
   # choco fails randomly with network errors on travis, have a few goes
@@ -115,7 +118,8 @@ function install_system_deps {
     x11-utils \
     subversion \
     graphviz \
-    pandoc
+    pandoc \
+    net-tools
 }
 
 function install_postgresql {

pytest-profiling/pytest_profiling.py

@@ -8,7 +8,7 @@
 import pstats
 import errno
 from hashlib import md5
-from subprocess import Popen, PIPE
+import subprocess
 
 import six
 import pytest
@@ -71,8 +71,8 @@ def pytest_sessionfinish(self, session, exitstatus):  # @UnusedVariable
 
                 # A handcrafted Popen pipe actually seems to work on both windows and unix:
                 # do it in 2 subprocesses, with a pipe in between
-                pdot = Popen(dot_args, stdin=PIPE, shell=True)
-                pgprof = Popen(gprof2dot_args, stdout=pdot.stdin, shell=True)
+                pdot = subprocess.Popen(dot_args, stdin=subprocess.PIPE, shell=True)
+                pgprof  = subprocess.Popen(gprof2dot_args, stdout=pdot.stdin, shell=True)
                 (stdoutdata1, stderrdata1) = pgprof.communicate()
                 (stdoutdata2, stderrdata2) = pdot.communicate()
                 if stderrdata1 is not None or pgprof.poll() > 0:

pytest-profiling/tests/unit/test_profile.py

@@ -5,27 +5,31 @@
 from six.moves import reload_module  # @UnresolvedImport
 
 import pytest_profiling
+
 reload_module(pytest_profiling)
 
+import os
+import subprocess
+
 from pytest_profiling import Profiling, pytest_addoption, pytest_configure
 
 try:
-    from unittest.mock import Mock, ANY, patch, sentinel
+    from unittest.mock import Mock, ANY, patch, sentinel, call
 except ImportError:
     # python 2
     from mock import Mock, ANY, patch, sentinel
 
 
 def test_creates_prof_dir():
-    with patch('os.makedirs', side_effect=OSError) as makedirs:
+    with patch("os.makedirs", side_effect=OSError) as makedirs:
         Profiling(False).pytest_sessionstart(Mock())
-    makedirs.assert_called_with('prof')
+    makedirs.assert_called_with("prof")
 
 
 def test_combines_profs():
     plugin = Profiling(False)
     plugin.profs = [sentinel.prof0, sentinel.prof1]
-    with patch('pstats.Stats') as Stats:
+    with patch("pstats.Stats") as Stats:
         plugin.pytest_sessionfinish(Mock(), Mock())
     Stats.assert_called_once_with(sentinel.prof0)
     Stats.return_value.add.assert_called_once_with(sentinel.prof1)
@@ -34,51 +38,73 @@ def test_combines_profs():
 
 def test_generates_svg():
     plugin = Profiling(True)
+    plugin.gprof2dot = "/somewhere/gprof2dot"
     plugin.profs = [sentinel.prof]
-    with patch('pstats.Stats'):
-        with patch('pipes.Template') as Template:
+    popen1 = Mock(
+        communicate=Mock(return_value=[None, None]), poll=Mock(return_value=0)
+    )
+    popen2 = Mock(
+        communicate=Mock(return_value=[None, None]), poll=Mock(return_value=0)
+    )
+    with patch("pstats.Stats"):
+        with patch("subprocess.Popen", side_effect=[popen1, popen2]) as popen:
             plugin.pytest_sessionfinish(Mock(), Mock())
-    assert any('gprof2dot' in args[0][0] for args in Template.return_value.append.call_args_list)
-    assert Template.return_value.copy.called
+    calls = popen.mock_calls
+    assert calls[0] == call(
+        ["dot", "-Tsvg", "-o", f"{os.getcwd()}/prof/combined.svg"],
+        stdin=subprocess.PIPE,
+        shell=True,
+    )
+    assert calls[1] == call(
+        ["/somewhere/gprof2dot", "-f", "pstats", f"{os.getcwd()}/prof/combined.prof"],
+        stdout=popen1.stdin,
+        shell=True,
+    )
 
 
 def test_writes_summary():
     plugin = Profiling(False)
     plugin.profs = [sentinel.prof]
     terminalreporter, stats = Mock(), Mock()
-    with patch('pstats.Stats', return_value=stats) as Stats:
+    with patch("pstats.Stats", return_value=stats) as Stats:
         plugin.pytest_sessionfinish(Mock(), Mock())
         plugin.pytest_terminal_summary(terminalreporter)
-    assert 'Profiling' in terminalreporter.write.call_args[0][0]
+    assert "Profiling" in terminalreporter.write.call_args[0][0]
     assert Stats.called_with(stats, stream=terminalreporter)
 
 
 def test_writes_summary_svg():
     plugin = Profiling(True)
     plugin.profs = [sentinel.prof]
     terminalreporter = Mock()
-    with patch('pstats.Stats'):
-        with patch('pipes.Template'):
+    popen1 = Mock(
+        communicate=Mock(return_value=[None, None]), poll=Mock(return_value=0)
+    )
+    popen2 = Mock(
+        communicate=Mock(return_value=[None, None]), poll=Mock(return_value=0)
+    )
+    with patch("pstats.Stats"):
+        with patch("subprocess.Popen", side_effect=[popen1, popen2]):
             plugin.pytest_sessionfinish(Mock(), Mock())
         plugin.pytest_terminal_summary(terminalreporter)
-    assert 'SVG' in terminalreporter.write.call_args[0][0]
+    assert "SVG" in terminalreporter.write.call_args[0][0]
 
 
 def test_adds_options():
     parser = Mock()
     pytest_addoption(parser)
-    parser.getgroup.assert_called_with('Profiling')
+    parser.getgroup.assert_called_with("Profiling")
     group = parser.getgroup.return_value
-    group.addoption.assert_any_call('--profile', action='store_true', help=ANY)
-    group.addoption.assert_any_call('--profile-svg', action='store_true', help=ANY)
+    group.addoption.assert_any_call("--profile", action="store_true", help=ANY)
+    group.addoption.assert_any_call("--profile-svg", action="store_true", help=ANY)
 
 
 def test_configures():
-    config = Mock(getvalue=lambda x: x == 'profile')
-    with patch('pytest_profiling.Profiling') as Profiling:
+    config = Mock(getvalue=lambda x: x == "profile")
+    with patch("pytest_profiling.Profiling") as Profiling:
         pytest_configure(config)
     config.pluginmanager.register.assert_called_with(Profiling.return_value)
 
 
 def test_clean_filename():
-    assert pytest_profiling.clean_filename('a:b/c\256d') == 'a_b_c_d'
+    assert pytest_profiling.clean_filename("a:b/c\256d") == "a_b_c_d"

pytest-pyramid-server/README.md

@@ -76,7 +76,7 @@ To do this you can use the underlying server class directly - this is an implene
                                                        'my_dbpass: 'bar'}}
            ) as server:
                assert not server.dead
-               assert 'my_dbname = foo' in server.working_config.text()
+               assert 'my_dbname = foo' in server.working_config.read_text()
 
         # Server should now be dead
         assert server.dead   

pytest-pyramid-server/pytest_pyramid_server.py

@@ -20,6 +20,7 @@
 from paste.deploy.loadwsgi import loadapp
 from pytest import yield_fixture
 
+from pytest_server_fixtures import CONFIG
 from pytest_server_fixtures.http import HTTPTestServer
 
 
@@ -83,8 +84,7 @@ def __init__(self, config_dir=None, config_filename=None, extra_config_vars=None
         # Always print debug output for this process
         os.environ['DEBUG'] = '1'
 
-        # Discover externally accessable hostname so selenium can get to it
-        kwargs['hostname'] = kwargs.get('hostname', socket.gethostbyname(os.uname()[1]))
+        kwargs['hostname'] = kwargs.get('hostname', CONFIG.fixture_hostname)
 
         super(PyramidTestServer, self).__init__(preserve_sys_path=True, **kwargs)
 
@@ -112,8 +112,8 @@ def pre_setup(self):
         except configparser.NoOptionError:
             parser.set('app:main', 'url_prefix', '')
 
-        # Set the uri to be the external hostname and the url prefix
-        self._uri = "http://%s:%s/%s" % (os.uname()[1], self.port, parser.get('app:main', 'url_prefix'))
+        # Set the uri to be the hostname and the url prefix
+        self._uri = "http://%s:%s/%s" % (self.hostname, self.port, parser.get('app:main', 'url_prefix'))
 
     @property
     def run_cmd(self):