Merge pull request #901 from MDAnalysis/issue-898-external-code

install external packages for tests

.travis.yml

@@ -15,6 +15,7 @@ env:
     - GIT_CI_USER: TravisCI
     - GIT_CI_EMAIL: TravisCI@mdanalysis.org
     - MDA_DOCDIR: package/doc/html
+    - MDA_OPTPACKAGES: opt/packages
   matrix:
     - SETUP=minimal PYTHON_VERSION=2.7
     - SETUP=full PYTHON_VERSION=2.7
@@ -37,14 +38,28 @@ install:
   - if [[ $SETUP == 'full' ]]; then conda create --yes -q -n pyenv python=$PYTHON_VERSION numpy scipy nose=1.3.7 sphinx=1.3 griddataformats six; fi
   - if [[ $SETUP == 'minimal' ]]; then conda create --yes -q -n pyenv python=$PYTHON_VERSION numpy nose=1.3.7 sphinx=1.3 griddataformats six; fi
   - source activate pyenv
-  - if [[ $SETUP == 'full' ]]; then conda install --yes cython biopython matplotlib networkx netcdf4; fi
+  - |
+    if [[ $SETUP == 'full' ]]; then \
+        conda install --yes cython biopython matplotlib networkx netcdf4; \
+        conda install -c biobuilds --yes clustalw=2.1; \
+    fi
   - if [[ $SETUP == 'minimal' ]]; then conda install --yes cython biopython networkx; fi
 # ensure that cython files are rebuilt
   - find . -name '*.pyx' -exec touch '{}' \;
   - pip install -v package/
   - pip install testsuite/
   - pip install coveralls
   - chmod +x testsuite/MDAnalysisTests/mda_nosetests
+# additional external tools (Issue #898) -- HOLE
+  - |
+     if [[ $SETUP == 'full' ]]; then \
+         bash ./maintainer/install_hole.sh $TRAVIS_OS_NAME "${HOME}/${MDA_OPTPACKAGES}"; \
+         HOLE_BINDIR="${HOME}/${MDA_OPTPACKAGES}/hole2/exe"; \
+         export PATH=${PATH}:${HOLE_BINDIR}; \
+     fi
+  # we should still be here but make doubly sure:
+  - cd ${TRAVIS_BUILD_DIR}
+
 # command to run tests
 script:
   - ./testsuite/MDAnalysisTests/mda_nosetests --with-coverage --cover-package MDAnalysis --processes=2 --process-timeout=300 --with-memleak

maintainer/install_hole.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+#
+# Written by Oliver Beckstein, 2016
+#
+# This script is placed into the Public Domain using the CC0 1.0 Public Domain
+# Dedication https://creativecommons.org/publicdomain/zero/1.0/
+#
+#
+# NOTE: IF YOU RUN THIS SCRIPT YOU MUST BE ABLE TO COMPLY WITH THE HOLE
+#       NOT-FOR-PROFIT LICENSE.
+# 
+#
+# Install HOLE from http://www.smartsci.uk/hole/
+#
+# arguments
+#    OSNAME : linux | darwin
+#    PREFIX : "path/to/installdir"
+#
+# PREFIX can be relative to CWD or an absolute path; it will be created
+# and HOLE unpacked under PREFIX (the tar file contains "hole2/...")
+#
+#
+# HOLE is used under the terms of the 'HOLE END USER LICENCE AGREEMENT
+# NOT-FOR-PROFIT VERSION' as provided in the installation tarball as file
+# 'doc/Licence-not-for-profit.asciidoc' and see copy at
+# https://github.com/MDAnalysis/mdanalysis/files/372246/Licence-not-for-profit.txt
+# (recent as of 2016-07-19).
+#
+
+
+set -o errexit -o nounset
+
+SCRIPT=$(basename $0)
+
+DOWNLOAD_URL_LINUX='https://www.dropbox.com/s/jukpwlohhi20r17/hole2-NotForProfit-2.2.004-Linux-x86_64.tar.gz?dl=1'
+TARFILE_LINUX='hole2-NotForProfit-2.2.004-Linux-x86_64.tar.gz'
+
+DOWNLOAD_URL_DARWIN='https://www.dropbox.com/s/5mzrsyp48i32je4/hole2-NotForProfit-2.2.004-Darwin-i386.tar.gz?dl=1'
+TARFILE_DARWIN=hole2-NotForProfit-2.2.004-Darwin-i386.tar.gz
+
+# path to dir with executables in the current HOLE distribution
+HOLE_EXE_DIR=hole2/exe
+
+function die () {
+    local msg="$1" err=$2
+    echo "[${SCRIPT}] ERROR: $msg [$err]"
+    exit $err
+}
+
+function is_native_executable () {
+    local filename="$1" OSNAME="$2"
+    file "${filename}" | grep -qi ${OFORMAT}
+    return $?
+}
+
+OSNAME="$1"
+case "$OSNAME" in
+    Linux|linux)
+	OSNAME=Linux
+	OFORMAT=Linux
+	DOWNLOAD_URL="${DOWNLOAD_URL_LINUX}"
+	TARFILE=${TARFILE_LINUX}
+	;;
+    OSX|osx|Darwin|darwin)
+	OSNAME=Darwin
+	OFORMAT="Mach-O"
+	DOWNLOAD_URL="${DOWNLOAD_URL_DARWIN}"
+	TARFILE=${TARFILE_DARWIN}
+	;;
+    *)
+	die "OS '${OSNAME}' not supported." 10;;
+esac
+
+PREFIX="$2"
+test -n "$PREFIX" || die "missing second argument PREFIX (installation directory)" 10
+
+#------------------------------------------------------------
+# start installation
+#------------------------------------------------------------
+
+mkdir -p "$PREFIX" && cd "$PREFIX" || die "Failed to create and cd to $PREFIX" 1
+if ! test -f ${TARFILE}; then
+    echo "Downloading ${TARFILE} from ${DOWNLOAD_URL}..."
+    # fixing curl on travis/anaconda, see http://stackoverflow.com/a/31060428/334357
+    export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+    curl -L ${DOWNLOAD_URL} -o ${TARFILE} || \
+    die "Failed to download ${TARFILE} from ${DOWNLOAD_URL}" 1
+fi
+
+# only install the executables in HOLE_EXE_DIR 
+tar xvf ${TARFILE} ${HOLE_EXE_DIR} || die "Failed 'tar xvf ${TARFILE} ${HOLE_EXE_DIR}'" 1
+
+MDA_HOLE_BINDIR="${PWD}/${HOLE_EXE_DIR}"
+HOLE_EXE="${MDA_HOLE_BINDIR}/hole"
+
+test -d "${MDA_HOLE_BINDIR}" || { \
+    echo "Content of ${PWD}:";
+    ls -la;
+    die "no HOLE exe dir ${MDA_HOLE_BINDIR} in $PWD" 2; }
+test -f "${HOLE_EXE}" || die "hole executable ${HOLE_EXE} not installed" 2
+is_native_executable ${HOLE_EXE} ${OFORMAT} || \
+    { file ${HOLE_EXE}; \
+      die "${HOLE_EXE} will not run on ${OSNAME} (object format ${OFORMAT})" 3; }
+
+echo "HOLE executables were installed into ${MDA_HOLE_BINDIR}"
+echo ">>> export PATH=\${PATH}:${MDA_HOLE_BINDIR}"
+
+# repeat this line in .travis.yml
+export PATH=${PATH}:${MDA_HOLE_BINDIR}

package/MDAnalysis/analysis/hole.py

@@ -23,7 +23,7 @@
 
 With the help of this module, the :program:`hole` program from the HOLE_ suite
 of tools [Smart1993]_ [Smart1996]_ can be run on frames in a trajectory. Data
-can be combined and analyzed.  
+can be combined and analyzed.
 
 
 HOLE_ must be installed separately and can be obtained in binary form from
@@ -70,10 +70,10 @@
 :class:`HOLEtraj`. In this case, provide a
 :class:`~MDAnalysis.core.AtomGroup.Universe`::
 
-   import MDAnalysis as mda   
+   import MDAnalysis as mda
    from MDAnalysis.analysis.hole import HOLEtraj
    from MDAnalysis.tests.datafiles import MULTIPDB_HOLE
-   
+
    u = mda.Universe(MULTIPDB_HOLE)
    H = HOLEtraj(u, executable="~/hole2/exe/hole")
    H.run()
@@ -99,17 +99,17 @@
    from MDAnalysis.analysis.rms import RMSD
 
    from MDAnalysis.tests.datafiles import PDB_HOLE, MULTIPDB_HOLE
-   
+
    mda.start_logging()
    ref = mda.Universe(PDB_HOLE)    # reference structure
    u = mda.Universe(MULTIPDB_HOLE) # trajectory
-   
+
    # calculate RMSD
    R = RMSD(u, reference=ref, select="protein", mass_weighted=True)
    R.run()
-   
+
    # HOLE analysis with order parameters
-   H = HOLEtraj(u, orderparameters=R.rmsd[:,2], 
+   H = HOLEtraj(u, orderparameters=R.rmsd[:,2],
                 executable="~/hole2/exe/hole")
    H.run()
 
@@ -121,7 +121,7 @@
 minimum radius as function of order parameter
 
 .. math::
- 
+
    r(\rho) = \min_\zeta R_\rho(\zeta)
 
 we iterate over the profiles and process them in turn::
@@ -181,7 +181,7 @@
       frame was analyzed then this will be ``HOLE.profiles[0]``. Note
       that the order is random; one needs to sort the keys first.
 
-      .. Note:: 
+      .. Note::
          Duplicate keys are not possible. The last key overwrites
          previous values. This is arguably a bug.
 
@@ -196,7 +196,7 @@
       parameter (if *orderparameters* was supplied). Note that the
       order is random; one needs to sort the keys first.
 
-      .. Note:: 
+      .. Note::
          Duplicate keys are not possible. The last key overwrites
          previous values. This is arguably a bug.
 
@@ -309,7 +309,7 @@ def write_simplerad2(filename="simple2.rad"):
 
     Does nothing if `filename` already exists.
 
-    Parameters 
+    Parameters
     ----------
     filename : string, optional
        output file name; the default is "simple2.rad"
@@ -382,7 +382,7 @@ def _process_plot_kwargs(self, kwargs):
         color = kwargs.pop('color', None)
         if color is None:
             cmap = kwargs.pop('cmap', matplotlib.cm.viridis)
-            normalize = matplotlib.colors.Normalize(vmin=np.min(frames), 
+            normalize = matplotlib.colors.Normalize(vmin=np.min(frames),
                                                     vmax=np.max(frames))
             colors = cmap(normalize(frames))
         else:
@@ -414,7 +414,7 @@ def plot(self, **kwargs):
         yshift : float, optional
               displace each :math:`R(\zeta)` profile by `yshift` in the
               :math:`y`-direction for clearer visualization. The default is 0,
-              i.e., not to shift any graph.        
+              i.e., not to shift any graph.
         frames : integer or array_like, optional
               only plot these specific frame(s); the default ``None`` is to
               plot everything (see also `step`)
@@ -545,7 +545,7 @@ def plot3D(self, **kwargs):
         ax.set_xlabel(r"pore coordinate $\zeta$ ($\AA$)")
         ax.set_ylabel(ylabel)
         ax.set_zlabel(r"HOLE radius $R$ ($\AA$)")
-        
+
         return ax
 
     def min_radius(self):
@@ -1015,27 +1015,37 @@ def create_vmd_surface(self, filename="hole.vmd", **kwargs):
         fd, tmp_sos = tempfile.mkstemp(suffix=".sos", text=True)
         os.close(fd)
         try:
-            rc = subprocess.call([
-                self.exe["sph_process"], "-sos", "-dotden", str(kwargs['dotden']),
-                "-color", self.sphpdb, tmp_sos])
-            if rc != 0:
-                logger.fatal("sph_process failed (%d)", rc)
-                raise OSError(rc, "sph_process failed")
-            with open(tmp_sos) as sos:
-                with open(filename, "w") as triangles:
-                    rc = subprocess.call([self.exe["sos_triangle"], "-s"], stdin=sos, stdout=triangles)
-            if rc != 0:
-                logger.fatal("sos_triangle failed (%d)", rc)
-                raise OSError(rc, "sos_triangle failed")
+            output = subprocess.check_output([self.exe["sph_process"], "-sos", "-dotden",
+                                              str(kwargs['dotden']), "-color", self.sphpdb,
+                                              tmp_sos], stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as err:
+            os.unlink(tmp_sos)
+            logger.fatal("sph_process failed ({0})".format(err.returncode))
+            raise OSError(err.returncode, "sph_process failed")
         except:
             os.unlink(tmp_sos)
             raise
+
+        try:
+            # Could check: os.devnull if subprocess.DEVNULL not available (>3.3)
+            # Suppress stderr messages of sos_triangle
+            with open(tmp_sos) as sos, open(filename, "w") as triangles, \
+                 open(os.devnull, 'w') as FNULL:
+                subprocess.check_call(
+                    [self.exe["sos_triangle"], "-s"], stdin=sos, stdout=triangles,
+                    stderr=FNULL)
+        except subprocess.CalledProcessError as err:
+            logger.fatal("sos_triangle failed ({0})".format(err.returncode))
+            raise OSError(err.returncode, "sos_triangle failed")
+        finally:
+            os.unlink(tmp_sos)
+
         return filename
 
     def collect(self, **kwargs):
         """Parse the output from a :class:`HOLE` run into numpy recarrays.
 
-        It can process outputs containing multiple frames (when a DCD was supplied to :program:`hole`). 
+        It can process outputs containing multiple frames (when a DCD was supplied to :program:`hole`).
 
         Output format::
 

testsuite/CHANGELOG

@@ -21,6 +21,8 @@ and https://github.com/MDAnalysis/mdanalysis/wiki/UnitTests
     - The test_failure test can be made fail by setting the MDA_FAILURE_TEST
       environment variable (PR #874)
     - replaced XTC_HOLE test trajectory with more meaningful one MULTIPDB_HOLE
+    - install external packages on Travis (SETUP == full: HOLE, clustalw)
+      to test additional analysis code (Issue #898)
 
 05/15/16  orbeckst, jbarnoud, pedrishi, fiona-naughton, jdetle
   * 0.15.0

testsuite/MDAnalysisTests/__init__.py

@@ -121,7 +121,9 @@
 # If MDAnalysis is imported here coverage accounting might fail because all the import
 #  code won't be run again under coverage's watch. See Issue 344.
 
-from os.path import dirname
+import os
+from contextlib import contextmanager
+
 # We get our nose from the plugins so that version-checking needs only be done there.
 from MDAnalysisTests.plugins import nose, loaded_plugins
 import sys
@@ -134,6 +136,13 @@
     raise ImportError("""numpy>=1.5  is required to run the test suite. Please install it first. """
                       """(For example, try "easy_install 'numpy>=1.5'").""")
 
+# Any tests that plot with matplotlib need to run with the simple agg backend because
+# on Travis there is no DISPLAY set
+try:
+    import matplotlib
+    matplotlib.use('agg')
+except ImportError:
+    pass
 
 def run(*args, **kwargs):
     """Test-running function that loads plugins, sets up arguments, and calls `nose.run_exit()`"""
@@ -154,7 +163,7 @@ def run(*args, **kwargs):
     except KeyError:
         kwargs['addplugins'] = loaded_plugins.values()
     # By default, test our testsuite
-    kwargs['defaultTest'] = dirname(__file__)
+    kwargs['defaultTest'] = os.path.dirname(__file__)
     return nose.run_exit(*args, **kwargs)
 
 
@@ -203,3 +212,33 @@ def parser_not_found(parser_name):
         return True
     else:
         return False
+
+
+@contextmanager
+def in_dir(dirname):
+    """Context manager for safely changing directories.
+
+    Arguments
+    ---------
+    dirname : string
+        directory to change into
+
+    Example
+    -------
+    Change into a temporary directory and always change back to the
+    current one::
+
+      with in_dir("/tmp") as tmpdir:
+          # do stuff
+
+    SeeAlso
+    -------
+    The :mod:`tmpdir` module provides functionality such as :func:`tmpdir.in_tmpdir`
+    to create temporary directories that are automatically deleted once they are no
+    longer used.
+    """
+
+    old_path = os.getcwd()
+    os.chdir(dirname)
+    yield dirname
+    os.chdir(old_path)