Merge pull request #159 from BD2KGenomics/features/update-s3am

Bump s3am and remove version pin (up to 2.0)

core/setup.py

@@ -4,28 +4,28 @@
 
 from version import cgcloud_version, bd2k_python_lib_dep, boto_dep, fabric_dep
 
-setup(
-    name='cgcloud-core',
-    version=cgcloud_version,
+setup( name='cgcloud-core',
+       version=cgcloud_version,
 
-    author='Hannes Schmidt',
-    author_email='[email protected]',
-    url='https://github.com/BD2KGenomics/cgcloud',
-    description='Efficient and reproducible software deployment for EC2 instances',
+       author='Hannes Schmidt',
+       author_email='[email protected]',
+       url='https://github.com/BD2KGenomics/cgcloud',
+       description='Efficient and reproducible software deployment for EC2 instances',
 
-    package_dir={ '': 'src' },
-    packages=find_packages( 'src', exclude=[ '*.test' ] ),
-    namespace_packages=[ 'cgcloud' ],
-    entry_points={
-        'console_scripts': [
-            'cgcloud = cgcloud.core.cli:main' ], },
-    install_requires=[
-        bd2k_python_lib_dep,
-        'cgcloud-lib==' + cgcloud_version,
-        'futures==3.0.4', # such that cgcloud-lib can use the futures backport for its thread_pool
-        boto_dep,
-        fabric_dep,
-        'futures==3.0.4',
-        'PyYAML==3.11',
-        'subprocess32==3.2.7' ],
-    test_suite='cgcloud.core.test' )
+       package_dir={ '': 'src' },
+       packages=find_packages( 'src', exclude=[ '*.test' ] ),
+       namespace_packages=[ 'cgcloud' ],
+       entry_points={
+           'console_scripts': [
+               'cgcloud = cgcloud.core.cli:main' ], },
+       install_requires=[ bd2k_python_lib_dep,
+                          'cgcloud-lib==' + cgcloud_version,
+                          'futures==3.0.4',
+                          # such that cgcloud-lib can use the futures backport for its thread_pool
+                          boto_dep,
+                          fabric_dep,
+                          'paramiko==1.16.0',
+                          'futures==3.0.4',
+                          'PyYAML==3.11',
+                          'subprocess32==3.2.7' ],
+       test_suite='cgcloud.core.test' )

core/src/cgcloud/fabric/operations.py

@@ -79,7 +79,7 @@ def virtualenv( name, distributions=None, pip_distribution='pip', executable=Non
     :param name: the name of the directory under /opt that will hold the virtualenv
 
     :param distributions: a list of distributions to be installed into the virtualenv. Defaults
-    to [ name ]
+    to [ name ]. You can also list other "pip install" options, like --pre.
 
     :param pip_distribution: if non-empty, the distribution and optional version spec to upgrade
     pip to. Defaults to the latest version of pip. Set to empty string to prevent pip from being

jenkins/setup.py

@@ -4,19 +4,17 @@
 
 from version import cgcloud_version, fabric_dep
 
-setup(
-    name='cgcloud-jenkins',
-    version=cgcloud_version,
+setup( name='cgcloud-jenkins',
+       version=cgcloud_version,
 
-    author="Hannes Schmidt",
-    author_email="[email protected]",
-    url='https://github.com/BD2KGenomics/cgcloud',
-    description='Setup and manage a Jenkins continuous integration cluster in EC2',
+       author="Hannes Schmidt",
+       author_email="[email protected]",
+       url='https://github.com/BD2KGenomics/cgcloud',
+       description='Setup and manage a Jenkins continuous integration cluster in EC2',
 
-    package_dir={ '': 'src' },
-    packages=find_packages( 'src' ),
-    namespace_packages=[ 'cgcloud' ],
-    install_requires=[
-        'cgcloud-lib==' + cgcloud_version,
-        'cgcloud-core=='  + cgcloud_version,
-        fabric_dep ] )
+       package_dir={ '': 'src' },
+       packages=find_packages( 'src' ),
+       namespace_packages=[ 'cgcloud' ],
+       install_requires=[ 'cgcloud-lib==' + cgcloud_version,
+                          'cgcloud-core==' + cgcloud_version,
+                          fabric_dep ] )

mesos/setup.py

@@ -4,20 +4,18 @@
 
 from version import cgcloud_version, bd2k_python_lib_dep, fabric_dep
 
-setup(
-    name='cgcloud-mesos',
-    version=cgcloud_version,
+setup( name='cgcloud-mesos',
+       version=cgcloud_version,
 
-    author='Christopher Ketchum',
-    author_email='[email protected]',
-    url='https://github.com/BD2KGenomics/cgcloud',
-    description='Setup and manage a Apache Mesos cluster in EC2',
+       author='Christopher Ketchum',
+       author_email='[email protected]',
+       url='https://github.com/BD2KGenomics/cgcloud',
+       description='Setup and manage a Apache Mesos cluster in EC2',
 
-    package_dir={ '': 'src' },
-    packages=find_packages( 'src' ),
-    namespace_packages=[ 'cgcloud' ],
-    install_requires=[
-        'cgcloud-lib==' + cgcloud_version,
-        'cgcloud-core==' + cgcloud_version,
-        bd2k_python_lib_dep,
-        fabric_dep ] )
+       package_dir={ '': 'src' },
+       packages=find_packages( 'src' ),
+       namespace_packages=[ 'cgcloud' ],
+       install_requires=[ 'cgcloud-lib==' + cgcloud_version,
+                          'cgcloud-core==' + cgcloud_version,
+                          bd2k_python_lib_dep,
+                          fabric_dep ] )

spark/setup.py

@@ -4,20 +4,18 @@
 
 from version import cgcloud_version, bd2k_python_lib_dep, fabric_dep
 
-setup(
-    name='cgcloud-spark',
-    version=cgcloud_version,
+setup( name='cgcloud-spark',
+       version=cgcloud_version,
 
-    author='Hannes Schmidt',
-    author_email='[email protected]',
-    url='https://github.com/BD2KGenomics/cgcloud',
-    description='Setup and manage a Apache Spark cluster in EC2',
+       author='Hannes Schmidt',
+       author_email='[email protected]',
+       url='https://github.com/BD2KGenomics/cgcloud',
+       description='Setup and manage a Apache Spark cluster in EC2',
 
-    package_dir={ '': 'src' },
-    packages=find_packages( 'src' ),
-    namespace_packages=[ 'cgcloud' ],
-    install_requires=[
-        bd2k_python_lib_dep,
-        'cgcloud-lib==' + cgcloud_version,
-        'cgcloud-core==' + cgcloud_version,
-        fabric_dep ] )
+       package_dir={ '': 'src' },
+       packages=find_packages( 'src' ),
+       namespace_packages=[ 'cgcloud' ],
+       install_requires=[ bd2k_python_lib_dep,
+                          'cgcloud-lib==' + cgcloud_version,
+                          'cgcloud-core==' + cgcloud_version,
+                          fabric_dep ] )

toil/setup.py

@@ -2,21 +2,19 @@
 from setuptools import setup, find_packages
 from version import cgcloud_version, bd2k_python_lib_dep, fabric_dep
 
-setup(
-    name='cgcloud-toil',
-    version=cgcloud_version,
+setup( name='cgcloud-toil',
+       version=cgcloud_version,
 
-    author='Christopher Ketchum',
-    author_email='[email protected]',
-    url='https://github.com/BD2KGenomics/cgcloud',
-    description='Setup and manage a toil and Apache Mesos cluster in EC2',
+       author='Christopher Ketchum',
+       author_email='[email protected]',
+       url='https://github.com/BD2KGenomics/cgcloud',
+       description='Setup and manage a toil and Apache Mesos cluster in EC2',
 
-    package_dir={ '': 'src' },
-    packages=find_packages( 'src' ),
-    namespace_packages=[ 'cgcloud' ],
-    install_requires=[
-        'cgcloud-lib==' + cgcloud_version,
-        'cgcloud-core==' + cgcloud_version,
-        'cgcloud-mesos==' + cgcloud_version,
-        bd2k_python_lib_dep,
-        fabric_dep ] )
+       package_dir={ '': 'src' },
+       packages=find_packages( 'src' ),
+       namespace_packages=[ 'cgcloud' ],
+       install_requires=[ 'cgcloud-lib==' + cgcloud_version,
+                          'cgcloud-core==' + cgcloud_version,
+                          'cgcloud-mesos==' + cgcloud_version,
+                          bd2k_python_lib_dep,
+                          fabric_dep ] )

toil/src/cgcloud/toil/toil_box.py

@@ -96,8 +96,9 @@ def __install_toil( self ):
 
     @fabric_task
     def __install_s3am( self ):
+        from version import s3am_dep
         virtualenv( name='s3am',
-                    distributions=[ 's3am==1.1' ],
+                    distributions=[ s3am_dep ],
                     pip_distribution='pip==8.0.2',
                     executable='s3am' )
 

version.py

@@ -1,15 +1,17 @@
 cgcloud_version = '1.4a1'
 bd2k_python_lib_dep = 'bd2k-python-lib>=1.11.dev6'
 boto_dep = 'boto==2.38.0'
-fabric_dep = 'Fabric==1.10.2'
+fabric_dep = 'Fabric==1.10.3'
+s3am_dep = 's3am>=2.0a1.dev99,<2.1'
 
 if __name__ == '__main__':
     import os
     from pkg_resources import parse_version
-    is_release_build = not parse_version(cgcloud_version).is_prerelease
+
+    is_release_build = not parse_version( cgcloud_version ).is_prerelease
     suffix = '' if is_release_build else '.dev' + os.environ.get( 'BUILD_NUMBER', '0' )
-    for name, value in globals().items():
-        if name.startswith('cgcloud_'):
+    for name, value in globals( ).items( ):
+        if name.startswith( 'cgcloud_' ):
             value += suffix
-        if name.endswith('_dep') or name.endswith('_version'):
-            print "%s='%s'" % ( name, value )
+        if name.split('_')[-1] in ('dep', 'version' ):
+            print "%s='%s'" % (name, value)