Merge pull request #39 from apls777/dev

Spotty v1.2.2

README.md

@@ -37,16 +37,7 @@ Use [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:
    - Read [this](https://medium.com/@apls/how-to-train-deep-learning-models-on-aws-spot-instances-using-spotty-8d9e0543d365) 
    article for a real-world example.
 
-2. Create an AMI. Run the following command from the root directory of your project:
-
-    ```bash
-    $ spotty aws create-ami
-    ```
-
-    In several minutes you will have an AMI with NVIDIA Docker that Spotty will use 
-    for all your projects within the AWS region.
-
-3. Start an instance:
+2. Start an instance:
 
     ```bash
     $ spotty start
@@ -55,11 +46,7 @@ Use [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:
     It will run a Spot Instance, restore snapshots if any, synchronize the project with the running instance 
     and start the Docker container with the environment.
 
-4. Train a model or run notebooks.
-
-    You can run custom scripts inside the Docker container using the `spotty run <SCRIPT_NAME>` command. Read more
-    about custom scripts in the documentation: 
-    [Configuration: "scripts" section](https://apls777.github.io/spotty/docs/configuration/#scripts-section-optional).
+3. Train a model or run notebooks.
 
     To connect to the running container via SSH, use the following command:
 
@@ -71,6 +58,10 @@ Use [pip](http://www.pip-installer.org/en/latest/) to install or upgrade Spotty:
     __`Ctrl + b`__, then __`d`__ combination of keys. To be attached to that session later, just use the
     `spotty ssh` command again.
 
+    Also, you can run your custom scripts inside the Docker container using the `spotty run <SCRIPT_NAME>` command. Read more
+    about custom scripts in the documentation: 
+    [Configuration: "scripts" section](https://apls777.github.io/spotty/docs/configuration/#scripts-section-optional).
+
 ## Contributions
 
 Any feedback or contributions are welcome! Please check out the [guidelines](CONTRIBUTING.md).

docs/docs/aws/instance_parameters.md

@@ -62,3 +62,15 @@ Read more: [How to cache a Docker image](/spotty/docs/faq/#how-to-cache-a-docker
 and `spotty sync` will do SSH connections to the instance using the IP address __127.0.0.1__ and the specified port. 
 It can be useful in case when the instance doesn't have a public IP address and SSH access is provided through a 
 tunnel to a local port.
+
+- __`managedPolicyArns`__ _(optional)_ - a list of Amazon Resource Names (ARNs) of the IAM managed policies that 
+you want to attach to the instance role. Read more about Managed Policies 
+[here](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_managed-vs-inline.html).
+
+- __`commands`__ _(optional)_ - commands that should be run on the host OS before a container is started. 
+For example, you could login to Amazon ECR to pull a Docker image from there 
+([Deep Learning Containers Images](https://docs.aws.amazon.com/dlami/latest/devguide/deep-learning-containers-images.html)):
+    ```yaml
+    commands: |
+      $(aws ecr get-login --no-include-email --region us-east-2 --registry-ids 763104351884)
+    ```

docs/index.md

@@ -43,17 +43,6 @@ Prepare a `spotty.yaml` file and put it to the root directory of your project:
    - Read [this](https://medium.com/@apls/how-to-train-deep-learning-models-on-aws-spot-instances-using-spotty-8d9e0543d365){:target="_blank"} 
    article for a real-world example.
 
-### __Create an AMI__
-
-Run the following command from the root directory of your project:
-
-```bash
-$ spotty aws create-ami
-```
-
-In several minutes you will have an AMI with NVIDIA Docker that Spotty will use 
-for all your projects within the AWS region.
-
 ### __Start an instance__
 
 Use the following command to launch an instance with the Docker container:

spotty/__init__.py

@@ -1 +1 @@
-__version__ = '1.2.1'
+__version__ = '1.2.2'

spotty/config/abstract_instance_config.py

@@ -26,3 +26,8 @@ def docker_data_root(self) -> str:
     def local_ssh_port(self) -> int:
         """Local SSH port to connect to the instance (in case of a tunnel)."""
         return self._params['localSshPort']
+
+    @property
+    def commands(self) -> str:
+        """Commands that should be run once an instance is started."""
+        return self._params['commands']

spotty/config/validation.py

@@ -125,6 +125,7 @@ def get_instance_parameters_schema(instance_parameters: dict, default_volume_typ
                 *volumes_checks,
             ),
             Optional('localSshPort', default=None): Or(None, And(int, lambda x: 0 <= x <= 65535)),
+            Optional('commands', default=''): str,
         },
         And(lambda x: not x['dockerDataRoot'] or
                       [True for v in x['volumes'] if v['parameters']['mountDir'] and

spotty/providers/aws/config/instance_config.py

@@ -36,6 +36,10 @@ def on_demand(self) -> bool:
     def ami_name(self) -> str:
         return self._params['amiName'] if self._params['amiName'] else DEFAULT_AMI_NAME
 
+    @property
+    def has_ami_name(self) -> bool:
+        return bool(self._params['amiName'])
+
     @property
     def ami_id(self) -> str:
         return self._params['amiId']
@@ -47,3 +51,7 @@ def root_volume_size(self) -> int:
     @property
     def max_price(self) -> float:
         return self._params['maxPrice']
+
+    @property
+    def managed_policy_arns(self) -> list:
+        return self._params['managedPolicyArns']

spotty/providers/aws/config/validation.py

@@ -26,6 +26,7 @@ def validate_instance_parameters(params: dict):
                                              And(lambda x: x > 0, error='"maxPrice" should be greater than 0 or '
                                                                         'should  not be specified.'),
                                              ),
+        Optional('managedPolicyArns', default=[]): [str],
     }
 
     volumes_checks = [

spotty/providers/aws/deployment/abstract_aws_deployment.py

@@ -1,6 +1,5 @@
 from abc import ABC, abstractmethod
 import boto3
-from spotty.providers.aws.aws_resources.image import Image
 from spotty.providers.aws.aws_resources.subnet import Subnet
 from spotty.providers.aws.aws_resources.vpc import Vpc
 from spotty.providers.aws.config.instance_config import InstanceConfig
@@ -40,11 +39,3 @@ def get_vpc_id(self) -> str:
             vpc_id = default_vpc.vpc_id
 
         return vpc_id
-
-    def get_ami(self) -> Image:
-        if self.instance_config.ami_id:
-            image = Image.get_by_id(self._ec2, self.instance_config.ami_id)
-        else:
-            image = Image.get_by_name(self._ec2, self.instance_config.ami_name)
-
-        return image

spotty/providers/aws/deployment/ami_deployment.py

@@ -1,4 +1,5 @@
 from spotty.commands.writers.abstract_output_writrer import AbstractOutputWriter
+from spotty.providers.aws.aws_resources.image import Image
 from spotty.providers.aws.deployment.abstract_aws_deployment import AbstractAwsDeployment
 from spotty.providers.aws.aws_resources.instance import Instance
 from spotty.providers.aws.deployment.cf_templates.ami_template import prepare_ami_template
@@ -31,9 +32,9 @@ def deploy(self, debug_mode: bool, output: AbstractOutputWriter):
             raise ValueError('"%s" is not a GPU instance' % instance_type)
 
         # check that an image with this name doesn't exist yet
-        ami = self.get_ami()
+        ami = Image.get_by_name(self._ec2, self.instance_config.ami_name)
         if ami:
-            raise ValueError('AMI with the name "%s" already exists.' % ami.name)
+            raise ValueError('AMI with the name "%s" already exists.' % self.instance_config.ami_name)
 
         # check availability zone and subnet
         check_az_and_subnet(self._ec2, self.instance_config.region, self.instance_config.availability_zone,
@@ -77,7 +78,7 @@ def delete(self, output: AbstractOutputWriter):
         stack_id = None
         if not self.stack.get_stack():
             # try to get the stack ID from the AMI tags (for older versions of Spotty)
-            ami = self.get_ami()
+            ami = Image.get_by_name(self._ec2, self.instance_config.ami_name)
             if not ami:
                 raise ValueError('AMI with the name "%s" not found.' % self.instance_config.ami_name)
 

spotty/providers/aws/deployment/cf_templates/data/instance.yaml

@@ -106,12 +106,16 @@ Resources:
           'Fn::Base64': !Sub |
             #!/bin/bash -x
 
-            # install CloudFormation tools
-            apt-get update
-            apt-get install -y python-setuptools
-            mkdir -p aws-cfn-bootstrap-latest
-            curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1
-            easy_install aws-cfn-bootstrap-latest
+            cd /root
+
+            # install CloudFormation tools if they are not installed yet
+            if [ ! -e /usr/local/bin/cfn-init ]; then
+              apt-get update
+              apt-get install -y python-setuptools
+              mkdir -p aws-cfn-bootstrap-latest
+              curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1
+              python2 -m easy_install aws-cfn-bootstrap-latest
+            fi
 
             # prepare the instance and run Docker container
             /usr/local/bin/cfn-init \
@@ -134,6 +138,7 @@ Resources:
             - prepare_instance_config
             - mount_volumes_config
             - sync_project_config
+            - startup_commands_config
             - docker_container_config
         prepare_instance_config:
           files:
@@ -223,6 +228,16 @@ Resources:
           commands:
             sync_project:
               command: !Sub '/bin/bash -xe /tmp/spotty/instance/scripts/sync_project.sh ${SyncCommandArgs}'
+        startup_commands_config:
+          files:
+            /tmp/spotty/instance/scripts/startup_commands.sh:
+              owner: ubuntu
+              group: ubuntu
+              mode: '000644'
+              content: '# no content'
+          commands:
+            startup_commands:
+              command: !Sub '/bin/bash -xe /tmp/spotty/instance/scripts/startup_commands.sh'
         docker_container_config:
           files:
             /tmp/spotty/instance/scripts/run_container.sh:
@@ -288,7 +303,7 @@ Resources:
               owner: ubuntu
               group: ubuntu
               mode: '000644'
-              content: 'echo "Nothing to do"'
+              content: '# no content'
           commands:
             run_container:
               command: '/bin/bash -xe /tmp/spotty/instance/scripts/run_container.sh'

spotty/providers/aws/deployment/cf_templates/data/instance_profile.yaml

@@ -3,37 +3,41 @@ Resources:
   InstanceProfile:
     Type: AWS::IAM::InstanceProfile
     Properties:
-      Path: '/'
       Roles:
-      - Ref: InstanceRole
+        - Ref: InstanceRole
   InstanceRole:
     Type: AWS::IAM::Role
     Properties:
       AssumeRolePolicyDocument:
         Version: '2012-10-17'
         Statement:
-        - Effect: Allow
-          Principal:
-            Service:
-            - ec2.amazonaws.com
-          Action:
-          - sts:AssumeRole
-      Path: '/'
-      Policies:
-      - PolicyName: S3RolePolicy
-        PolicyDocument:
-          Version: '2012-10-17'
-          Statement:
           - Effect: Allow
+            Principal:
+              Service:
+                - ec2.amazonaws.com
             Action:
-            - s3:ListAllMyBuckets
-            - s3:GetBucketLocation
-            - s3:ListBucket
-            - s3:GetObject
-            - s3:PutObject
-            - s3:DeleteObject
-            Resource:
-            - arn:aws:s3:::*
+              - sts:AssumeRole
+      {{#HAS_MANAGED_POLICIES}}
+      ManagedPolicyArns:
+        {{#MANAGED_POLICY_ARNS}}
+        - {{MANAGED_POLICY_ARN}}
+        {{/MANAGED_POLICY_ARNS}}
+      {{/HAS_MANAGED_POLICIES}}
+      Policies:
+        - PolicyName: S3AccessPolicy
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - s3:ListAllMyBuckets
+                  - s3:GetBucketLocation
+                  - s3:ListBucket
+                  - s3:GetObject
+                  - s3:PutObject
+                  - s3:DeleteObject
+                Resource:
+                  - arn:aws:s3:::*
 Outputs:
   ProfileArn:
     Value: !GetAtt InstanceProfile.Arn

spotty/providers/aws/deployment/cf_templates/instance_profile_template.py

@@ -1,8 +1,16 @@
 import os
+import chevron
 
 
-def prepare_instance_profile_template():
+def prepare_instance_profile_template(managed_policy_arns: list):
     with open(os.path.join(os.path.dirname(__file__), 'data', 'instance_profile.yaml')) as f:
-        template = f.read()
+        content = f.read()
+
+    parameters = {
+        'HAS_MANAGED_POLICIES': len(managed_policy_arns),
+        'MANAGED_POLICY_ARNS': [{'MANAGED_POLICY_ARN': arn} for arn in managed_policy_arns]
+    }
+
+    template = chevron.render(content, parameters)
 
     return template

spotty/providers/aws/deployment/cf_templates/instance_template.py

@@ -72,6 +72,12 @@ def prepare_instance_template(instance_config: InstanceConfig, volumes: List[Abs
         output.write('- maximum Spot Instance price: %s'
                      % (('%.04f' % instance_config.max_price) if instance_config.max_price else 'on-demand'))
 
+    # set initial instance commands
+    if instance_config.commands:
+        template['Resources']['InstanceLaunchTemplate']['Metadata']['AWS::CloudFormation::Init'] \
+            ['startup_commands_config']['files']['/tmp/spotty/instance/scripts/startup_commands.sh']['content'] \
+            = instance_config.commands
+
     # set initial docker commands
     if container.config.commands:
         template['Resources']['InstanceLaunchTemplate']['Metadata']['AWS::CloudFormation::Init'] \