playbook.yml

---
- hosts: "{{ hosts_string }}"
  remote_user: root
  gather_facts: False
  pre_tasks:
  - name: Update apt packages
    raw: apt -y update
  - name: Install python if not present
    raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
    retries: 2
    delay: 2
  tasks:
  # Set environment variables on job executor
  - name: Add environment variables to .bash_profile
    shell: echo "export {{ item }}" >> ~/.bash_profile
    with_items:
    - DB_NAME={{ lookup('env', 'DB_NAME') }}
    - DB_PASS={{ lookup('env', 'DB_PASS') }}
    - DB_PORT={{ lookup('env', 'DB_PORT') }}
    - DB_USER={{ lookup('env', 'DB_USER') }} 
    - JM_USER={{ lookup('env', 'JM_USER') }}
    - JM_PASS={{ lookup('env', 'JM_PASS') }}
    - JOB_ID={{ job_id }}
    - JOB_MANAGER='http://206.81.5.140'
    - JOB_NAME={{ job_name }}
    - MAX_QUEUE_LEN=5000
    - PRE_REDUCTIONS_QUEUE_BASE='pre-reductions'
    - PRE_VECTORS_QUEUE_BASE='pre-vectors' 
    - RABBITMQ_LOCATION='localhost'
    - REDUCER_PREFETCH_COUNT=10 
    - REDUCTIONS_QUEUE_BASE='reductions' 
    - SPELL_PASS={{ lookup('env', 'SPELL_PASS') }}
    - SPELL_USERNAME={{ lookup('env', 'SPELL_USERNAME') }}
    - VECTORIZER_PREFETCH_COUNT=10
    - VECTORS_QUEUE_BASE='vectors'
    - WRITER_PREFETCH_COUNT=100 
  # General system config
  - name: Install python
    apt:
      name: python3
      update_cache: yes
  - name: Update cache and install pip3 
    apt:
      name: python3-pip
      update_cache: yes
  - name: Install psql
    apt: name={{item}} state=present
    with_items:
    - postgresql-client-common
    - postgresql-client
  - name: Install virtualenv
    pip:
      name: virtualenv
      executable: pip3
  - name: Install python-dev (psycopg2 requirement)
    apt:
       name: python-dev
  - name: Install libpq-dev (psycopg2 requirement)
    apt:
      name: libpq-dev
  - name: Create jobs folder
    file: path=/var/lib/jobs state=directory
  - name: Clone job repository to remote host
    git:
      repo: '{{ repo }}'
      dest: /var/lib/jobs/{{ job_name }}
      version: '{{ job_hash }}'
  # Networking (communication w nlp-job-manager)
  - name: Copy private key to remote servers to allow communication with this host 
    copy:
      src: /root/sharedkeys/id_ecdsa
      dest: /root/.ssh/id_ecdsa
      owner: root
      group: root
      mode: 0600
  - name: Copy public key to remote servers to allow communication with this host 
    copy:
      src: /root/sharedkeys/id_ecdsa.pub
      dest: /root/.ssh/id_ecdsa.pub
      owner: root
      group: root
      mode: 0644
  - name: Turn off strict host checking (allows autossh)
    shell: echo "{{ item }}" >> ~/.ssh/config
    with_items:
    - 'Host *'
    - '  StrictHostKeyChecking no'
  - name: Add job manager master server to known hosts 
    local_action: shell ssh-keyscan {{ inventory_hostname }} >> ~/.ssh/known_hosts
  - name: Install autossh to create long-running ssh tunnels
    apt:
      name: autossh
  # ELK stack logging
  - name: Create important directories for storing SSL Certificate and logs 
    file:
      path: "{{ item }}"
      state: directory
    with_items:
      - /etc/pki/tls/certs
      - /var/log/jobrunnerlogs
      - /var/log/reducerlogs
      - /var/log/vectorizerlogs
      - /var/log/systemmonitorlogs
      - /var/log/trainerlogs
  - name: Copy the SSL Certificate for the ELK server (nlp job log manager) into place
    copy:
      src: /etc/pki/tls/certs/logstash-forwarder.crt
      dest: /etc/pki/tls/certs/logstash-forwarder.crt
  - name: Install filebeat package, create the Beats source list
    shell: echo "deb https://packages.elastic.co/beats/apt stable main" |  tee -a /etc/apt/sources.list.d/beats.list
  - name: Install filebeat package, add elastic search GPG key
    shell: wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add - 
  - name: Install filebeat package, finish install
    apt:
      name: filebeat
      update_cache: yes
  - name: Configure filebeat
    copy:
      src: /var/lib/jobs/{{ job_name }}/logging/filebeat.yml
      dest: /etc/filebeat/filebeat.yml
      remote_src: yes
  - name: Restart filebeat
    systemd:
      state: restarted
      enabled: yes
      name: filebeat
  - name: Restart filebeat (2)
    shell: update-rc.d filebeat defaults 95 10
  # Prepare system monitor to run
  - name: Install system monitor requirements 
    pip:
      requirements: /var/lib/jobs/{{ job_name }}/requirements.txt
      virtualenv: /var/lib/jobs/{{ job_name }}/venv
      virtualenv_python: python3
  # Prepare vectorizer and reducer and start job
  - name: Install reducer requirements
    pip:
      requirements: /var/lib/jobs/{{ job_name }}/reducer/requirements.txt
      virtualenv: /var/lib/jobs/{{ job_name }}/reducer/venv
      virtualenv_python: python3
  - name: Install vectorizer requirements
    pip:
      requirements: /var/lib/jobs/{{ job_name }}/vectorizer/requirements.txt
      virtualenv: /var/lib/jobs/{{ job_name }}/vectorizer/venv
      virtualenv_python: python3
  - name: Install train requirements (installs spell)
    pip:
      requirements: /var/lib/jobs/{{ job_name }}/train/requirements.txt
      virtualenv: /var/lib/jobs/{{ job_name }}/train/venv
      virtualenv_python: python3
  - name: Login to spell
    shell: /var/lib/jobs/{{ job_name }}/train/venv/bin/spell login --identity {{ lookup('env', 'SPELL_USERNAME') }} --password {{ lookup('env', 'SPELL_PASS') }}
  - name: Change the working directory, then execute start script
    shell: nohup ./start.sh > /var/log/jobrunnerlogs/{{ job_name }}.log 2>&1 </dev/null &
    args:
      chdir: /var/lib/jobs/{{ job_name }}/
      executable: /bin/bash
...