Run insights tests on ubuntu 16.04

Update to running docker containers that are built using the
configuration repo.  Run all tests on the container with the new code
cross mounted.

Remove the old shell scripts used to setup analytics-data-api and run
acceptance tests.

.travis.yml

@@ -1,35 +1,37 @@
 language:
   - python
-  - node_js
-python: "2.7"
-node_js: "5.2.0"
-sudo: false
+python:
+  - "2.7"
+sudo: required
+services:
+  - docker 
+
 env:
   # Make sure to update this string on every Insights or Data API release
-  - DATA_API_VERSION="0.17.0-rc.1"
+  DATA_API_VERSION: "0.17.0-rc.1"
+  DOCKER_COMPOSE_VERSION: "1.9.0"
+
 before_install:
-  - "export DISPLAY=:99.0"
-  - "sh -e /etc/init.d/xvfb start"
-addons:
-    firefox: "46.0"
-    apt:
-        packages:
-            - language-pack-en
-install:
-    - make develop
-    - make migrate
-    - ./scripts/install_analytics_data_api.sh $DATA_API_VERSION
-    - pip install coveralls
+    # Instal a newer version of docker-compose
+    # Remove once dockers default is this version: https://docs.travis-ci.com/user/docker/#Using-Docker-Compose
+    - sudo rm /usr/local/bin/docker-compose
+    - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+    - chmod +x docker-compose
+    - sudo mv docker-compose /usr/local/bin
+
+    # Start up the relevant services
+    - docker-compose -f .travis/docker-compose-travis.yml up -d
+    - docker exec analytics_api make -C /edx/app/analytics_api/analytics_api test.requirements
+    - docker exec analytics_api make -C /edx/app/analytics_api/analytics_api travis
+
 script:
-    - make static_no_compress
-    - make validate_translations
-    - make validate
-    - make generate_fake_translations
-    - make accept
-    - make a11y
+    - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_tests.sh
+
 after_success:
-    - coveralls
-    - bash ./scripts/build-stats-to-datadog.sh
+    - pip install -U codecov
+    - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_coverage.sh
+    - codecov
+
 deploy:
     - provider: s3
       access_key_id: $S3_ACCESS_KEY_ID

.travis/docker-compose-travis.yml

@@ -0,0 +1,37 @@
+version: "2.1"
+
+services:
+  es:
+    image: elasticsearch:1.5.2
+  analytics_api:
+    image: edxops/analytics_api:${DATA_API_VERSION:-latest}
+    container_name: analytics_api
+    environment:
+      ELASTICSEARCH_LEARNERS_HOST: 'http://es:9200/'
+      ELASTICSEARCH_LEARNERS_INDEX: 'learner'
+      ELASTICSEARCH_LEARNERS_UPDATE_INDEX: 'index_update'
+    command: /edx/app/analytics_api/venvs/analytics_api/bin/python /edx/app/analytics_api/analytics_api/manage.py runserver 0.0.0.0:80
+  insights:
+    image: edxops/insights:latest
+    container_name: insights_testing
+    volumes:
+      - ..:/edx/app/insights/edx_analytics_dashboard
+    command: tail -f /dev/null
+    environment:
+      # Pull these from the host environment.
+      TRAVIS_BRANCH:
+      TRAVIS_PULL_REQUEST:
+      DATADOG_API_KEY:
+      # Rest of the environment variables for testing.
+      API_SERVER_URL: http://analytics_api/api/v0
+      API_AUTH_TOKEN: edx
+      LMS_HOSTNAME: lms
+      LMS_PASSWORD: pass
+      LMS_USERNAME: user
+      ENABLE_AUTO_AUTH: "True"
+      ENABLE_OAUTH_TESTS: "False"
+      ENABLE_ERROR_PAGE_TESTS: "False"
+      DISPLAY_LEARNER_ANALYTICS: "True"
+    depends_on:
+      - "es"
+      - "analytics_api"

.travis/run_coverage.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -xe
+. /edx/app/insights/venvs/insights/bin/activate
+. /edx/app/insights/nodeenvs/insights/bin/activate
+
+cd /edx/app/insights/insights
+
+coverage xml
+
+bash ./scripts/build-stats-to-datadog.sh

.travis/run_tests.sh

@@ -0,0 +1,38 @@
+#!/bin/bash -xe
+. /edx/app/insights/venvs/insights/bin/activate
+. /edx/app/insights/nodeenvs/insights/bin/activate
+
+apt update
+apt install -y xvfb language-pack-en firefox # gettext
+
+# Need firefox 46 specifically, later versions don't work with Karma(frontend testing library).
+curl -O https://ftp.mozilla.org/pub/firefox/releases/46.0/linux-x86_64/en-US/firefox-46.0.tar.bz2
+tar xvf firefox-46.0.tar.bz2
+mv -f firefox /opt
+mv -f /usr/bin/firefox /usr/bin/firefox_default
+ln -s /opt/firefox/firefox /usr/bin/firefox
+
+cd /edx/app/insights/edx_analytics_dashboard
+export PATH=$PATH:$PWD/node_modules/.bin
+
+# Make it so bower can run without sudo.
+# https://github.com/GeoNode/geonode/pull/1070
+echo '{ "allow_root": true }' > /root/.bowerrc
+
+make develop
+make migrate
+
+# Compile assets and run validation
+make static_no_compress
+make validate_translations
+make validate
+make generate_fake_translations
+
+# The following tests need insights running. We have to do it here
+# because we can't run the production version without rsyslog
+# and we can't run `runserver` until after the development requiremnts
+# have been installed(This is not generally true but insights needs nose
+# to startup runserver.)
+/edx/bin/python.insights /edx/bin/manage.insights runserver 0.0.0.0:9000 --noreload --traceback > dashboard.log 2>&1 &
+xvfb-run make accept
+xvfb-run make a11y

Makefile

@@ -35,21 +35,27 @@ test_python_no_compress: clean
 	--cover-package=analytics_dashboard --cover-package=common --cover-branches --cover-html --cover-html-dir=$(COVERAGE)/html/ \
 	--with-ignore-docstrings --cover-xml --cover-xml-file=$(COVERAGE)/coverage.xml
 
-test_compress:
+test_compress: static_no_compress
 	python manage.py compress --settings=analytics_dashboard.settings.test
 
 test_python: test_compress test_python_no_compress
 
 accept:
-	./scripts/runTests.sh acceptance_tests
+ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True")
+	./manage.py waffle_flag enable_learner_analytics on --create --everyone
+endif
+	nosetests -v acceptance_tests -e NUM_PROCESSES=1 --exclude-dir=acceptance_tests/course_validation
 
 # local acceptance tests are typically run with by passing in environment variables on the commandline
 # e.g. API_SERVER_URL="http://localhost:9001/api/v0" API_AUTH_TOKEN="edx" make accept_local
 accept_local:
 	nosetests -v acceptance_tests --exclude-dir=acceptance_tests/course_validation
 
 a11y:
-	BOKCHOY_A11Y_CUSTOM_RULES_FILE=./node_modules/edx-custom-a11y-rules/lib/custom_a11y_rules.js SELENIUM_BROWSER=phantomjs ./scripts/runTests.sh a11y_tests
+ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True")
+	./manage.py waffle_flag enable_learner_analytics on --create --everyone
+endif
+	BOKCHOY_A11Y_CUSTOM_RULES_FILE=./node_modules/edx-custom-a11y-rules/lib/custom_a11y_rules.js SELENIUM_BROWSER=phantomjs nosetests -v a11y_tests -e NUM_PROCESSES=1 --exclude-dir=acceptance_tests/course_validation
 
 course_validation:
 	python -m acceptance_tests.course_validation.generate_report

analytics_dashboard/settings/local.py

@@ -22,3 +22,7 @@
     }
 }
 ########## END DATABASE CONFIGURATION
+
+########## DATA API CONFIGURATION
+DATA_API_URL = os.getenv("API_SERVER_URL", DATA_API_URL)
+########## END DATA API CONFIGURATION

package.json

@@ -16,10 +16,10 @@
     "eslint-config-edx": "^1.2.0",
     "gulp": "^3.8.8",
     "gulp-eslint": "^2.0.0",
-    "gulp-karma": "0.0.4",
+    "gulp-karma": "0.0.5",
     "jasmine-core": "^2.4.1",
     "jscs": "^1.10.0",
-    "karma": "^0.12.16",
+    "karma": "^1.3.0",
     "karma-coverage": "^0.2.6",
     "karma-chrome-launcher": "^0.2.3",
     "karma-jasmine": "^0.3.6",