use db cache in jenkins bok choy builds

move migration calculation to reset test-db script

pavelib/database.py

@@ -22,7 +22,7 @@
     'bok_choy_schema_student_module_history.sql'
 ]
 
-# Output files from scripts/calculate-bokchoy-migrations.sh
+# Output files from scripts/reset-test-db.sh --calculate_migrations
 MIGRATION_OUTPUT_FILES = [
     'bok_choy_default_migrations.yaml',
     'bok_choy_student_module_history_migrations.yaml'

pavelib/paver_tests/test_database.py

@@ -114,7 +114,7 @@ def test_load_data_from_local_cache(self, _mock_sh):
             # Make sure that the local cache files are used - NOT downloaded from s3
             self.assertFalse(_mock_get_file.called)
         calls = [
-            call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)),
+            call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
             call('{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT))
         ]
         _mock_sh.assert_has_calls(calls)
@@ -155,7 +155,7 @@ def test_load_data_from_s3_fingerprint(self, _mock_sh):
                 'moto_test_bucket', self.fingerprint_filename, db_utils.CACHE_FOLDER
             )
         calls = [
-            call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)),
+            call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
             call('{}/scripts/reset-test-db.sh'.format(Env.REPO_ROOT))
         ]
         _mock_sh.assert_has_calls(calls)
@@ -183,7 +183,7 @@ def test_load_data_and_run_migrations(self, _mock_sh):
 
         database.update_local_bokchoy_db_from_s3()
         calls = [
-            call('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT)),
+            call('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT)),
             call('{}/scripts/reset-test-db.sh --rebuild_cache'.format(Env.REPO_ROOT))
         ]
         _mock_sh.assert_has_calls(calls)

pavelib/utils/db_utils.py

@@ -110,7 +110,7 @@ def calculate_bokchoy_migrations(migration_output_files):
     NOTE: the script first clears out the database, then calculates
           what migrations need to be run, which is all of them.
     """
-    sh('{}/scripts/calculate-bokchoy-migrations.sh'.format(Env.REPO_ROOT))
+    sh('{}/scripts/reset-test-db.sh --calculate_migrations'.format(Env.REPO_ROOT))
     verify_files_exist(migration_output_files)
 
 

pavelib/utils/test/suites/bokchoy_suite.py

@@ -21,6 +21,7 @@
 )
 from pavelib.utils.test import utils as test_utils
 from pavelib.utils.timer import timed
+from pavelib.database import update_local_bokchoy_db_from_s3
 
 import os
 
@@ -135,8 +136,15 @@ def get_test_course(options):
 def reset_test_database():
     """
     Reset the database used by the bokchoy tests.
+
+    If the tests are being run on Jenkins, use the database cache automation
+    defined in pavelib/database.py
+    If not, reset the test database and apply migrations
     """
-    sh("{}/scripts/reset-test-db.sh --migrations".format(Env.REPO_ROOT))
+    if os.environ.get('USER', None) == 'jenkins':
+        update_local_bokchoy_db_from_s3()
+    else:
+        sh("{}/scripts/reset-test-db.sh --migrations".format(Env.REPO_ROOT))
 
 
 @task

scripts/reset-test-db.sh

@@ -43,6 +43,9 @@ for i in "$@"; do
         -m|--migrations)
             APPLY_MIGRATIONS=true
             ;;
+        -c|--calculate_migrations)
+            CALCULATE_MIGRATIONS=true
+            ;;
     esac
 done
 
@@ -51,6 +54,14 @@ declare -a database_order
 databases=(["default"]="edxtest" ["student_module_history"]="student_module_history_test")
 database_order=("default" "student_module_history")
 
+calculate_migrations() {
+    echo "Calculating migrations for fingerprinting."
+    output_file="common/test/db_cache/bok_choy_${db}_migrations.yaml"
+    # Redirect stdout to /dev/null because the script will print
+    # out all migrations to both stdout and the output file.
+    ./manage.py lms --settings $SETTINGS show_unapplied_migrations --database $db --output_file $output_file 1>/dev/null
+}
+
 run_migrations() {
     echo "Running the lms migrations on the $db bok_choy DB."
     ./manage.py lms --settings $SETTINGS migrate --database $db --traceback --noinput
@@ -115,4 +126,8 @@ elif [[ $APPLY_MIGRATIONS ]]; then
     for db in "${database_order[@]}"; do
         run_migrations
     done
+elif [[ $CALCULATE_MIGRATIONS ]]; then
+    for db in "${database_order[@]}"; do
+        calculate_migrations
+    done
 fi