Fix to link correct version of test data for minor releases (#328)

* first

* fix typo

test/CMakeLists.txt

@@ -114,13 +114,14 @@ elseif( DEFINED GIT_TAG_FUNC )
         ${IODA_DOWNLOAD_BASE_URL}
         ${CMAKE_SOURCE_DIR}/test-data-release
         ${ioda_test_data}
-        ${checksum} )
+        ${GIT_TAG_FUNC}
+        ioda)
   message( STATUS "Test data will be downloaded from: ${IODA_DOWNLOAD_BASE_URL}" )
 
   # Create test-data-release in source directory
   file( MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/test-data-release )
 
-  set ( IODA_DATA_DOWNLOADER ioda_data_downloader.py )
+  set ( IODA_DATA_DOWNLOADER ioda_dash_downloader.py.in )
 else()
   # Any branch of IODA is being built.
   # ioda-data repository is already cloned by bundle/CMakeLists.txt.

test/ioda_dash_downloader.py.in

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import stat
+import tarfile
+import urllib.request
+import shutil
+
+bucket_name = "jedi-test-files"
+
+download_base_url = sys.argv[1]
+testfiles_path = sys.argv[2]
+testfiles_name = sys.argv[3]
+repo_branch = sys.argv[4]
+repo_name = sys.argv[5]
+
+def DownloadUntar(download_base_url, testfiles_path, testfiles_name):
+  urllib.request.urlretrieve( download_base_url+"/"+testfiles_name, testfiles_path+"/"+testfiles_name)
+  tar_file = tarfile.open(testfiles_path+"/"+testfiles_name)
+  # untar dash data in a clean dash_data directory 
+  if os.path.exists(testfiles_path+"/dash_data"):
+    shutil.rmtree(testfiles_path+"/dash_data", ignore_errors=False, onerror=None)
+  tar_file.extractall(testfiles_path+"/dash_data")
+  tar_file.close()
+
+
+def CheckTag(testfiles_path, testfiles_name, repo_branch):
+  # determine test data tag version after untar
+  # if test data tag version is different from repo tag, link
+  data_branch = os.listdir(testfiles_path+"/dash_data/"+repo_name)
+  if data_branch[0] != repo_branch:
+    print("Latest tag available for "+repo_name+" test data is "+data_branch[0])
+    if not os.path.exists(testfiles_path+"/"+repo_name):
+      os.mkdir(testfiles_path+"/"+repo_name)
+
+    # removing the existing link and creating a new one 
+    if os.path.exists(testfiles_path+"/"+repo_name+"/"+repo_branch):
+      os.remove(testfiles_path+"/"+repo_name+"/"+repo_branch) 
+    os.symlink(testfiles_path+"/dash_data/"+repo_name+"/"+data_branch[0], testfiles_path+"/"+repo_name+"/"+repo_branch)
+
+# downloading release data from DASH
+if os.path.isfile(testfiles_path+"/"+testfiles_name):
+  print("local RELEASE file found in "+ testfiles_path+"/"+testfiles_name)
+else:
+  print ("downloading RELEASE data from "+download_base_url+"/"+testfiles_name)
+  DownloadUntar(download_base_url, testfiles_path, testfiles_name)
+  CheckTag(testfiles_path, testfiles_name, repo_branch)