From 5bd372e548eb61d5f7b56a0b3d4cf0ca05e49e75 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Fri, 2 Jul 2021 15:02:40 -0600 Subject: [PATCH] Fix to link correct version of test data for minor releases (#328) * first * fix typo --- test/CMakeLists.txt | 5 +-- test/ioda_dash_downloader.py.in | 48 ++++++++++++++++++++++++++++ test/ioda_data_checker.py.in | 0 test/ioda_data_downloader.py.in | 55 --------------------------------- 4 files changed, 51 insertions(+), 57 deletions(-) create mode 100755 test/ioda_dash_downloader.py.in mode change 100644 => 100755 test/ioda_data_checker.py.in delete mode 100755 test/ioda_data_downloader.py.in diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 24687a79..ec1439d7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -114,13 +114,14 @@ elseif( DEFINED GIT_TAG_FUNC ) ${IODA_DOWNLOAD_BASE_URL} ${CMAKE_SOURCE_DIR}/test-data-release ${ioda_test_data} - ${checksum} ) + ${GIT_TAG_FUNC} + ioda) message( STATUS "Test data will be downloaded from: ${IODA_DOWNLOAD_BASE_URL}" ) # Create test-data-release in source directory file( MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/test-data-release ) - set ( IODA_DATA_DOWNLOADER ioda_data_downloader.py ) + set ( IODA_DATA_DOWNLOADER ioda_dash_downloader.py.in ) else() # Any branch of IODA is being built. # ioda-data repository is already cloned by bundle/CMakeLists.txt. diff --git a/test/ioda_dash_downloader.py.in b/test/ioda_dash_downloader.py.in new file mode 100755 index 00000000..0e5cf08a --- /dev/null +++ b/test/ioda_dash_downloader.py.in @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import os +import sys +import stat +import tarfile +import urllib.request +import shutil + +bucket_name = "jedi-test-files" + +download_base_url = sys.argv[1] +testfiles_path = sys.argv[2] +testfiles_name = sys.argv[3] +repo_branch = sys.argv[4] +repo_name = sys.argv[5] + +def DownloadUntar(download_base_url, testfiles_path, testfiles_name): + urllib.request.urlretrieve( download_base_url+"/"+testfiles_name, testfiles_path+"/"+testfiles_name) + tar_file = tarfile.open(testfiles_path+"/"+testfiles_name) + # untar dash data in a clean dash_data directory + if os.path.exists(testfiles_path+"/dash_data"): + shutil.rmtree(testfiles_path+"/dash_data", ignore_errors=False, onerror=None) + tar_file.extractall(testfiles_path+"/dash_data") + tar_file.close() + + +def CheckTag(testfiles_path, testfiles_name, repo_branch): + # determine test data tag version after untar + # if test data tag version is different from repo tag, link + data_branch = os.listdir(testfiles_path+"/dash_data/"+repo_name) + if data_branch[0] != repo_branch: + print("Latest tag available for "+repo_name+" test data is "+data_branch[0]) + if not os.path.exists(testfiles_path+"/"+repo_name): + os.mkdir(testfiles_path+"/"+repo_name) + + # removing the existing link and creating a new one + if os.path.exists(testfiles_path+"/"+repo_name+"/"+repo_branch): + os.remove(testfiles_path+"/"+repo_name+"/"+repo_branch) + os.symlink(testfiles_path+"/dash_data/"+repo_name+"/"+data_branch[0], testfiles_path+"/"+repo_name+"/"+repo_branch) + +# downloading release data from DASH +if os.path.isfile(testfiles_path+"/"+testfiles_name): + print("local RELEASE file found in "+ testfiles_path+"/"+testfiles_name) +else: + print ("downloading RELEASE data from "+download_base_url+"/"+testfiles_name) + DownloadUntar(download_base_url, testfiles_path, testfiles_name) + CheckTag(testfiles_path, testfiles_name, repo_branch) diff --git a/test/ioda_data_checker.py.in b/test/ioda_data_checker.py.in old mode 100644 new mode 100755 diff --git a/test/ioda_data_downloader.py.in b/test/ioda_data_downloader.py.in deleted file mode 100755 index fdc53961..00000000 --- a/test/ioda_data_downloader.py.in +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import stat -import tarfile -import urllib.request - -bucket_name = "jedi-test-files" - -download_base_url = sys.argv[1] -testfiles_path = sys.argv[2] -testfiles_name = sys.argv[3] -md5check = sys.argv[4] - -def DownloadUntar(download_base_url, testfiles_path, testfiles_name, md5check): - if (md5check == "1"): - urllib.request.urlretrieve( download_base_url+"/"+testfiles_name+".md5", testfiles_path+"/"+testfiles_name+".md5") - - urllib.request.urlretrieve( download_base_url+"/"+testfiles_name, testfiles_path+"/"+testfiles_name) - tar_file = tarfile.open(testfiles_path+"/"+testfiles_name) - tar_file.extractall(testfiles_path) - tar_file.close() - -if (md5check == "1") : - # if .tar.gz and .tar.gz.md5 exist - # then download s3 md5 - # and compare with local md5 - if os.path.isfile(testfiles_path+"/"+testfiles_name) and os.path.isfile(testfiles_path+"/"+testfiles_name+".md5") : - print("local files found") - - # dl md5 save it as *.md5.dl - urllib.request.urlretrieve( download_base_url+"/"+testfiles_name+".md5", testfiles_path+"/"+testfiles_name+".md5.dl") - - # compare *md5.dl with md5 local - with open(testfiles_path+"/"+testfiles_name+".md5", 'r') as f: - md5_local = f.read() - with open(testfiles_path+"/"+testfiles_name+".md5.dl", 'r') as f: - md5_dl = f.read() - if md5_local == md5_dl : - print("no update in dataset") - else: - print("update found; download new dataset") - DownloadUntar(download_base_url, testfiles_path, testfiles_name, md5check) - else: - print("local file not found; download from host") - print("downloading "+ download_base_url+"/"+testfiles_name) - DownloadUntar(download_base_url, testfiles_path, testfiles_name, md5check) -else: - # downloading release data from DASH - if os.path.isfile(testfiles_path+"/"+testfiles_name): - print("local RELEASE file found") - else: - print ("downloading RELEASE data from "+download_base_url+"/"+testfiles_name) - DownloadUntar(download_base_url, testfiles_path, testfiles_name, md5check)