Merge pull request #13 from JCSDA/release/1.0.0

Release/1.0.0 (master)

.gitattributes

@@ -0,0 +1,3 @@
+*.nc filter=lfs diff=lfs merge=lfs -text
+*.nc4 filter=lfs diff=lfs merge=lfs -text
+*.odb filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+*DS_Store*
+._*
+*.swp

.travis.yml

@@ -0,0 +1,70 @@
+#======================================================================
+# Project settings
+#======================================================================
+branches:
+  only:
+    - develop
+
+language: cpp
+
+services:
+  - docker
+
+before_install:
+  - date
+  - REPO_SOURCE_DIR=${TRAVIS_BUILD_DIR}/jcsda/src_repo
+
+#  create the same file structure on Travis VM and Docker container
+  - mkdir -p ${TRAVIS_BUILD_DIR}/jcsda/src_repo
+# echo branch info
+  - echo $TRAVIS_PULL_REQUEST
+  - echo $TRAVIS_PULL_REQUEST_BRANCH
+  - echo $TRAVIS_BRANCH
+  - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo $TRAVIS_PULL_REQUEST_BRANCH; fi)
+  - echo "TRAVIS_BRANCH=$TRAVIS_BRANCH, PR=$PR, BRANCH=$BRANCH"
+
+#======================================================================
+# Clone all repos
+#======================================================================
+#  clone the testing branch of ioda
+  - git clone -b $BRANCH https://github.com/jcsda/ioda.git ${REPO_SOURCE_DIR}/ioda-bundle/ioda
+
+#  checkout for the same branch name in oops
+  - git clone https://github.com/jcsda/oops.git ${REPO_SOURCE_DIR}/ioda-bundle/oops
+  - cd ${REPO_SOURCE_DIR}/ioda-bundle/oops
+  - git checkout $BRANCH || echo "No branch named $BRANCH in oops repo"
+  - cd $REPO_SOURCE_DIR
+
+  - if [ "$BRANCH" = "develop" ];
+    then export BRANCH_FORK="release-stable";
+    else export BRANCH_FORK=$BRANCH;
+    fi
+
+#  checkout for the same branch name in fckit
+  - git clone https://github.com/jcsda-internal/fckit.git ${REPO_SOURCE_DIR}/ioda-bundle/fckit
+  - cd ${REPO_SOURCE_DIR}/ioda-bundle/fckit
+  - git checkout $BRANCH_FORK || echo "No branch named $BRANCH in atlas repo"; git checkout release-stable
+  - cd $REPO_SOURCE_DIR
+
+#  checkout for the same branch name in atlas
+  - git clone https://github.com/jcsda-internal/atlas.git ${REPO_SOURCE_DIR}/ioda-bundle/atlas
+  - cd ${REPO_SOURCE_DIR}/ioda-bundle/atlas
+  - git checkout $BRANCH_FORK || echo "No branch named $BRANCH in atlas repo"; git checkout release-stable
+  - cd $REPO_SOURCE_DIR
+
+#  move CMakeLists.txt
+  - cp ${TRAVIS_BUILD_DIR}/CI/CMakeLists.txt ${REPO_SOURCE_DIR}/ioda-bundle
+  - cp -r ${TRAVIS_BUILD_DIR}/CI/cmake ${REPO_SOURCE_DIR}/ioda-bundle/
+#======================================================================
+# Pull docker image
+#======================================================================
+  - docker pull jcsda/docker-clang-mpich-dev
+  - docker images
+
+script:
+  - docker run -d -t --name clang_container -v ${REPO_SOURCE_DIR}/ioda-bundle:/jcsda/src_repo jcsda/docker-clang-mpich-dev
+  - docker exec clang_container bash -c 'ls /jcsda/src_repo'
+  - docker exec clang_container bash -c 'mkdir /build_container'
+  - docker exec clang_container bash -c 'cd /build_container && ecbuild /jcsda/src_repo'
+  - docker exec clang_container bash -c 'cd /build_container/ioda && make -j4'
+  - docker exec clang_container bash -c 'cd /build_container/ioda && ctest --output-on-failure'

CI/CMakeLists.txt

@@ -0,0 +1,38 @@
+# (C) Copyright 2017 UCAR
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+#
+# IODA bundle
+#
+
+cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
+
+project( ioda-bundle LANGUAGES C CXX Fortran )
+
+find_package(ecbuild)
+set( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_CURRENT_SOURCE_DIR}/ecbuild/cmake;${CMAKE_MODULE_PATH}")
+include( ecbuild_bundle )
+
+set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
+
+ecbuild_bundle_initialize()
+
+option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON") #Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
+ecbuild_bundle( PROJECT fckit GIT "https://github.com/JCSDA-internal/fckit.git" )
+ecbuild_bundle( PROJECT atlas GIT "https://github.com/JCSDA-internal/atlas.git" )
+
+ecbuild_bundle( PROJECT oops         GIT "https://github.com/JCSDA/oops.git" )
+ecbuild_bundle( PROJECT ioda         GIT "https://github.com/JCSDA/ioda.git" )
+
+# Build Doxygen documentation
+option(BUILD_IODA_BUNDLE_DOC "Build documentation" OFF)
+if(BUILD_IODA_BUNDLE_DOC)
+   add_subdirectory( Documentation )
+endif(BUILD_IODA_BUNDLE_DOC)
+
+ecbuild_bundle_finalize()
+
+include(cmake/cdash-integration.cmake)
+include(CTest)

CI/Dockerfile

@@ -0,0 +1,35 @@
+FROM  jcsda/docker-gnu-openmpi-dev:latest
+
+RUN touch /env.txt
+RUN printenv > /env.txt
+
+RUN mkdir -p /var/run/sshd \
+    && ssh-keygen -A \
+    && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config
+
+RUN groupadd jcsda -g 9999
+RUN adduser jcsdauser
+
+RUN mkdir -p /jcsda /build_container \
+ &&  chown -R jcsdauser:jcsda /jcsda /build_container  /usr/local \
+ &&  chmod 6755 /jcsda /build_container /usr/local
+
+
+RUN mkdir /jcsda/.ssh ; echo "StrictHostKeyChecking no" > /jcsda/.ssh/config
+COPY default-mca-params.conf /jcsda/.openmpi/mca-params.conf
+RUN mkdir -p /jcsda/.openmpi
+RUN chown -R jcsdauser:jcsda /jcsda/
+
+USER jcsdauser
+WORKDIR /jcsda
+
+RUN ssh-keygen -f /jcsda/.ssh/id_rsa -t rsa -N '' \
+    && chmod 600 /jcsda/.ssh/config \
+    && chmod 700 /jcsda/.ssh \
+    && cp /jcsda/.ssh/id_rsa.pub /jcsda/.ssh/authorized_keys
+
+VOLUME /jcsda
+
+CMD ["/bin/bash"]

CI/buildspec_clang.yml

@@ -0,0 +1,49 @@
+version: 0.2
+
+env:
+  parameter-store:
+    GIT_USER: "/CodeBuild/Git_USER"
+    GIT_PASS: "/CodeBuild/Git_PASS"
+
+phases:
+  install:
+    commands:
+      - echo Executing install phase
+
+      - echo $CODEBUILD_RESOLVED_SOURCE_VERSION
+      - echo $CODEBUILD_SOURCE_REPO_URL
+      - echo $CODEBUILD_SOURCE_VERSION
+
+      - echo $CODEBUILD_WEBHOOK_MERGE_COMMIT
+      - echo $CODEBUILD_WEBHOOK_PREV_COMMIT
+      - echo $CODEBUILD_WEBHOOK_HEAD_REF
+      - echo $CODEBUILD_WEBHOOK_ACTOR_ACCOUNT_ID
+      - echo $CODEBUILD_WEBHOOK_EVENT
+      - echo $CODEBUILD_WEBHOOK_TRIGGER
+      - echo $CODEBUILD_WEBHOOK_BASE_REF
+
+      - ls
+      - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PULL_REQUEST_MERGED" ];
+        then export CODEBUILD_GIT_BRANCH="develop";
+        echo "Merging to develop";
+        else export CODEBUILD_GIT_BRANCH=${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/};
+        fi
+
+      - echo "CODEBUILD_GIT_BRANCH=${CODEBUILD_GIT_BRANCH}"
+      - echo "CODEBUILD_SOURCE_VERSION=${CODEBUILD_SOURCE_VERSION}"
+
+  pre_build:
+    commands:
+      - echo Executing pre_build phase
+      - git lfs install # creates .gitconfig
+      - cp ~/.gitconfig .
+
+
+      # get jedi-build-package
+      - git clone https://$GIT_USER:[email protected]/jcsda-internal/jedi-build-package
+      - cd jedi-build-package
+      - pip install --user -e .
+  build:
+    commands:
+      - ~/.local/bin/jedi-build --gh-token=$GIT_PASS -j 4 --env-id=docker-clang --branch-map atlas:release-stable fckit:release-stable -br ${CODEBUILD_GIT_BRANCH} -p ioda -vvv -3 -u --submit-dashboard --site AWS --abort-on-build-errors --abort-on-test-errors
+

CI/buildspec_gnu.yml

@@ -0,0 +1,131 @@
+version: 0.2
+
+env:
+  parameter-store:
+    GIT_USER: "/CodeBuild/Git_USER"
+    GIT_PASS: "/CodeBuild/Git_PASS"
+
+phases:
+  install:
+    commands:
+      - echo Executing install phase
+
+      - echo $CODEBUILD_RESOLVED_SOURCE_VERSION
+      - echo $CODEBUILD_SOURCE_REPO_URL
+      - echo $CODEBUILD_SOURCE_VERSION
+
+      - echo $CODEBUILD_WEBHOOK_MERGE_COMMIT
+      - echo $CODEBUILD_WEBHOOK_PREV_COMMIT
+      - echo $CODEBUILD_WEBHOOK_HEAD_REF
+      - echo $CODEBUILD_WEBHOOK_ACTOR_ACCOUNT_ID
+      - echo $CODEBUILD_WEBHOOK_EVENT
+      - echo $CODEBUILD_WEBHOOK_TRIGGER
+      - echo $CODEBUILD_WEBHOOK_BASE_REF
+
+      - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PULL_REQUEST_MERGED" ];
+        then export CODEBUILD_GIT_BRANCH="develop";
+        echo "Merging to develop";
+        else export CODEBUILD_GIT_BRANCH=${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/};
+        fi
+
+      - echo "CODEBUILD_GIT_BRANCH=${CODEBUILD_GIT_BRANCH}"
+      - echo "CODEBUILD_SOURCE_VERSION=${CODEBUILD_SOURCE_VERSION}"
+
+      - echo MPI setup for Docker
+      - mkdir -p /var/run/sshd
+      - ssh-keygen -A
+      - sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config
+      - sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config
+      - sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config
+
+      - groupadd jcsda -g 9999
+      - useradd jcsdauser
+      - mkdir -p /jcsda /build_container
+      - chown -R jcsdauser:jcsda /build_container  /usr/local
+      - chmod 6755 /jcsda /build_container /usr/local
+
+      - mkdir /jcsda/.ssh ; echo "StrictHostKeyChecking no" > /jcsda/.ssh/config
+      - mkdir -p /jcsda/.openmpi
+      - mkdir -p /home/jcsdauser/.openmpi
+
+      - cp CI/default-mca-params.conf /home/jcsdauser/.openmpi/mca-params.conf
+      - cat /home/jcsdauser/.openmpi/mca-params.conf
+      - chown -R jcsdauser:jcsda /jcsda/
+
+      - su - jcsdauser -c "ssh-keygen -f /jcsda/.ssh/id_rsa -t rsa -N ''
+        && chmod 600 /jcsda/.ssh/config
+        && chmod 700 /jcsda/.ssh
+        && cp /jcsda/.ssh/id_rsa.pub /jcsda/.ssh/authorized_keys
+        && echo MPI setup for Docker done"
+      - su - jcsdauser -c "echo $CC
+        && echo $CXX
+        && echo $FC
+        && whereis mpicc"
+
+  pre_build:
+    commands:
+      - echo Executing pre_build phase
+      - git lfs install # creates .gitconfig
+      - cp ~/.gitconfig /home/jcsdauser/
+
+      - if [ "$CODEBUILD_GIT_BRANCH" = "develop" ];
+        then export CODEBUILD_GIT_BRANCH_FORK="release-stable";
+        else export CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH};
+        echo "CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH_FORK}";
+        fi
+
+      - cd CI
+      # ioda
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda/ioda $CODEBUILD_GIT_BRANCH ioda /jcsda/ioda-bundle develop
+
+      # oops
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda/oops $CODEBUILD_GIT_BRANCH oops /jcsda/ioda-bundle develop
+
+
+      # internal repos
+      #fckit
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/ioda-bundle release-stable
+
+      # atlas
+      - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/ioda-bundle release-stable
+
+      # move CMakeLists.txt
+      - cp CMakeLists.txt /jcsda/ioda-bundle
+      - cp -r cmake /jcsda/ioda-bundle/
+
+      - cd /jcsda/ioda-bundle
+      - sed -i 's/BRANCH develop  UPDATE//' CMakeLists.txt
+      - sed -i 's/BRANCH develop//' CMakeLists.txt
+  build:
+    commands:
+      - echo Executing build phase
+      - echo $CODEBUILD_BUILD_SUCCEEDING
+      - su - jcsdauser -c "cd /build_container 
+        && echo $CC
+        && echo $CXX
+        && echo $FC
+        && CC=mpicc CXX=mpicxx FC=mpifort cmake -DCMAKE_MODULE_PATH=/usr/local/share/ecbuild/cmake/ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_GPROF=ON /jcsda/ioda-bundle/
+        && cd /build_container/ioda
+        && make -j4"
+
+      - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ];
+        then export BUILD_STATUS="1";
+        echo "Build passed";
+        fi
+      - echo $BUILD_STATUS
+      - su - jcsdauser -c "cd /build_container 
+        && CC=mpicc CXX=mpicxx FC=mpifort
+        && cd /build_container/ioda
+        && ctest"
+
+  post_build:
+    commands:
+      - echo Executing post_build phase
+      - echo $CODEBUILD_BUILD_SUCCEEDING
+      - if [ "$BUILD_STATUS" = "1" ] && [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ];
+        then echo "Build passed, rerunning failed tests";
+        su - jcsdauser -c "cd /build_container/ioda
+        && ctest -VV --rerun-failed"; 
+        fi
+      - cd /build_container/ioda
+      - bash /jcsda/ioda-bundle/ioda/CI/codecov_script.sh