diff --git a/.travis.yml b/.travis.yml index 3892e294..35cd6bba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,13 +6,11 @@ python: - 2.7 - 3.4 - 3.5 - - 3.6 - -env: - global: - - BATCH_SHIPYARD_CLI_ARTIFACT=batch-shipyard-$TRAVIS_TAG-cli-$TRAVIS_OS_NAME-x86_64 - - secure: "CsiojO04pOwYbCduqXcZfSsJZ01n0bc4hflza4tk9ZHLotcZkCRKGVsemdPVKZVNlzy+gUpzyc4Tc4/h0391piQ51CCTThde2csDWbXssoO71b4U/jIuKhOqMHyp+5dGznBXOcFye9d5gpdHF0z3AZbo9V7zkuO+3YD/ES1YLWJ5+6RRAR11FRofipfNFYQ7OPgPzrqI0IHHKOYG02JeN7ZmpX+fEitH/J561y9RMIhEtk2udBd51JSTIYao7IENEpHoD41CwwlpOLWKhr7oo9wmyOTdvwyNzLh6Z/CxBUMvmvKn4zsK2d/WzM77EwN4bcBwu2vJmfR+Hi5CD61L9KdhVM+rWUn8+OJUmiNo+ZYVM5w/cHdyC7BYe5hHDAEVmvg0dQIxFXTov5+PjAi0HjA2RNrI2i2ABX1N8wrtPCxlkL8Pnbb93DfTPWmYQVS6LsXrh77KuD7gRxu1s5KAmlfcG//bGuKvpDz6vm7+/cXxnB4sZX+Et9S/F1GV1VYOAr3YwLuE2Tl8MPyoUJa/PdK+md3klL9WteQWXJHLcLbnt34VvstF0jxGh/hqI/3lKtN1U5bsfRbGNBAvYGbGQKpVleF5/0yDLI5/kWI7bIuEF4FHYgIOSvIAzVdC+GXYAeXgrjlOHDFVKwOGL1lhvY3nA/fbENNhHeEN3RQc6N8=" - - secure: "XgD2xFqMvimEDzwRvUwNmA7EAl6Y/ydwgXXq+O4eEE09Xl/UQvmCZ4soTwFqOUnf798Xqn6R2qXHdqpl2l4IXMILa4LteNUzRKd0txOj2lPGftFs1s09Odu729PkoEs9qbM446yjvK+PhbDUvD9GboQ3sWbzYSUPPDiJuPWiJ5KasooGdyfahV542hBo2iOc0Ezq+sziBwBffY407XcX7mTmXjEtQuwSezJGWynHUSD7gIywCXAuzTqdK3ikYYdSJTUn6SnlZZlLuICoOzVULT97VzA+o+puFU/rSnG9+czUUcXILpDzjpl231b858sQ7RdLocPdTiyIVx06FOfW/XfKUyDbRHGhiVZzsJ4xDMxtQk9haLb5NJjwZGFWmyy2tHnj86yNwxwmocbiRymqC2sz/rcIpmwMneKIGJG9MuXRMlGqeMxCcDb3FoaoUp6JIflkx5qsmgsaTEs4m0tcdyivf6ugW1dpBIGek8sf/LISmdakZTNeJxGUyMNYRFdSRWTTZvQ4fj3ZVxLoTCHLszwwHZ1l3AstbiYk86/5Zn8AS+5cWnVEZP3ayPkWqVKMjLxQrHQsLKGgpJBjDXTUp4NixssgXWmDmJMuME0TEjoT58hhoWDQk9PdKnpiiOHyne5jAVliTwhsDcKYCaqt9F8AAAOyuRkWobU32YpIEnU=" +matrix: + include: + - python: 3.7 + dist: xenial + sudo: true before_install: - wget -c https://goo.gl/ZzKHFv -O - | tar -xvJ -C /tmp/ @@ -27,55 +25,5 @@ install: script: - flake8 --statistics shipyard.py convoy/*.py - - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then flake8 --statistics cascade/*.py cargo/*.py heimdall/*.py; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then flake8 --statistics cascade/*.py cargo/*.py heimdall/*.py; fi - shellcheck ./*.sh cargo/*.sh cascade/*.sh docker/*.sh scripts/*.sh - -after_success: -- | - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then - echo "Not building binary with PR $TRAVIS_PULL_REQUEST" - return - fi - if [[ "$TRAVIS_PYTHON_VERSION" != "3.6" ]]; then - echo "Not building binary without Python 3.6" - return - fi - if [[ ! -z "$TRAVIS_TAG" ]]; then - export UPLOAD_PATH="releases/$TRAVIS_TAG" - elif [[ "$TRAVIS_BRANCH" == "master" ]] || [[ "$TRAVIS_BRANCH" == "develop" ]]; then - export BATCH_SHIPYARD_CLI_ARTIFACT="batch-shipyard-${TRAVIS_BRANCH}-${TRAVIS_BUILD_NUMBER}-cli-$TRAVIS_OS_NAME-x86_64" - export UPLOAD_PATH="builds/$TRAVIS_BRANCH" - else - echo "Invalid tag=$TRAVIS_TAG or branch=$TRAVIS_BRANCH to build binary" - return - fi - echo "BATCH_SHIPYARD_CLI_ARTIFACT=$BATCH_SHIPYARD_CLI_ARTIFACT UPLOAD_PATH=$UPLOAD_PATH" - if [[ ! -e "bin/$BATCH_SHIPYARD_CLI_ARTIFACT" ]]; then - virtualenv -p python3 pyi - $SHELL -c "set -e; source pyi/bin/activate; \ - pip install pyinstaller; \ - pip install --upgrade -r requirements.txt; \ - pip install --upgrade --no-deps -r req_nodeps.txt; \ - pyinstaller -F -n $BATCH_SHIPYARD_CLI_ARTIFACT -p batch-shipyard --add-data schemas:schemas --add-data scripts:scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin shipyard.py; \ - deactivate" - chmod +x bin/$BATCH_SHIPYARD_CLI_ARTIFACT - sha256sum bin/$BATCH_SHIPYARD_CLI_ARTIFACT | cut -d' ' -f1 > bin/${BATCH_SHIPYARD_CLI_ARTIFACT}.sha256 - cat bin/${BATCH_SHIPYARD_CLI_ARTIFACT}.sha256 - travis_retry pip install --upgrade blobxfer - blobxfer upload --remote-path $UPLOAD_PATH --local-path bin/${BATCH_SHIPYARD_CLI_ARTIFACT} --strip-components 1 --file-md5 --overwrite - fi - -deploy: - - provider: releases - skip_cleanup: true - on: - tags: true - condition: "$TRAVIS_PYTHON_VERSION == 3.6" - draft: true - overwrite: true - tag_name: $TRAVIS_TAG - file: - - bin/${BATCH_SHIPYARD_CLI_ARTIFACT} - - bin/${BATCH_SHIPYARD_CLI_ARTIFACT}.sha256 - api_key: - secure: Mz2sWn3YAr4tenO1NflOR6QJ9xiy4bjQvnFl9HzZ4zifxrjwLyN/Nbms6X2l8lq7+HTIDOn4Nw8DsE60Upq7yG0UEkwv0BJJtl2qzPnWGOAymykjnTP5Vl5MlCcqEVtUmWL+Wl7WMid10eJLdxSo7TUHp/AW/NShJuKj4rqQsdOtWBpWHuE2rBVTYeuPpUVku74rGH813erxgX/BPzcSjWcloha3Zp25jglJQ4sjqtGrmdts3hXYI1/6oEvdnmFn2xePRMfrEhJxkHkGhqEukkehi4QMnxQubcv9Y9ETQTROf0QjtTeJ+h/HsK2wY5VdI4dcqe3b0/F0m5sZ2Z0PThzBoi/ga18OKeU8P/w1xWk2+DtvG9jGZ8yjbnfRLxOfEg8KNF8Y6B6KntkJWpoNBlIdDU9jOndnvdAjFu90KMmgerz53FtLw5DmZQxIUjoryRrFQMjnvR2dbaP8QF2LlbdlSPDXcGSlO4zVh9BxXFQJgJ9PYA9AQnVii32o1RL5+CelV59AKaheybtk2pqUVvF1mz+prPVbsIJOPo97YpaAfuNprqVvE8kfE9t5W7aOhMuqM+bS4LpENzqDsPIro2cJ2FROtFmB9Ts1YTt6P0UK0XY+m2v3URWjw8v7U4IqGPeNMdQUC6wEJgNxPH1Ko+T1ZMhQ9y94FWO2at0wF3Q= diff --git a/.vsts/pipeline.yml b/.vsts/pipeline.yml new file mode 100644 index 00000000..a5ea5389 --- /dev/null +++ b/.vsts/pipeline.yml @@ -0,0 +1,341 @@ +name: $(SourceBranch)$(Rev:.r) + +phases: + - phase: Windows + queue: + name: Hosted VS2017 + parallel: 1 + matrix: + Python36: + python.version: '3.6' + PYENV_VERSION: '3.6.6' + steps: + - task: InstallPython@1 + inputs: + version: 'python==$(PYENV_VERSION)' + - powershell: | + gci env:* | sort-object name + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install --no-deps -r req_nodeps.txt + pip install flake8 + pip list --format=columns --outdated + displayName: Fetch Dependencies + - powershell: | + flake8 --statistics shipyard.py convoy\*.py + if ($env:PYTHON_VERSION -eq "3.6") { + flake8 --statistics cascade\*.py cargo\*.py heimdall\*.py + } + displayName: Static Analysis + - powershell: | + $artifactCli = "batch-shipyard-" + $env:BUILD_SOURCEBRANCHNAME + "-" + $env:BUILD_BUILDID + "-cli-win-amd64.exe" + $artifactUploadPath = "builds/" + $env:BUILD_SOURCEBRANCHNAME + $buildverDotted = "0.0.0" + $bvt0,$bvt1,$bvt2 = $buildverDotted.split('.') + $buildverTuple = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:BUILD_BUILDID) + echo "##vso[task.setvariable variable=BUILDVER_DOTTED;]$buildverDotted" + echo "##vso[task.setvariable variable=BUILDVER_TUPLE;]$buildverTuple" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]$artifactCli" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]$artifactUploadPath" + displayName: Pre-build Environment (Branch) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6')) + - powershell: | + git tag -l --points-at $env:BUILD_SOURCEVERSION | Tee-Object -Variable gitTag + $gitTag -match "^([\d\.])+" + $buildverDotted = $matches[0] + if ($gitTag -match "([a-z]+\d+)") { + $nugetPkgVer = $buildverDotted + "-" + $matches[1] + } else { + $nugetPkgVer = $buildverDotted + } + $artifactCli = "batch-shipyard-" + $gitTag + "-cli-win-amd64.exe" + $artifactUploadPath = "releases/" + $gitTag + $bvt0,$bvt1,$bvt2 = $buildverDotted.split('.') + $buildverTuple = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:BUILD_BUILDID) + echo "##vso[task.setvariable variable=BUILDVER_DOTTED;]$buildverDotted" + echo "##vso[task.setvariable variable=BUILDVER_TUPLE;]$buildverTuple" + echo "##vso[task.setvariable variable=NUGET_PACKAGE_VERSION;]$nugetPkgVer" + echo "##vso[task.setvariable variable=GIT_TAG;]$gitTag" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]$artifactCli" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]$artifactUploadPath" + displayName: Pre-build Environment (Tagged Release) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6')) + - powershell: | + $artifactCliPath = "bin\\" + $env:ARTIFACT_CLI + echo "##vso[task.setvariable variable=ARTIFACT_CLI_PATH;]$artifactCliPath" + $branchGitSha1 = [string]::Format("{0}@{1}",$env:BUILD_SOURCEBRANCHNAME,$env:BUILD_SOURCEVERSION.Substring(0,7)) + echo "$env:ARTIFACT_CLI $env:BUILDVER_TUPLE $branchGitSha1" + $fileverInfo = "docker\\windows\\file_version_info.txt" + (Get-Content $fileverInfo).replace('{BUILDVER_TUPLE}', $env:BUILDVER_TUPLE) | Set-Content $fileverInfo + (Get-Content $fileverInfo).replace('{BUILDVER_DOTTED}', $env:BUILDVER_DOTTED) | Set-Content $fileverInfo + (Get-Content $fileverInfo).replace('{BRANCH_GITSHA1}', $branchGitSha1) | Set-Content $fileverInfo + (Get-Content $fileverInfo).replace('{EXE}', $env:ARTIFACT_CLI) | Set-Content $fileverInfo + pip install virtualenv + python -m virtualenv pyi + displayName: Pre-build + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - script: | + echo "ARTIFACT_CLI=%ARTIFACT_CLI% ARTIFACT_CLI_PATH=%ARTIFACT_CLI_PATH% ARTIFACT_UPLOAD_PATH=%ARTIFACT_UPLOAD_PATH%" + call pyi\\Scripts\\activate.bat + pip -V + pip install --no-cache-dir pyinstaller + pip install --upgrade -r requirements.txt + pip install --upgrade --no-deps -r req_nodeps.txt + pyinstaller -F -n "%ARTIFACT_CLI%" -p batch-shipyard --add-data schemas;schemas --add-data scripts;scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon docker\\windows\\azure.ico --version-file docker\\windows\\file_version_info.txt --distpath bin shipyard.py + blobxfer upload --storage-account "$(blobxfer.storageAccount)" --storage-account-key "$(blobxfer.storageAccountKey)" --remote-path "%ARTIFACT_UPLOAD_PATH%" --local-path "%ARTIFACT_CLI_PATH%" --strip-components 1 --file-md5 --overwrite + call pyi\\Scripts\\deactivate.bat + displayName: Build + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - powershell: | + Invoke-WebRequest "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" -OutFile "nuget.exe" + $env:GIT_TAG | Out-File site-extension\\version.txt -Force -NoNewline + Get-Content -Path site-extension\\version.txt + $nugetArtifact = 'BatchShipyard.' + $env:NUGET_PACKAGE_VERSION + '.nupkg' + echo "##vso[task.setvariable variable=NUGET_ARTIFACT;]$nugetArtifact" + .\\nuget.exe pack site-extension\\BatchShipyard.nuspec -Version $env:NUGET_PACKAGE_VERSION + [Reflection.Assembly]::LoadWithPartialName('System.IO.Compression.FileSystem') + $zip = [IO.Compression.ZipFile]::OpenRead($nugetArtifact) + foreach($entry in $zip.Entries) { + [string]::Format("{0}: {1} -> {2}", $entry.FullName, $entry.Length, $entry.CompressedLength) + } + $zip.Dispose() + move $nugetArtifact bin\\$nugetArtifact + dir bin + displayName: Nuget Pack + condition: and(succeeded(), ne(variables['NUGET_PACKAGE_VERSION'], '')) + - powershell: | + $versionTag = "bin\\version_tag.txt" + $env:GIT_TAG | Out-File $versionTag -Force -NoNewline + Get-Content -Path $versionTag + copy .vsts\\populate_tag.ps1 bin\\populate_tag.ps1 + dir bin + displayName: Generate Version Tag File + condition: and(succeeded(), ne(variables['GIT_TAG'], '')) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: bin + artifactName: BatchShipyard-Windows + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - powershell: | + docker version + docker login -u="$(docker.username)" -p="$(docker.password)" + $cargoImage = "alfpark/batch-shipyard:" + $env:GIT_TAG + "-cargo-windows" + $cliImage = "alfpark/batch-shipyard:" + $env:GIT_TAG + "-cli-windows" + pushd docker\\windows\\cargo + docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cargoImage . + docker push $cargoImage + popd + pushd docker\\windows\\cli + docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cliImage . + docker push $cliImage + popd + displayName: Docker build + condition: and(succeeded(), ne(variables['GIT_TAG'], '')) + + - phase: Linux + queue: + name: Hosted Linux Preview + parallel: 1 + matrix: + Python36: + python.version: '3.6' + PYENV_VERSION: '3.6.6' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: x64 + - script: | + set -e + set -o pipefail + env + curl -fSsL https://goo.gl/ZzKHFv | tar -xJvp -C /tmp/ + echo "##vso[task.prependpath]/tmp/shellcheck-latest" + if [[ "$PYENV_VERSION" == 3.* ]]; then + which python3 + python3 --version + echo "##vso[task.setvariable variable=PYTHON;]python3" + echo "##vso[task.setvariable variable=PIP;]pip3" + else + which python + python --version + echo "##vso[task.setvariable variable=PYTHON;]python" + echo "##vso[task.setvariable variable=PIP;]pip" + fi + displayName: Initialize Build + - script: | + set -e + set -o pipefail + echo "PATH=$PATH" + echo "PYTHON=$PYTHON PIP=$PIP" + which shellcheck + $PYTHON -m pip install --upgrade pip + $PIP install -r requirements.txt + $PIP install --no-deps -r req_nodeps.txt + $PIP install flake8 + $PIP list --format=columns --outdated + displayName: Fetch Dependencies + - script: | + set -e + set -o pipefail + flake8 --statistics shipyard.py convoy/*.py + if [[ "$PYENV_VERSION" == 3.6* ]]; then + flake8 --statistics cascade/*.py cargo/*.py heimdall/*.py + fi + shellcheck ./*.sh cargo/*.sh cascade/*.sh docker/*.sh scripts/*.sh + displayName: Static Analysis + - script: | + set -e + set -o pipefail + ARTIFACT_CLI="batch-shipyard-${BUILD_SOURCEBRANCHNAME}-${BUILD_BUILDID}-cli-linux-x86_64" + ARTIFACT_UPLOAD_PATH="builds/${BUILD_SOURCEBRANCHNAME}" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}" + echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH" + displayName: Pre-build Environment (Branch) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6')) + - script: | + GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION) + ARTIFACT_CLI="batch-shipyard-${GIT_TAG}-cli-linux-x86_64" + ARTIFACT_UPLOAD_PATH="releases/${GIT_TAG}" + echo "##vso[task.setvariable variable=GIT_TAG;]${GIT_TAG}" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}" + displayName: Pre-build Environment (Tagged Release) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6')) + - template: ./pyenv.yml + - script: | + set -e + set -o pipefail + echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH GIT_TAG=$GIT_TAG" + ARTIFACT_CLI_PATH="bin/${ARTIFACT_CLI}" + $PIP install virtualenv + $PYTHON -m virtualenv -p $PYTHON pyi + /bin/bash -c \ + "set -e; source pyi/bin/activate; \ + pip install pyinstaller; \ + pip install --upgrade -r requirements.txt; \ + pip install --upgrade --no-deps -r req_nodeps.txt; \ + pyinstaller -F -n ${ARTIFACT_CLI} -p batch-shipyard --add-data schemas:schemas --add-data scripts:scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin shipyard.py; \ + deactivate" + chmod +x ${ARTIFACT_CLI_PATH} + sha256sum ${ARTIFACT_CLI_PATH} | cut -d' ' -f1 > ${ARTIFACT_CLI_PATH}.sha256 + cat ${ARTIFACT_CLI_PATH}.sha256 + export BLOBXFER_STORAGE_ACCOUNT=$(blobxfer.storageAccount) + export BLOBXFER_STORAGE_ACCOUNT_KEY="$(blobxfer.storageAccountKey)" + blobxfer upload --remote-path ${ARTIFACT_UPLOAD_PATH} --local-path ${ARTIFACT_CLI_PATH} --strip-components 1 --file-md5 --file-attributes --overwrite + displayName: Build + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: bin + artifactName: BatchShipyard-Linux + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + + - phase: MacOS + queue: + name: Hosted macOS Preview + parallel: 1 + matrix: + Python36: + python.version: '3.6' + PYENV_VERSION: '3.6.6' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: x64 + - script: | + set -e + set -o pipefail + env + if [[ "$PYENV_VERSION" == 3.* ]]; then + which python3 + python3 --version + echo "##vso[task.setvariable variable=PYTHON;]python3" + echo "##vso[task.setvariable variable=PIP;]pip3" + else + which python + python --version + echo "##vso[task.setvariable variable=PYTHON;]python" + echo "##vso[task.setvariable variable=PIP;]pip" + fi + displayName: Initialize Build + - script: | + set -e + set -o pipefail + echo "PATH=$PATH" + echo "PYTHON=$PYTHON PIP=$PIP" + $PYTHON -m pip install --upgrade pip + $PIP install -r requirements.txt + $PIP install --no-deps -r req_nodeps.txt + $PIP install flake8 + $PIP list --format=columns --outdated + displayName: Fetch Dependencies + - script: | + set -e + set -o pipefail + flake8 --statistics shipyard.py convoy/*.py + if [[ "$PYENV_VERSION" == 3.6* ]]; then + flake8 --statistics cascade/*.py cargo/*.py heimdall/*.py + fi + displayName: Static Analysis + - script: | + set -e + set -o pipefail + ARTIFACT_CLI="batch-shipyard-${BUILD_SOURCEBRANCHNAME}-${BUILD_BUILDID}-cli-mac-x86_64" + ARTIFACT_UPLOAD_PATH="builds/${BUILD_SOURCEBRANCHNAME}" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}" + echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH" + displayName: Pre-build Environment (Branch) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + in(variables['Build.SourceBranchName'], 'master', 'develop'), eq(variables['python.version'], '3.6')) + - script: | + GIT_TAG=$(git tag -l --points-at $BUILD_SOURCEVERSION) + ARTIFACT_CLI="batch-shipyard-${GIT_TAG}-cli-mac-x86_64" + ARTIFACT_UPLOAD_PATH="releases/${GIT_TAG}" + echo "##vso[task.setvariable variable=GIT_TAG;]${GIT_TAG}" + echo "##vso[task.setvariable variable=ARTIFACT_CLI;]${ARTIFACT_CLI}" + echo "##vso[task.setvariable variable=ARTIFACT_UPLOAD_PATH;]${ARTIFACT_UPLOAD_PATH}" + displayName: Pre-build Environment (Tagged Release) + condition: > + and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'), + startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.6')) + - template: ./pyenv.yml + - script: | + set -e + set -o pipefail + echo "ARTIFACT_CLI=$ARTIFACT_CLI ARTIFACT_UPLOAD_PATH=$ARTIFACT_UPLOAD_PATH GIT_TAG=$GIT_TAG" + ARTIFACT_CLI_PATH="bin/${ARTIFACT_CLI}" + $PIP install virtualenv + $PYTHON -m virtualenv -p $PYTHON pyi + /bin/bash -c \ + "set -e; source pyi/bin/activate; \ + pip install pyinstaller; \ + pip install --upgrade -r requirements.txt; \ + pip install --upgrade --no-deps -r req_nodeps.txt; \ + pyinstaller -F -n ${ARTIFACT_CLI} -p batch-shipyard --add-data schemas:schemas --add-data scripts:scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin shipyard.py; \ + deactivate" + chmod +x ${ARTIFACT_CLI_PATH} + shasum -a 256 ${ARTIFACT_CLI_PATH} | cut -d' ' -f1 > ${ARTIFACT_CLI_PATH}.sha256 + cat ${ARTIFACT_CLI_PATH}.sha256 + export BLOBXFER_STORAGE_ACCOUNT=$(blobxfer.storageAccount) + export BLOBXFER_STORAGE_ACCOUNT_KEY="$(blobxfer.storageAccountKey)" + blobxfer upload --remote-path ${ARTIFACT_UPLOAD_PATH} --local-path ${ARTIFACT_CLI_PATH} --strip-components 1 --file-md5 --file-attributes --overwrite + displayName: Build + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: bin + artifactName: BatchShipyard-MacOS + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) diff --git a/.vsts/populate_tag.ps1 b/.vsts/populate_tag.ps1 new file mode 100644 index 00000000..62c52d20 --- /dev/null +++ b/.vsts/populate_tag.ps1 @@ -0,0 +1,7 @@ +try { + $buildver = Get-Content "version_tag.txt" + echo "##vso[task.setvariable variable=VERSION_TAG;]$buildver" + Write-Host "tag version: $buildver" +} catch { + Write-Host "version.txt file not found" +} diff --git a/.vsts/pyenv.yml b/.vsts/pyenv.yml new file mode 100644 index 00000000..0ac9c514 --- /dev/null +++ b/.vsts/pyenv.yml @@ -0,0 +1,20 @@ +steps: + - script: | + set -e + set -o pipefail + echo "ARTIFACT_CLI=$ARTIFACT_CLI" + if [ "${SYSTEM_PHASENAME}" == "Linux" ]; then + apt-get update + apt-get install -y libbz2-dev libreadline-dev libsqlite3-dev + fi + export PYENV_ROOT="$(Agent.WorkFolder)/.pyenv" + git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT + export PATH="$PYENV_ROOT/bin:$PATH" + pyenv --version + PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $PYENV_VERSION + pyenv global $PYENV_VERSION + echo $PATH + echo "##vso[task.prependpath]$PYENV_ROOT/bin" + echo "##vso[task.prependpath]$PYENV_ROOT/shims" + displayName: Install Python + condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], '')) diff --git a/README.md b/README.md index 943df111..96fe430b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Build Status](https://azurebatch.visualstudio.com/batch-shipyard/_apis/build/status/batch-shipyard-CI)](https://azurebatch.visualstudio.com/batch-shipyard/_build/latest?definitionId=11) [![Build Status](https://travis-ci.org/Azure/batch-shipyard.svg?branch=master)](https://travis-ci.org/Azure/batch-shipyard) [![Build status](https://ci.appveyor.com/api/projects/status/3a0j0gww57o6nkpw/branch/master?svg=true)](https://ci.appveyor.com/project/alfpark/batch-shipyard) [![Docker Pulls](https://img.shields.io/docker/pulls/alfpark/batch-shipyard.svg)](https://hub.docker.com/r/alfpark/batch-shipyard) diff --git a/appveyor.yml b/appveyor.yml index 01811b25..2e64574a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,21 +12,19 @@ environment: PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6" + PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + PYTHON_ARCH: "64" + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + PYTHON_ARCH: "64" + - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" - BLOBXFER_STORAGE_ACCOUNT_KEY: - secure: kc+BqqjCZH3FGmN+4ngtNa8p4p2CR93gTGB9ecXvAZBdeOC3aaQoAV1JSSxXemfYF1I8B8rWM6gp6YyFusqfkpH4KMTZAyI8pTzV9l8dDaKhTBfE21kYlXgsRpV+JUnj - BLOBXFER_STORAGE_ACCOUNT: - secure: Dlu9Wk8y1Mo4eGjx6cH6ex+35JEB2g7kpsK15bSZeAI= - DOCKER_USERNAME: - secure: S8n3Geq7JUkN7ZQKXo8CLg== - DOCKER_PASSWORD: - secure: PFjy27XjnoH8YI46VpXUi13ZJLa8G6EJ6YzR9P3ualY= - DOCKER_IMAGE_TAG_PREFIX: "alfpark/batch-shipyard:" init: -- choco install checksum - echo %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH% install: @@ -43,193 +41,6 @@ build: off test_script: - flake8 --statistics shipyard.py convoy\\*.py -- IF "%PYTHON_VERSION%"=="3.6" ( +- IF "%PYTHON_VERSION%"=="3.7" ( flake8 --statistics cascade\\*.py cargo\\*.py ) - -after_test: -- ps: >- - if ($env:APPVEYOR_REPO_TAG -eq "true") { - $env:APPVEYOR_REPO_TAG_NAME -match "^([\d\.])+" - $env:BUILDVER_DOTTED = $matches[0] - if ($env:APPVEYOR_REPO_TAG_NAME -match "([a-z]+\d+)") { - $env:NUGET_PACKAGE_VERSION = $env:BUILDVER_DOTTED + "-" + $matches[1] - } - else { - $env:NUGET_PACKAGE_VERSION = $env:BUILDVER_DOTTED - } - } - else { - $env:NUGET_PACKAGE_VERSION = $null - $env:BUILDVER_DOTTED = "0.0.0" - } - -- echo is pr %APPVEYOR_PULL_REQUEST_NUMBER% is commit tag %APPVEYOR_REPO_TAG% name %APPVEYOR_REPO_TAG_NAME% branch %APPVEYOR_REPO_BRANCH% nugetpkg %NUGET_PACKAGE_VERSION% -- ps: >- - if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) { - Write-Host "Build is from a PR, not creating binary" - return - } - if (!($env:PYTHON_VERSION -eq "3.6")) { - Write-Host "Python environment is not 3.6, not creating binary" - return - } - if ($env:APPVEYOR_REPO_TAG -eq "true") { - $env:BATCH_SHIPYARD_CLI_ARITFACT = "batch-shipyard-" + $env:APPVEYOR_REPO_TAG_NAME + "-cli-win-amd64.exe" - $env:UPLOAD_PATH="releases/" + $env:APPVEYOR_REPO_TAG_NAME - } - else { - if ($env:APPVEYOR_REPO_BRANCH -eq "master" -Or $env:APPVEYOR_REPO_BRANCH -eq "develop") { - $env:BATCH_SHIPYARD_CLI_ARITFACT = "batch-shipyard-" + $env:APPVEYOR_REPO_BRANCH + "-" + $env:APPVEYOR_BUILD_NUMBER + "-cli-win-amd64.exe" - $env:UPLOAD_PATH="builds/" + $env:APPVEYOR_REPO_BRANCH - } - else { - Write-Host "Invalid tag or branch $env:APPVEYOR_REPO_BRANCH to build binary" - return - } - } - - $bvt0,$bvt1,$bvt2 = $env:BUILDVER_DOTTED.split('.') - - $env:BUILDVER_TUPLE = [string]::Format("({0}, {1}, {2}, {3})",$bvt0,$bvt1,$bvt2,$env:APPVEYOR_BUILD_NUMBER) - - $env:BRANCH_GITSHA1 = [string]::Format("{0}@{1}",$env:APPVEYOR_REPO_BRANCH,$env:APPVEYOR_REPO_COMMIT.Substring(0,7)) - -- IF "%BATCH_SHIPYARD_CLI_ARITFACT%"=="" ( - echo "Batch Shipyard CLI artifact not defined" - ) ELSE ( - echo "Batch Shipyard CLI is %BATCH_SHIPYARD_CLI_ARITFACT% upload path %UPLOAD_PATH%" && - virtualenv -p "%PYTHON%\\python.exe" pyi && - pyi\\Scripts\\activate.bat && - pip install pyinstaller && - pip install --upgrade -r requirements.txt && - pip install --upgrade --no-deps -r req_nodeps.txt && - sed -i -e "s/{BUILDVER_TUPLE}/%BUILDVER_TUPLE%/g" docker\\windows\\file_version_info.txt && - sed -i -e "s/{BUILDVER_DOTTED}/%BUILDVER_DOTTED%/g" docker\\windows\\file_version_info.txt && - sed -i -e "s/{BRANCH_GITSHA1}/%BRANCH_GITSHA1%/g" docker\\windows\\file_version_info.txt && - sed -i -e "s/{EXE}/%BATCH_SHIPYARD_CLI_ARITFACT%/g" docker\\windows\\file_version_info.txt && - pyinstaller -F -n "%BATCH_SHIPYARD_CLI_ARITFACT%" -p batch-shipyard --add-data schemas;schemas --add-data scripts;scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon docker\\windows\\azure.ico --version-file docker\\windows\\file_version_info.txt --distpath bin shipyard.py && - pyi\\Scripts\\deactivate.bat && - checksum "bin\\%BATCH_SHIPYARD_CLI_ARITFACT%" -t=sha256 > "bin\\%BATCH_SHIPYARD_CLI_ARITFACT%.sha256" && - cat "bin\\%BATCH_SHIPYARD_CLI_ARITFACT%.sha256" && - appveyor PushArtifact "bin\\%BATCH_SHIPYARD_CLI_ARITFACT%" && - appveyor PushArtifact "bin\\%BATCH_SHIPYARD_CLI_ARITFACT%.sha256" && - pip install --upgrade blobxfer && - blobxfer upload --remote-path %UPLOAD_PATH% --local-path bin\\%BATCH_SHIPYARD_CLI_ARITFACT% --strip-components 1 --file-md5 --overwrite - ) -- ps: >- - if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) { - Write-Host "Build is from a PR, not creating a Docker image for cargo" - return - } - if (!($env:PYTHON_VERSION -eq "3.6")) { - Write-Host "Python environment is not 3.6, not creating a Docker image for cargo" - return - } - - $DOCKER_IMAGE_TAG_SUFFIX = $null - - if ($env:APPVEYOR_REPO_TAG -eq "true") { - $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-cargo-windows' - } - if ($DOCKER_IMAGE_TAG_SUFFIX -eq $null) { - Write-Host "Image tag suffix is null, not creating a Docker image for cargo" - return - } - - $DOCKER_IMAGE_TAG = $env:DOCKER_IMAGE_TAG_PREFIX + $DOCKER_IMAGE_TAG_SUFFIX - - Write-Host "Creating Docker image for cargo with tag: $DOCKER_IMAGE_TAG" - - docker version - - pushd docker\\windows\\cargo - - docker build --build-arg GIT_BRANCH=$env:APPVEYOR_REPO_BRANCH --build-arg GIT_COMMIT=$env:APPVEYOR_REPO_COMMIT -t $DOCKER_IMAGE_TAG . - - docker login -u="$env:DOCKER_USERNAME" -p="$env:DOCKER_PASSWORD" - - docker push $DOCKER_IMAGE_TAG - - popd - -- ps: >- - if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) { - Write-Host "Build is from a PR, not creating a Docker image for CLI" - return - } - if (!($env:PYTHON_VERSION -eq "3.6")) { - Write-Host "Python environment is not 3.6, not creating a Docker image for CLI" - return - } - - $DOCKER_IMAGE_TAG_SUFFIX = $null - - if ($env:APPVEYOR_REPO_TAG -eq "true") { - $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-cli-windows' - } - if ($DOCKER_IMAGE_TAG_SUFFIX -eq $null) { - Write-Host "Image tag suffix is null, not creating a Docker image for CLI" - return - } - - $DOCKER_IMAGE_TAG = $env:DOCKER_IMAGE_TAG_PREFIX + $DOCKER_IMAGE_TAG_SUFFIX - - Write-Host "Creating Docker image for CLI with tag: $DOCKER_IMAGE_TAG" - - docker version - - pushd docker\\windows\\cli - - docker build --build-arg GIT_BRANCH=$env:APPVEYOR_REPO_BRANCH --build-arg GIT_COMMIT=$env:APPVEYOR_REPO_COMMIT -t $DOCKER_IMAGE_TAG . - - docker login -u="$env:DOCKER_USERNAME" -p="$env:DOCKER_PASSWORD" - - docker push $DOCKER_IMAGE_TAG - - popd - -- IF "%APPVEYOR_REPO_TAG%"=="true" IF "%PYTHON_VERSION%"=="3.6" ( - set UPLOAD_NUGET=1& - echo %APPVEYOR_REPO_TAG_NAME%> site-extension\\version.txt & - nuget pack site-extension\\BatchShipyard.nuspec -Version %NUGET_PACKAGE_VERSION% & - 7z l BatchShipyard.*.nupkg & - appveyor PushArtifact BatchShipyard.%NUGET_PACKAGE_VERSION%.nupkg || set UPLOAD_NUGET=0& - blobxfer upload --remote-path %UPLOAD_PATH% --local-path . --include BatchShipyard.*.nupkg --file-md5 --overwrite) - -deploy: -# TODO remove NuGet package generation and migrate to VSTS -#- provider: NuGet -# api_key: -# secure: Bl6flZ+gBJZ3aYPTKERDcnMOsNgbLs/MG/gj6XsCC9I9E/erIw0ffpWJ+WSQrWiX -# skip_symbols: true -# artifact: /.*\.nupkg/ -# on: -# appveyor_repo_tag: true -# PYTHON_VERSION: "3.6" -# UPLOAD_NUGET: "1" -- provider: GitHub - tag: $(APPVEYOR_REPO_TAG_NAME) - description: 'Batch Shipyard release' - auth_token: - secure: +f4N6Qsv3HvJyii0Bs+8qBx3YS7+7FJUWbFSiAdEIUDubFQnNkJgFnBw0Ew2SLkv - artifact: /.*\.nupkg/ - draft: true - prerelease: true - force_update: true - on: - appveyor_repo_tag: true - PYTHON_VERSION: "3.6" - UPLOAD_NUGET: "1" -- provider: GitHub - tag: $(APPVEYOR_REPO_TAG_NAME) - description: 'Batch Shipyard release' - auth_token: - secure: +f4N6Qsv3HvJyii0Bs+8qBx3YS7+7FJUWbFSiAdEIUDubFQnNkJgFnBw0Ew2SLkv - artifact: /.*\.exe.*/ - draft: true - prerelease: true - force_update: true - on: - appveyor_repo_tag: true - PYTHON_VERSION: "3.6" diff --git a/docs/25-batch-shipyard-platform-image-support.md b/docs/25-batch-shipyard-platform-image-support.md index f6dee360..51635224 100644 --- a/docs/25-batch-shipyard-platform-image-support.md +++ b/docs/25-batch-shipyard-platform-image-support.md @@ -57,8 +57,9 @@ SLES is not supported at this time. | MicrosoftWindowsServer | WindowsServerSemiAnnual | Datacenter-Core-1803-with-Containers-smalldisk | | | X | | ## Notes -1. IB/RDMA is supported for this host OS with a custom image. Please -see the [packer](../contrib/packer) scripts and consult the +1. IB/RDMA is supported for this host OS with a custom image unless +utilizing the native conversion option. Please see the +[packer](../contrib/packer) scripts and consult the [custom image guide](63-batch-shipyard-custom-images.md) for information on how to create a compliant custom image. 2. Native conversion of this platform image will enable IB/RDMA automatically.