.azure-pipelines.yml

# See https://aka.ms/yaml for documentation

trigger:
  branches:
    include:
    - '*'
  tags:
    include:
    - '*'

variables:
  python.version: '3.6'
  # pyinstaller is picky on the version to use
  python.version_for_macos_packaging: '3.8'
  postgresql.version: '10'
  winfsp.version: '1.7.20172'
  pytest.base_args: |
    --log-level=DEBUG \
    --durations=10 -v \
    --cov=parsec --cov-config=setup.cfg --cov-append --cov-report= \
  debug.stress_flaky_tests: 0  # Repeat all flaky tests X times

jobs:


#################################################################################
# Step 0: Wheel build


- job: 's0_build_wheel'
  displayName: 'Build wheel'
  timeoutInMinutes: 10
  pool:
    vmImage: 'ubuntu-latest'
  steps:
  - checkout: self
    submodules: true
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'
  - bash: |
      set -eux
      sudo apt-get update
      python --version
      pip install pip --user --upgrade
      pip install -r pre-requirements.txt
    displayName: 'Bootstrap'
  - bash: |
      set -eux
      python setup.py bdist_wheel
    displayName: 'Build Wheel'
  - publish: dist/
    artifact: wheel


#################################################################################
# Step 1: Tests


- job: 's1_qa'
  displayName: 'Q&A'
  timeoutInMinutes: 5
  pool:
    vmImage: 'ubuntu-latest'
  steps:
  - checkout: self
    submodules: true
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'
  - bash: |
      set -eux
      sudo apt-get update
      python --version
      pip install pre-commit
    displayName: 'Bootstrap'
  - bash: |
      set -eux
      # Patch .pre-commit-config.yaml to force mixed-line-ending checks to LF
      sed -i '/id: mixed-line-ending/a\      args:\n\        - "--fix=lf"' .pre-commit-config.yaml
      cat .pre-commit-config.yaml
      # Run all pre-commit hooks on all files
      pre-commit run --all-files --show-diff-on-failure
    displayName: 'Pre-commit hooks check'
  - bash: |
      if ([[ $(Build.SourceBranch) = "refs/heads/master" ]] || [[ $(Build.SourceBranch) = "refs/tags/"* ]])
      then exit 0; fi
      for FILENAME in newsfragments/*
      do
        # If file never existed in master, consider as a new newsfragment
        # Cannot git diff against master branch here given newsfragments removed in master will be considered as
        # new items in our branch
        if git log origin/master --exit-code -- "$FILENAME" > /dev/null
        then exit 0; fi
      done
      echo "NO NEW NEWSFRAGMENT FOUND" >&2
    displayName: 'Newsfragment'
    failOnStderr: true
  - bash: |
      git fetch --tags  # Needed by releaser.py to have a consistent `git describe`
      python ./misc/releaser.py check --verbose
    displayName: 'Version check'


#################################################################################


- job: 's1_linux_test'
  displayName: 'Linux tests'
  dependsOn: s0_build_wheel
  timeoutInMinutes: 60
  pool:
    vmImage: 'ubuntu-latest'
  variables:
    PGINSTALLATION: '/usr/lib/postgresql/$(postgresql.version)/bin'
  steps:
  - checkout: self
    submodules: true
  - download: current
    artifact: wheel
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'
  - bash: |
      set -eux
      sudo apt-get update
      python --version
      pip install pip --user --upgrade
      pip install -r pre-requirements.txt
      sudo apt-get install \
        postgresql-$(postgresql.version) \
        desktop-file-utils  # Provides `update-desktop-database` used by `tests/scripts/run_testenv.sh`
      psql --version
    displayName: 'Bootstrap'
  - bash: |
      set -eux
      # Remove the parsec folder from the source to force use of the wheel
      rm -rf parsec
      # Install the parsec wheel with all dependencies
      WHEEL=$(ls $(Pipeline.Workspace)/wheel/parsec_cloud-*.whl)
      echo $WHEEL
      pip install --use-feature=2020-resolver $WHEEL[all]
      # Check dependency compatibility
      pip check parsec[all]
    displayName: 'Install'
  - bash: |
      set -eux
      py.test $(pytest.base_args) \
        tests --runmountpoint --runslow \
        -n auto --max-worker-restart=0 -x \
        --junitxml=test-results-memory.xml
    displayName: 'Tests memory'
  - bash: |
      set -eux
      py.test $(pytest.base_args) \
        tests/backend tests/test_cli.py --postgresql --runslow \
        -n auto --max-worker-restart=0 -x \
        --junitxml=test-results-postgresql.xml
    displayName: 'Tests postgresql'
  # The full disk test are run separately since they rely on running a sqlite3 database
  # on a fuse mountpoint which can deadlock if run with the rest of the test suite
  # TEMPORARILY DISABLED BECAUSE THEY CAUSE ISSUES
  # - bash: |
  #     set -eux
  #     py.test $(pytest.base_args) \
  #       tests --rundiskfull --runslow -m diskfull -x \
  #       --junitxml=test-results-fulldisk.xml
  #   displayName: 'Tests full disk'
  # TODO: run gui tests with xvfb
  # - bash: |
  #     set -eux
  #     py.test $(pytest.base_args) \
  #       tests -m gui --runmountpoint --runslow --rungui -x \
  #       --junitxml=test-results-gui.xml
  #   displayName: 'Tests GUI'
  - bash: |
      set -eux
      pip install -U codecov
      coverage xml -o coverage.xml
      grep -E '( filename=|<package )' coverage.xml
      codecov --file=coverage.xml --commit=$(Build.SourceVersion)
    displayName: 'Codecov upload'
  - task: PublishTestResults@2
    inputs:
      testResultsFiles: 'test-results-*.xml'
      testRunTitle: 'Linux'
    condition: succeededOrFailed()


#################################################################################


- job: 's1_macos_test'
  displayName: 'MacOS tests'
  dependsOn: s0_build_wheel
  timeoutInMinutes: 60
  pool:
    vmImage: 'macOS-10.15'
  variables:
    PGINSTALLATION: '/usr/local/bin'
  steps:
  - checkout: self
    submodules: true
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'
  - download: current
    artifact: wheel
  - bash: |
      set -eux
      python --version
      pip install pip --user --upgrade
      pip install -r pre-requirements.txt
      brew install \
        postgres \
        desktop-file-utils  # Provides `update-desktop-database` used by `tests/scripts/run_testenv.sh`
      brew install --cask osxfuse
      psql --version
    displayName: 'Bootstrap'
  - bash: |
      set -eux
      # Remove the parsec folder from the source to force use of the wheel
      rm -rf parsec
      # Install the parsec wheel with all dependencies
      WHEEL=$(ls $(Pipeline.Workspace)/wheel/parsec_cloud-*.whl)
      echo $WHEEL
      pip install --use-feature=2020-resolver $WHEEL[all]
      # Check dependency compatibility
      pip check parsec[all]
    displayName: 'Install'
  # Postgresql tests almost working, but _run function in tests/test_cli.py gives
  # returncode of 1 instead of 0. Removed test_cli.py from this call as temporary fix
  # doing tests only on tests/backend/*
  # - bash: |
  #     set -eux
  #     py.test $(pytest.base_args) \
  #       tests/backend --postgresql --runslow \
  #       -n auto --max-worker-restart=0 -x \
  #       --junitxml=test-results-postgresql.xml
  #   displayName: 'Tests postgresql'
  - bash: |
      set -eux
      py.test $(pytest.base_args) \
        tests --runmountpoint --runslow -m mountpoint -x \
        --junitxml=test-results-mountpoint.xml
    displayName: 'Tests mountpoint'
  #- bash: |
  #    set -eux
  #    py.test $(pytest.base_args) \
  #      tests -m gui --runmountpoint --runslow --rungui -x \
  #      --junitxml=test-results-gui.xml
  #  displayName: 'Tests GUI'
  - bash: |
      set -eux
      pip install -U codecov
      coverage xml -o coverage.xml
      grep -E '( filename=|<package )' coverage.xml
      codecov --file=coverage.xml --commit=$(Build.SourceVersion)
    displayName: 'Codecov upload'
  - task: PublishTestResults@2
    inputs:
      testResultsFiles: 'test-results-*.xml'
      testRunTitle: 'macOS'
    condition: succeededOrFailed()


#################################################################################


- job: 's1_windows_test'
  displayName: 'Windows tests'
  dependsOn: s0_build_wheel
  timeoutInMinutes: 60
  pool:
    vmImage: 'windows-latest'
  variables:
    PGINSTALLATION: C:\\Program Files\\PostgreSQL\\$(postgresql.version)\\bin
  steps:
  - checkout: self
    submodules: true
  - download: current
    artifact: wheel
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'
      architecture: 'x64'
  - bash: |
      set -eux
      python.exe --version
      pip install pip --user --upgrade
      pip install -r pre-requirements.txt
      choco install -y --limit-output winfsp --pre --version=$(winfsp.version)
      # Install winfsp-test and put it in PATH
      mkdir winfsp-test
      curl -L https://github.com/billziss-gh/winfsp/releases/download/v1.7/winfsp-tests-1.7.20172.zip -o winfsp-test/winfsp-tests.zip
      unzip winfsp-test/winfsp-tests.zip -d winfsp-test
      echo "##vso[task.prependpath]$(Build.SourcesDirectory)\\winfsp-test"
    displayName: 'Bootstrap'
  - bash: |
      set -eux
      # Remove the parsec folder from the source to force use of the wheel
      rm -rf parsec
      # Install the parsec wheel with all dependencies
      WHEEL_DIRECTORY=`python -c 'print(r"$(Pipeline.Workspace)/wheel".replace("\\\\", "/"))'`
      WHEEL=$(ls $WHEEL_DIRECTORY/parsec_cloud-*.whl)
      echo $WHEEL
      pip install --use-feature=2020-resolver $WHEEL[all]
      # Check dependency compatibility
      pip check parsec[all]
      # Check winfsp-tests availability
      python.exe -c "import winfspy.tests.winfsp_tests"
    displayName: 'Install'
  - bash: |
      set -eux
      # Install pytest-repeat
      pip install pytest-repeat
      # Repeat flaky tests X times
      py.test tests --runmountpoint --runslow --rungui -m flaky --count $(debug.stress_flaky_tests) -v
    displayName: 'Debug: stress flaky tests'
    condition: and(succeeded(), gt(variables['debug.stress_flaky_tests'], 0))
  - bash: |
      set -eux
      py.test $(pytest.base_args) \
        tests -n auto --max-worker-restart=0 -x \
        --junitxml=test-results-fast.xml
    displayName: 'Tests fast'
  - bash: |
      set -eux
      py.test $(pytest.base_args) \
        tests --runmountpoint --runslow -m mountpoint -x \
        --junitxml=test-results-mountpoint.xml
    displayName: 'Tests mountpoint'
  - bash: |
      set -eux
      py.test $(pytest.base_args) \
        tests --runmountpoint --runslow --rungui -m gui -x \
        --junitxml=test-results-gui.xml
    displayName: 'Tests GUI'
  - bash: |
      set -eux
      py.test $(pytest.base_args) \
        tests --runslow -m slow \
        -n auto --max-worker-restart=0 -x \
        --junitxml=test-results-slow.xml
    displayName: 'Tests slow'
  - bash: |
      set -eux
      pip install -U codecov
      coverage xml -o coverage.xml
      grep -E '( filename=|<package )' coverage.xml
      codecov --file=coverage.xml --commit=$(Build.SourceVersion)
    displayName: 'Codecov upload'
  - task: PublishTestResults@2
    inputs:
      testResultsFiles: 'test-results-*.xml'
      testRunTitle: 'Windows'
    condition: succeededOrFailed()


#################################################################################
# Step 2: Release packages build


- job: 's2_linux_snap_build'
  displayName: 'Linux snap build'
  condition: startsWith(variables['build.sourceBranch'], 'refs/tags/')
  timeoutInMinutes: 60
  pool:
    # 18.04 requested to build a core18 image with --destructive-mode
    vmImage: 'ubuntu-18.04'
  steps:
  - checkout: self
    submodules: true
  - bash: |
      set -eux
      sudo apt-get update
      sudo snap install snapcraft --classic
      snapcraft --version
    displayName: 'Bootstrap'
  - bash: |
      set -eux
      cp -R $(Build.SourcesDirectory)/packaging/snap snapbuild
      cd snapbuild
      mkdir src
      cp -R $(Build.SourcesDirectory)/parsec src
      cp $(Build.SourcesDirectory)/setup.py src
      cp $(Build.SourcesDirectory)/README.rst src
      cp $(Build.SourcesDirectory)/HISTORY.rst src
      VERBOSE_TAG=`git -C $(Build.SourcesDirectory) describe --tag`
      sed -i "s#^\\(__version__ = \\).*\$#\\1'$VERBOSE_TAG'#" src/parsec/_version.py
      sed -i "s/__PARSEC_VERSION__/$VERBOSE_TAG/" snap/snapcraft.yaml
      snapcraft --destructive-mode
      mkdir dist
      cp parsec*.snap dist
    displayName: 'Build snap'
    workingDirectory: $(Agent.TempDirectory)
  - publish: $(Agent.TempDirectory)/snapbuild/dist
    artifact: linux_amd64_snap


#################################################################################


- job: 's2_macos_app_build'
  displayName: 'MacOS app build'
  condition: startsWith(variables['build.sourceBranch'], 'refs/tags/')
  timeoutInMinutes: 60
  pool:
    vmImage: 'macOS-10.15'
  steps:
  - checkout: self
    submodules: true
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version_for_macos_packaging)'
  - bash: |
      set -eux
      brew install coreutils
      sh macos_pyinstaller.sh
      tar -jcvf parsec-macos-amd64.tar.bz2 dist/Parsec.app
    displayName: 'Build'
    workingDirectory: $(Build.SourcesDirectory)/packaging/macOS
  - publish:  $(Build.SourcesDirectory)/packaging/macOS/parsec-macos-amd64.tar.bz2
    artifact: macos_amd64_app


#################################################################################


- job: 's2_windows_installer_build'
  displayName: 'Windows installer build'
  condition: startsWith(variables['build.sourceBranch'], 'refs/tags/')
  timeoutInMinutes: 60
  pool:
    vmImage: 'windows-latest'
  strategy:
    matrix:
      "64bits":
        python.arch: 'x64'
        vs.arch: 'amd64'
      "32bits":
        python.arch: 'x86'
        vs.arch: 'x86'
  steps:
  - checkout: self
    submodules: true
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'
      architecture: '$(python.arch)'
  - task: BatchScript@1
    inputs:
      filename: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat"'
      arguments: $(vs.arch)
      modifyEnvironment: true
  - bash: |
      set -eux
      pip install pip --user --upgrade
      # Path with backslash are a pain...
      SOURCES_DIRECTORY=`python -c 'print(r"$(Build.SourcesDirectory)".replace("\\\\", "/"))'`
      TEMP_DIRECTORY=`python -c 'print(r"$(Agent.TempDirectory)".replace("\\\\", "/"))'`
      VERBOSE_TAG=`git -C $SOURCES_DIRECTORY describe --tag`
      sed -i "s#^\\(__version__ = \\).*\$#\\1'$VERBOSE_TAG'#" $SOURCES_DIRECTORY/parsec/_version.py
      cp -R $SOURCES_DIRECTORY/packaging/win32/ winbuild
    displayName: 'Prepare build'
    workingDirectory: $(Agent.TempDirectory)
  # Must use cmd.exe here instead of bash otherwise `C:\Program Files\Git\bin\link.exe`
  # takes priority over visual studio link.exe in PATH
  - script: |
      python freeze_parsec.py $(Build.SourcesDirectory)
    displayName: 'Freeze Parsec'
    workingDirectory: $(Agent.TempDirectory)/winbuild
  - bash: |
      set -eux
      curl -L https://github.com/billziss-gh/winfsp/releases/download/v1.7/winfsp-1.7.20172.msi -o winfsp-1.7.20172.msi
      makensis.exe installer.nsi
      mkdir dist
      cp build/parsec*-setup.exe dist
    displayName: 'Build NSIS setup'
    workingDirectory: $(Agent.TempDirectory)/winbuild
  - publish: $(Agent.TempDirectory)/winbuild/dist
    artifact: win_$(vs.arch)_installer


#################################################################################
# Step 3: Release packages tests


- job: 's3_linux_snap_test'
  displayName: 'Never trust a snap'
  dependsOn: s2_linux_snap_build
  timeoutInMinutes: 10
  pool:
    vmImage: 'ubuntu-latest'
  steps:
  # Note there no point specifying Python version given snap doesn't take it into account
  - checkout: none
  - download: current
    artifact: linux_amd64_snap
  - bash: |
      set -eux
      sudo apt-get update
      # TODO: xvfb doesn't currently work
      # sudo apt-get install -y xvfb xorg
      sudo snap install --classic --dangerous $(Pipeline.Workspace)/linux_amd64_snap/parsec*.snap
      parsec.cli --version
      # Make sure snap contains core but not backend
      NOT_AVAILABLES=$(parsec.cli --help | grep -i "not available")
      test "$(echo $NOT_AVAILABLES | grep -i 'backend')"
      test -z "$(echo $NOT_AVAILABLES | grep -i 'core')"
      # xvfb-run parsec &
      # sleep 5
      # jobs  # returns error if parsec is not started
      # kill %1
    displayName: 'Test Snap'


#################################################################################
# Step 4: Release packages upload


- job: 's4_linux_snap_upload'
  displayName: 'Upload snap'
  condition: startsWith(variables['build.sourceBranch'], 'refs/tags/')
  dependsOn: [s0_build_wheel, s1_qa, s1_linux_test, s1_macos_test, s1_windows_test, s2_linux_snap_build, s2_macos_app_build, s2_windows_installer_build, s3_linux_snap_test]
  timeoutInMinutes: 10
  pool:
    vmImage: 'ubuntu-latest'
  steps:
  - checkout: none
  - download: current
    artifact: linux_amd64_snap
  - bash: |
      sudo apt-get update
      sudo snap install snapcraft --classic
      echo $SNAPCRAFT_CREDENTIALS | snapcraft login --with -
    displayName: 'Bootstrap'
    env:
      SNAPCRAFT_CREDENTIALS: $(SNAPCRAFT_CREDENTIALS)
  - bash: |
      snapcraft upload --release=edge $(Pipeline.Workspace)/linux_amd64_snap/parsec*.snap
    displayName: 'Upload Snap'


#################################################################################


- job: 's4_wheel_pypi_upload'
  displayName: 'Upload wheel on PyPI'
  condition: startsWith(variables['build.sourceBranch'], 'refs/tags/')
  dependsOn: [s0_build_wheel, s1_qa, s1_linux_test, s1_macos_test, s1_windows_test, s2_linux_snap_build, s2_macos_app_build, s2_windows_installer_build, s3_linux_snap_test]
  timeoutInMinutes: 10
  pool:
    vmImage: 'ubuntu-latest'
  steps:
  - checkout: none
  - download: current
    artifact: wheel
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'
  - bash: |
      set -eux
      sudo apt-get update
      pip install twine
    displayName: 'Bootstrap'
  - bash: |
      set -eux
      echo "
      [pypi]
      username = __token__
      password = $PYPI_CREDENTIALS
      " > ~/.pypirc
      WHEEL=$(ls $(Pipeline.Workspace)/wheel/parsec_cloud-*.whl)
      echo $WHEEL
      twine upload $WHEEL
    env:
      PYPI_CREDENTIALS: $(PYPI_CREDENTIALS)
    displayName: 'Upload to PyPI'