CI: reorganize pipelines

[skip ci]
cctbx · Jun 12, 2020 · 33c0af4 · 33c0af4
1 parent b1dc8ab
commit 33c0af4
Show file tree

Hide file tree

Showing 12 changed files with 160 additions and 77 deletions.
diff --git a/.azure-pipelines/azure-pipelines-cache.yml → ...pipelines/azure-pipelines-build-cache.yml b/.azure-pipelines/azure-pipelines-cache.yml → ...pipelines/azure-pipelines-build-cache.yml
@@ -7,11 +7,18 @@ pr: none
 # Daily builds
 schedules:
 - cron: '0 1 * * *'
-  displayName: Daily rebuild of cache artifacts
+  displayName: Daily update of build cache artifacts
   branches:
     include:
     - master
 
+resources:
+  pipelines:
+  - pipeline: data_cache
+    source: 'Update data cache'
+  - pipeline: build_cache
+    source: 'Update build cache'
+
 stages:
 - stage: Preparation
   displayName: Download

diff --git a/.azure-pipelines/azure-pipelines-ci-branch.yml b/.azure-pipelines/azure-pipelines-ci-branch.yml
@@ -11,6 +11,13 @@ trigger:
     exclude:
       - master
 
+resources:
+  pipelines:
+  - pipeline: data_cache
+    source: 'Update data cache'
+  - pipeline: build_cache
+    source: 'Update build cache'
+
 stages:
 - stage: Preparation
   displayName: Download

diff --git a/...e-pipelines/azure-pipelines-artifacts.yml → ...-pipelines/azure-pipelines-data-cache.yml b/...e-pipelines/azure-pipelines-artifacts.yml → ...-pipelines/azure-pipelines-data-cache.yml
@@ -8,10 +8,10 @@
 trigger: none
 pr: none
 
-# update every other hour
+# update every day
 schedules:
-- cron: '30 */2 * * *'
-  displayName: Scheduled update for artifact data
+- cron: '0 0 * * *'
+  displayName: Daily update of data cache artifacts
   branches:
     include:
     - master
@@ -20,7 +20,7 @@ schedules:
 resources:
   pipelines:
   - pipeline: data_cache
-    source: 'Update data'
+    source: 'Update data cache'
 
 jobs:
 - job: artifacts
@@ -51,14 +51,19 @@ jobs:
     artifact: miniconda
 
   # chem_data
-  - download: data_cache
-    artifact: 'chem_data'
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: 'specific'
+      project: '$(resources.pipeline.data_cache.projectID)'
+      pipeline: '$(resources.pipeline.data_cache.pipelineID)'
+      allowPartiallySucceededBuilds: true
+      artifact: 'chem_data'
+      path: $(System.DefaultWorkingDirectory)/chem_data
     displayName: Download chem_data
+    continueOnError: true
 
   - script: |
-      if [ -d "$(Pipeline.Workspace)/data_cache/chem_data" ]; then
-        mv $(Pipeline.Workspace)/data_cache/chem_data $(System.DefaultWorkingDirectory)/chem_data
-      else
+      if [ ! -d "$(System.DefaultWorkingDirectory)/chem_data" ]; then
         svn co svn+ssh://[email protected]/chem_data/trunk chem_data
       fi
       cd $(System.DefaultWorkingDirectory)/chem_data
@@ -69,14 +74,19 @@ jobs:
     artifact: chem_data
 
   # phenix_regression
-  - download: data_cache
-    artifact: 'phenix_regression'
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: 'specific'
+      project: '$(resources.pipeline.data_cache.projectID)'
+      pipeline: '$(resources.pipeline.data_cache.pipelineID)'
+      allowPartiallySucceededBuilds: true
+      artifact: 'phenix_regression'
+      path: $(System.DefaultWorkingDirectory)/phenix_regression
     displayName: Download phenix_regression
+    continueOnError: true
 
   - script: |
-      if [ -d "$(Pipeline.Workspace)/data_cache/phenix_regression" ]; then
-        mv $(Pipeline.Workspace)/data_cache/phenix_regression $(System.DefaultWorkingDirectory)/phenix_regression
-      else
+      if [ ! -d "$(System.DefaultWorkingDirectory)/phenix_regression" ]; then
         git clone https://gitlab.com/phenix_project/phenix_regression.git
       fi
       cd $(System.DefaultWorkingDirectory)/phenix_regression
@@ -90,14 +100,19 @@ jobs:
     artifact: phenix_regression
 
   # phenix_examples
-  - download: data_cache
-    artifact: 'phenix_examples'
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: 'specific'
+      project: '$(resources.pipeline.data_cache.projectID)'
+      pipeline: '$(resources.pipeline.data_cache.pipelineID)'
+      allowPartiallySucceededBuilds: true
+      artifact: 'phenix_examples'
+      path: $(System.DefaultWorkingDirectory)/phenix_examples
     displayName: Download phenix_examples
+    continueOnError: true
 
   - script: |
-      if [ -d "$(Pipeline.Workspace)/data_cache/phenix_examples" ]; then
-        mv $(Pipeline.Workspace)/data_cache/phenix_examples $(System.DefaultWorkingDirectory)/phenix_examples
-      else
+      if [ ! -d "$(System.DefaultWorkingDirectory)/phenix_examples" ]; then
         git clone https://gitlab.com/phenix_project/phenix_examples.git
       fi
       cd $(System.DefaultWorkingDirectory)/phenix_examples
@@ -111,14 +126,19 @@ jobs:
     artifact: phenix_examples
 
   # xfel_regression
-  - download: data_cache
-    artifact: 'xfel_regression'
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      source: 'specific'
+      project: '$(resources.pipeline.data_cache.projectID)'
+      pipeline: '$(resources.pipeline.data_cache.pipelineID)'
+      allowPartiallySucceededBuilds: true
+      artifact: 'xfel_regression'
+      path: $(System.DefaultWorkingDirectory)/xfel_regression
     displayName: Download xfel_regression
+    continueOnError: true
 
   - script: |
-      if [ -d "$(Pipeline.Workspace)/data_cache/xfel_regression" ]; then
-        mv $(Pipeline.Workspace)/data_cache/xfel_regression $(System.DefaultWorkingDirectory)/xfel_regression
-      else
+      if [ ! -d "$(System.DefaultWorkingDirectory)/xfel_regression" ]; then
         svn co svn+ssh://[email protected]/xfel_regression/trunk xfel_regression
       fi
       cd $(System.DefaultWorkingDirectory)/xfel_regression

diff --git a/.azure-pipelines/azure-pipelines-nightly.yml → .azure-pipelines/azure-pipelines-full.yml b/.azure-pipelines/azure-pipelines-nightly.yml → .azure-pipelines/azure-pipelines-full.yml
@@ -3,6 +3,13 @@
 # no continuous integration builds
 trigger: none
 
+resources:
+  pipelines:
+  - pipeline: data_cache
+    source: 'Update data cache'
+  - pipeline: build_cache
+    source: 'Update build cache'
+
 stages:
 - stage: Preparation
   displayName: Download

diff --git a/.azure-pipelines/azure-pipelines-weekly.yml b/.azure-pipelines/azure-pipelines-weekly.yml
@@ -12,6 +12,13 @@ schedules:
     include:
     - master
 
+resources:
+  pipelines:
+  - pipeline: data_cache
+    source: 'Update data cache'
+  - pipeline: build_cache
+    source: 'Update build cache'
+
 stages:
 - stage: Preparation
   displayName: Download

diff --git a/.azure-pipelines/base-win.yml b/.azure-pipelines/base-win.yml
@@ -47,11 +47,11 @@ jobs:
   - checkout: none
 
   - task: DownloadPipelineArtifact@2
-    displayName: Download modules tarball
     inputs:
       source: 'current'
       artifact: modules
       path: $(Pipeline.Workspace)
+    displayName: Download modules tarball
 
   - script: |
       cd $(Pipeline.Workspace)

diff --git a/.azure-pipelines/conda-linux.yml b/.azure-pipelines/conda-linux.yml
@@ -42,8 +42,14 @@ jobs:
 
   variables:
     artifact_name: ${{ parameters.distribution }}_${{ join('.', parameters.version) }}_$(PYTHON_VERSION)
+    day: $(Get-Date -Format dddd)
 
   steps:
+  - powershell: |
+      Write-Host $(day)
+      Write-Host "##vso[task.setVariable variable=day]$(day)"
+    displayName: Get day of week
+
   # https://github.com/ApexAI/performance_test/blob/master/azure-pipelines.yml#L9-L17
 
   # centos setup
@@ -92,28 +98,29 @@ jobs:
     displayName: Show System variables
 
   # download daily cached build directory for builds
-  # always do builds from scratch for "Update build cache" and "Weekly"
-  # pipelines
+  # always do builds from scratch for "Update build cache" on Saturday night (Pacific)
+  # always do builds from scratch for "Weekly" pipeline
   - task: DownloadPipelineArtifact@2
     inputs:
       source: 'specific'
-      project: '$(System.TeamProjectId)'
-      pipeline: 8
+      project: '$(resources.pipeline.build_cache.projectID)'
+      pipeline: '$(resources.pipeline.build_cache.pipelineID)'
       artifact: '$(artifact_name)'
       path: $(Pipeline.Workspace)
     displayName: Download cached build
     condition: >
-      and(ne(variables['Build.DefinitionName'], 'Update build cache'),
-          ne(variables['Build.DefinitionName'], 'Weekly'))
+      or(and(eq(variables['Build.DefinitionName'], 'Update build cache'),
+             ne(variables['day'], 'Sunday')),
+         eq(variables['Build.DefinitionName'], 'CI'),
+         eq(variables['Build.DefinitionName'], 'CI branch'),
+         eq(variables['Build.DefinitionName'], 'Full'))
     continueOnError: true
 
   - script: |
       cd $(Pipeline.Workspace)
       tar -xf build.tar
     displayName: Extract build tarball
-    condition: >
-      and(ne(variables['Build.DefinitionName'], 'Update build cache'),
-          ne(variables['Build.DefinitionName'], 'Weekly'))
+    condition: succeeded()
     continueOnError: true
     failOnStderr: false
 
@@ -139,10 +146,5 @@ jobs:
   # cache build directory every week for regular optimization build
   - publish: $(Pipeline.Workspace)/build.tar
     artifact: $(artifact_name)
-    condition: >
-      and(eq(variables['Build.DefinitionName'], 'Update build cache'),
-          or(eq(variables['Build.Reason'], 'Schedule'),
-             eq(variables['Build.Reason'], 'Manual')),
-          eq(variables['System.StageAttempt'], 1),
-          eq('${{ parameters.modules }}', 'modules'))
+    condition: succeeded()
     continueOnError: true
diff --git a/.azure-pipelines/conda-osx.yml b/.azure-pipelines/conda-osx.yml
@@ -39,8 +39,14 @@ jobs:
 
   variables:
     artifact_name: ${{ format('xcode_{0}', join('_', parameters.xcode_version))  }}_$(PYTHON_VERSION)
+    day: $(Get-Date -Format dddd)
 
   steps:
+  - powershell: |
+      Write-Host $(day)
+      Write-Host "##vso[task.setVariable variable=day]$(day)"
+    displayName: Get day of week
+
   - script: |
       echo "Selecting Xcode version ${{ join('.', parameters.xcode_version) }}"
       sudo xcode-select --switch /Applications/Xcode_${{ join('.', parameters.xcode_version) }}.app/Contents/Developer
@@ -55,28 +61,29 @@ jobs:
     displayName: Remove homebrew
 
   # download daily cached build directory for builds
-  # always do builds from scratch for "Update build cache" and "Weekly"
-  # pipelines
+  # always do builds from scratch for "Update build cache" on Saturday night (Pacific)
+  # always do builds from scratch for "Weekly" pipeline
   - task: DownloadPipelineArtifact@2
     inputs:
       source: 'specific'
-      project: '$(System.TeamProjectId)'
-      pipeline: 8
+      project: '$(resources.pipeline.build_cache.projectID)'
+      pipeline: '$(resources.pipeline.build_cache.pipelineID)'
       artifact: '$(artifact_name)'
       path: $(Pipeline.Workspace)
     displayName: Download cached build
     condition: >
-      and(ne(variables['Build.DefinitionName'], 'Update build cache'),
-          ne(variables['Build.DefinitionName'], 'Weekly'))
+      or(and(eq(variables['Build.DefinitionName'], 'Update build cache'),
+             ne(variables['day'], 'Sunday')),
+         eq(variables['Build.DefinitionName'], 'CI'),
+         eq(variables['Build.DefinitionName'], 'CI branch'),
+         eq(variables['Build.DefinitionName'], 'Full'))
     continueOnError: true
 
   - script: |
       cd $(Pipeline.Workspace)
       tar -xf build.tar
     displayName: Extract build tarball
-    condition: >
-      and(ne(variables['Build.DefinitionName'], 'Update build cache'),
-          ne(variables['Build.DefinitionName'], 'Weekly'))
+    condition: succeeded()
     continueOnError: true
     failOnStderr: false
 
@@ -99,10 +106,5 @@ jobs:
   # cache build directory every week for regular optimization build
   - publish: $(Pipeline.Workspace)/build.tar
     artifact: $(artifact_name)
-    condition: >
-      and(eq(variables['Build.DefinitionName'], 'Update build cache'),
-          or(eq(variables['Build.Reason'], 'Schedule'),
-             eq(variables['Build.Reason'], 'Manual')),
-          eq(variables['System.StageAttempt'], 1),
-          eq('${{ parameters.modules }}', 'modules'))
+    condition: succeeded()
     continueOnError: true
diff --git a/.azure-pipelines/conda-win.yml b/.azure-pipelines/conda-win.yml
@@ -33,8 +33,14 @@ jobs:
 
   variables:
     artifact_name: win_$(PYTHON_VERSION)
+    day: $(Get-Date -Format dddd)
 
   steps:
+  - powershell: |
+      Write-Host $(day)
+      Write-Host "##vso[task.setVariable variable=day]$(day)"
+    displayName: Get day of week
+
   # install compilers
   - script: |
       choco install vcpython27 -fdv -y --debug
@@ -124,23 +130,24 @@ jobs:
   - task: DownloadPipelineArtifact@2
     inputs:
       source: 'specific'
-      project: '$(System.TeamProjectId)'
-      pipeline: 8
+      project: '$(resources.pipeline.build_cache.projectID)'
+      pipeline: '$(resources.pipeline.build_cache.pipelineID)'
       artifact: '$(artifact_name)'
       path: $(Pipeline.Workspace)
     displayName: Download cached build
     condition: >
-      and(ne(variables['Build.DefinitionName'], 'Update build cache'),
-          ne(variables['Build.DefinitionName'], 'Weekly'))
+      or(and(eq(variables['Build.DefinitionName'], 'Update build cache'),
+             ne(variables['day'], 'Sunday')),
+         eq(variables['Build.DefinitionName'], 'CI'),
+         eq(variables['Build.DefinitionName'], 'CI branch'),
+         eq(variables['Build.DefinitionName'], 'Full'))
     continueOnError: true
 
   - script: |
       cd $(Pipeline.Workspace)
       tar -xf build.tar
     displayName: Extract build tarball
-    condition: >
-      and(ne(variables['Build.DefinitionName'], 'Update build cache'),
-          ne(variables['Build.DefinitionName'], 'Weekly'))
+    condition: succeeded()
     continueOnError: true
     failOnStderr: false
 
@@ -217,10 +224,5 @@ jobs:
   # cache build directory every week for regular optimization build
   - publish: $(Pipeline.Workspace)/build.tar
     artifact: $(artifact_name)
-    condition: >
-      and(eq(variables['Build.DefinitionName'], 'Update build cache'),
-          or(eq(variables['Build.Reason'], 'Schedule'),
-             eq(variables['Build.Reason'], 'Manual')),
-          eq(variables['System.StageAttempt'], 1),
-          eq('${{ parameters.modules }}', 'modules'))
+    condition: succeeded()
     continueOnError: true