Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

<DO_NOT_MERGE>[Troubleshooting] [BUGFIX] Troubleshooting the "slow" tests group in Azure DevOps CI.</DO_NOT_MERGE> #3747

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 37 additions & 35 deletions .github/workflows/cherry-pick-release.yml
Original file line number Diff line number Diff line change
@@ -1,35 +1,37 @@
name: Cherry-Pick into Release Branch

on:
push:
branches: ["master"]

jobs:
release_pull_requestk:
name: cherry_pick_into_${{ matrix.release-branch }}
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
release-branch: [release-0.7, release-0.8]

steps:
- name: checkout
uses: actions/checkout@v1

- name: Check PR labels action step
id: check_pr_labels
uses: shioyang/[email protected]
with:
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
labels: '["${{ matrix.release-branch }}"]'

- name: Create PR into release branch
if: steps.check_pr_labels.outputs.result == 'true'
uses: gorillio/github-action-cherry-pick@master
with:
pr_branch: ${{ matrix.release-branch }}
env:
GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
DRY_RUN: false
# TODO: <Alex>ALEX</Alex>
#name: Cherry-Pick into Release Branch
#
#on:
# push:
# branches: ["master"]
#
#jobs:
# release_pull_requestk:
# name: cherry_pick_into_${{ matrix.release-branch }}
# runs-on: ubuntu-latest
#
# strategy:
# fail-fast: false
# matrix:
# release-branch: [release-0.7, release-0.8]
#
# steps:
# - name: checkout
# uses: actions/checkout@v1
#
# - name: Check PR labels action step
# id: check_pr_labels
# uses: shioyang/[email protected]
# with:
# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
# labels: '["${{ matrix.release-branch }}"]'
#
# - name: Create PR into release branch
# if: steps.check_pr_labels.outputs.result == 'true'
# uses: gorillio/github-action-cherry-pick@master
# with:
# pr_branch: ${{ matrix.release-branch }}
# env:
# GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
# DRY_RUN: false
# TODO: <Alex>ALEX</Alex>
274 changes: 138 additions & 136 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
@@ -1,136 +1,138 @@
name: docker

on:
schedule:
- cron: "0 10 * * *" # everyday at 10am
push:
branches: ["master", "release-*"]
tags: ["v*.*.*"]

jobs:
start-runner:
name: Start self-hosted EC2 runner
if: >
github.event_name == 'schedule' && github.repository == 'ludwig-ai/ludwig' ||
github.event_name == 'push' && github.repository == 'ludwig-ai/ludwig' ||
github.event_name == 'pull_request' && github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && !github.event.pull_request.head.repo.fork
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}

steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}

- name: Start EC2 runner
id: start-ec2-runner
uses: machulav/[email protected]
with:
mode: start
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
ec2-image-id: ami-0759580dedc953d1f
ec2-instance-type: r5.large
subnet-id: subnet-0983be43
security-group-id: sg-4cba0d08
aws-resource-tags: >
[
{"Key": "Name", "Value": "ludwig-github-${{ github.head_ref || github.sha }}"},
{"Key": "GitHubRepository", "Value": "${{ github.repository }}"},
{"Key": "GitHubHeadRef", "Value": "${{ github.head_ref }}"},
{"Key": "GitHubSHA", "Value": "${{ github.sha }}"}
]

docker:
name: Build docker image ${{ matrix.docker-image }} (push=${{ github.event_name != 'pull_request' }})
if: needs.start-runner.result != 'skipped'
needs: start-runner # required to start the main job when the runner is ready
runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runners

# we want an ongoing run of this workflow to be canceled by a later commit
# so that there is only one concurrent run of this workflow for each branch
concurrency:
group: docker-${{ matrix.docker-image }}-${{ github.head_ref || github.sha }}
cancel-in-progress: true

strategy:
fail-fast: false
matrix:
docker-image:
- ludwig
- ludwig-gpu
- ludwig-ray
- ludwig-ray-gpu

steps:
- name: Checkout
uses: actions/checkout@v2

- name: Docker meta
id: meta
uses: crazy-max/ghaction-docker-meta@v2
with:
# list of Docker images to use as base name for tags
images: |
ludwigai/${{ matrix.docker-image }}
# generate Docker tags based on the following events/attributes
tags: |
type=schedule
type=ref,event=branch
type=ref,event=pr
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha

- name: Set up QEMU
uses: docker/setup-qemu-action@v1

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

- name: Login to DockerHub
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
file: ./docker/${{ matrix.docker-image }}/Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

stop-runner:
name: Stop self-hosted EC2 runner

# required to stop the runner even if the error happened in the previous job
if: always() && needs.start-runner.result != 'skipped'
needs:
- start-runner # required to get output from the start-runner job
- docker # required to wait when the main job is done
runs-on: ubuntu-latest

steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}

- name: Stop EC2 runner
uses: machulav/[email protected]
with:
mode: stop
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
label: ${{ needs.start-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
# TODO: <Alex>ALEX</Alex>
#name: docker
#
#on:
# schedule:
# - cron: "0 10 * * *" # everyday at 10am
# push:
# branches: ["master", "release-*"]
# tags: ["v*.*.*"]
#
#jobs:
# start-runner:
# name: Start self-hosted EC2 runner
# if: >
# github.event_name == 'schedule' && github.repository == 'ludwig-ai/ludwig' ||
# github.event_name == 'push' && github.repository == 'ludwig-ai/ludwig' ||
# github.event_name == 'pull_request' && github.event.pull_request.base.repo.full_name == 'ludwig-ai/ludwig' && !github.event.pull_request.head.repo.fork
# runs-on: ubuntu-latest
# outputs:
# label: ${{ steps.start-ec2-runner.outputs.label }}
# ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
#
# steps:
# - name: Configure AWS credentials
# uses: aws-actions/configure-aws-credentials@v1
# with:
# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-region: ${{ secrets.AWS_REGION }}
#
# - name: Start EC2 runner
# id: start-ec2-runner
# uses: machulav/[email protected]
# with:
# mode: start
# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
# ec2-image-id: ami-0759580dedc953d1f
# ec2-instance-type: r5.large
# subnet-id: subnet-0983be43
# security-group-id: sg-4cba0d08
# aws-resource-tags: >
# [
# {"Key": "Name", "Value": "ludwig-github-${{ github.head_ref || github.sha }}"},
# {"Key": "GitHubRepository", "Value": "${{ github.repository }}"},
# {"Key": "GitHubHeadRef", "Value": "${{ github.head_ref }}"},
# {"Key": "GitHubSHA", "Value": "${{ github.sha }}"}
# ]
#
# docker:
# name: Build docker image ${{ matrix.docker-image }} (push=${{ github.event_name != 'pull_request' }})
# if: needs.start-runner.result != 'skipped'
# needs: start-runner # required to start the main job when the runner is ready
# runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runners
#
# # we want an ongoing run of this workflow to be canceled by a later commit
# # so that there is only one concurrent run of this workflow for each branch
# concurrency:
# group: docker-${{ matrix.docker-image }}-${{ github.head_ref || github.sha }}
# cancel-in-progress: true
#
# strategy:
# fail-fast: false
# matrix:
# docker-image:
# - ludwig
# - ludwig-gpu
# - ludwig-ray
# - ludwig-ray-gpu
#
# steps:
# - name: Checkout
# uses: actions/checkout@v2
#
# - name: Docker meta
# id: meta
# uses: crazy-max/ghaction-docker-meta@v2
# with:
# # list of Docker images to use as base name for tags
# images: |
# ludwigai/${{ matrix.docker-image }}
# # generate Docker tags based on the following events/attributes
# tags: |
# type=schedule
# type=ref,event=branch
# type=ref,event=pr
# type=semver,pattern={{version}}
# type=semver,pattern={{major}}.{{minor}}
# type=semver,pattern={{major}}
# type=sha
#
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v1
#
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v1
#
# - name: Login to DockerHub
# if: github.event_name != 'pull_request'
# uses: docker/login-action@v1
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
#
# - name: Build and push
# uses: docker/build-push-action@v2
# with:
# context: .
# file: ./docker/${{ matrix.docker-image }}/Dockerfile
# push: ${{ github.event_name != 'pull_request' }}
# tags: ${{ steps.meta.outputs.tags }}
# labels: ${{ steps.meta.outputs.labels }}
#
# stop-runner:
# name: Stop self-hosted EC2 runner
#
# # required to stop the runner even if the error happened in the previous job
# if: always() && needs.start-runner.result != 'skipped'
# needs:
# - start-runner # required to get output from the start-runner job
# - docker # required to wait when the main job is done
# runs-on: ubuntu-latest
#
# steps:
# - name: Configure AWS credentials
# uses: aws-actions/configure-aws-credentials@v1
# with:
# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# aws-region: ${{ secrets.AWS_REGION }}
#
# - name: Stop EC2 runner
# uses: machulav/[email protected]
# with:
# mode: stop
# github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
# label: ${{ needs.start-runner.outputs.label }}
# ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
# TODO: <Alex>ALEX</Alex>
Loading