-
Notifications
You must be signed in to change notification settings - Fork 21
118 lines (100 loc) · 4.41 KB
/
custom_image_generation.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
name: Custom Image Generation
on:
workflow_dispatch:
inputs:
run-dataproc-override:
description: 'Set this to `true` to run dataproc image generation script.'
required: false
default: false
type: boolean
run-gce-override:
description: 'Set this to `true` to run gce image generation script.'
required: false
default: false
type: boolean
env:
GOOGLE_PROJECT: broad-dsp-gcr-public
GOOGLE_CREDENTIAL_FILE_NAME: application_default_credentials.json
REGION: us-central1
ZONE: us-central1-a
jobs:
run-image-generation-script-gce:
runs-on: self-hosted
if: ${{ inputs.run-gce-override }} # [DEBUG COMMENT]
permissions:
contents: 'read'
id-token: 'write'
env:
GCE_IMAGE_BUCKET: gs://leo-gce-image-creation-logs
DAISY_IMAGE: gcr.io/compute-image-tools/daisy:release
GCP_DAISY_GCR_KEY: ${{secrets.GCP_DAISY_GCR_KEY}}
OUTPUT_FILE_RELATIVE_PATH: jenkins/gce-custom-images/output.txt
steps:
- uses: actions/checkout@v3
- id: 'auth'
name: 'Authenticate to Google Cloud'
uses: 'google-github-actions/auth@v1'
with:
# Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
service_account: '[email protected]'
- name: 'Set up Cloud SDK'
uses: 'google-github-actions/setup-gcloud@v1'
- name: Create svc acct file
run: 'echo "$GCP_DAISY_GCR_KEY" | base64 -d > "$GITHUB_WORKSPACE/jenkins/gce-custom-images/$GOOGLE_CREDENTIAL_FILE_NAME"'
shell: bash
env:
GCP_DAISY_GCR_KEY: ${{secrets.GCP_DAISY_GCR_KEY}}
- name: Setup for script
run: |
gsutil ls $GCE_IMAGE_BUCKET || gsutil mb -p $GOOGLE_PROJECT -l $REGION $GCE_IMAGE_BUCKET
docker pull $DAISY_IMAGE
- name: Run image generation script
shell: bash
run: |
OUTPUT_FILE_PATH="$GITHUB_WORKSPACE/$OUTPUT_FILE_RELATIVE_PATH" GCE_IMAGE_BUCKET=$GCE_IMAGE_BUCKET $GITHUB_WORKSPACE/jenkins/gce-custom-images/create_gce_image.sh
gsutil rm -r $GCE_IMAGE_BUCKET || true
- name: Output image URI
run: |
cat $GITHUB_WORKSPACE/$OUTPUT_FILE_RELATIVE_PATH
echo "`cat $GITHUB_WORKSPACE/$OUTPUT_FILE_RELATIVE_PATH` > $GITHUB_STEP_SUMMARY"
run-image-generate-script-dataproc:
runs-on: ubuntu-latest
if: ${{ inputs.run-dataproc-override }} # [DEBUG COMMENT]
permissions:
contents: 'read'
id-token: 'write'
env:
GCP_DAISY_GCR_KEY: ${{secrets.GCP_DAISY_GCR_KEY}}
OUTPUT_FILE_RELATIVE_PATH: jenkins/dataproc-custom-images/output.txt
DATAPROC_IMAGE_BUCKET: gs://leo-dataproc-image-creation-logs
steps:
- uses: actions/checkout@v3
- id: 'auth'
name: 'Authenticate to Google Cloud'
uses: 'google-github-actions/auth@v1'
with:
# Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
service_account: '[email protected]'
- name: 'Set up Cloud SDK'
uses: 'google-github-actions/setup-gcloud@v1'
- name: Create svc acct file
run: 'echo "$GCP_DAISY_GCR_KEY" | base64 -d > "$GITHUB_WORKSPACE/jenkins/dataproc-custom-images/$GOOGLE_CREDENTIAL_FILE_NAME"'
shell: bash
env:
GCP_DAISY_GCR_KEY: ${{secrets.GCP_DAISY_GCR_KEY}}
- name: Init submodule
run: |
git submodule sync
git submodule update --init --recursive
- name: Run image generation script
shell: bash
run: |
gsutil ls $DATAPROC_IMAGE_BUCKET || gsutil mb -b on -p $GOOGLE_PROJECT -l $REGION "$DATAPROC_IMAGE_BUCKET"
OUTPUT_FILE_PATH="$GITHUB_WORKSPACE/$OUTPUT_FILE_RELATIVE_PATH" DATAPROC_IMAGE_BUCKET=$DATAPROC_IMAGE_BUCKET $GITHUB_WORKSPACE/jenkins/dataproc-custom-images/create_dataproc_image.sh
gsutil rm -r $DATAPROC_IMAGE_BUCKET || true
- name: Output image URI
run: |
cat $GITHUB_WORKSPACE/$OUTPUT_FILE_RELATIVE_PATH
echo "`cat $GITHUB_WORKSPACE/$OUTPUT_FILE_RELATIVE_PATH` > $GITHUB_STEP_SUMMARY"