-
Notifications
You must be signed in to change notification settings - Fork 22
/
Copy pathopenqa-label-known-issues
executable file
·191 lines (172 loc) · 8.63 KB
/
openqa-label-known-issues
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
#!/bin/bash -e
# Usage
# openqa-label-known-issues https://openqahost/tests/1234
set -o pipefail -o errtrace
# shellcheck source=/dev/null
. "$(dirname "${BASH_SOURCE[0]}")"/_common
host="${host:-"openqa.opensuse.org"}"
scheme="${scheme:-"https"}"
host_url="$scheme://$host"
dry_run="${dry_run:-"0"}"
min_search_term="${min_search_term:-"16"}"
issue_marker="${issue_marker:-"auto_review%3A"}"
issue_query="${issue_query:-"https://progress.opensuse.org/projects/openqav3/issues.json?limit=200&subproject_id=*&subject=~${issue_marker}"}"
force_result_tracker="${force_result_tracker:-"openqa-force-result"}"
reason_min_length="${reason_min_length:-"8"}"
grep_timeout="${grep_timeout:-5}"
email_unreviewed=${email_unreviewed:-false}
notification_address=${notification_address:-}
from_email=${from_email:[email protected]}
curl_args=(-L --user-agent "openqa-label-known-issues")
retries="${retries:-"3"}"
OPENQA_CLI_RETRY_SLEEP_TIME_S=${OPENQA_CLI_RETRY_SLEEP_TIME_S:-120}
MOJO_CONNECT_TIMEOUT=${MOJO_CONNECT_TIMEOUT:-30}
client_args=(api --header 'User-Agent: openqa-label-known-issues (https://github.com/os-autoinst/scripts)' --host "$host_url" --retries="$retries")
out="${REPORT_FILE:-$(mktemp -t openqa-label-known-issues--output-XXXX)}"
trap 'error-handler "$LINENO"' ERR
trap '_cleanup' EXIT
_cleanup() {
test "$KEEP_REPORT_FILE" == "1" || rm "$out"
test "$KEEP_JOB_HTML_FILE" == "1" || rm -f "$html_out"
}
echoerr() { echo "$@" >&2; }
handle_unreachable() {
local testurl=$1
local timeago
html_out=${JOB_HTML_FILE:-$(mktemp -t openqa-label-known-issues--job-details-XXXX)}
if ! curl "${curl_args[@]}" -s --head "$testurl" -o /dev/null; then
# the page might be gone, try the scheme+host we configured (might be different one though)
if ! grep -q "$host_url" <<< "$testurl"; then
echoerr "'$testurl' is not reachable and 'host_url' parameter does not match '$testurl', can not check further, continuing with next"
return 1
fi
if ! curl "${curl_args[@]}" -s --head "$host_url"; then
echoerr "'$host_url' is not reachable, bailing out"
curl "${curl_args[@]}" --head "$host_url"
fi
echoerr "'$testurl' is not reachable, assuming deleted, continuing with next"
return 1
fi
# resorting to downloading the job details page instead of the
# log
if ! curl "${curl_args[@]}" -s "$testurl" -o "${html_out}"; then
echoerr "'$testurl' can be reached but not downloaded, bailing out"
curl "${curl_args[@]}" "$testurl"
exit 2
fi
if hxnormalize -x "${html_out}" | hxselect -s '\n' -c '.links_a .resborder' | grep -qPzo '(?s)Gru job failed.*connection error.*Inactivity timeout'; then
"${client_call[@]}" -X POST jobs/"$id"/comments text='poo#62456 test incompletes after failing in GRU download task on "Inactivity timeout" with no logs'
"${client_call[@]}" -X POST jobs/"$id"/restart
return 1
fi
# Checking timestamp given by job details page
timeago=$(grep timeago "$html_out" | hxselect -s '\n' -c '.timeago::attr(title)')
if [[ $(date -uIs -d '-14days') > $timeago ]]; then
# if the page is there but not even an autoinst-log.txt exists
# then the job might be too old and logs are already deleted.
echoerr "'$testurl' job#${id} without autoinst-log.txt older than 14 days. Do not label"
return 1
fi
}
label_on_issues_from_issue_tracker() {
local id=$1
# Iterate over all progress issues that have the search term included
# 1. line: issue id
# 2. line: subject
echo "$issues" | (
while read -r issue; do
read -r subject
read -r tracker
after=${subject#*\"}
search=${after%\"*}
force_result=''
label="poo#$issue $subject"
if [[ ${#search} -ge $min_search_term ]]; then
if [[ $after =~ :force_result:([a-z_]+) ]]; then
if [[ $tracker == "$force_result_tracker" ]]; then
force_result=${BASH_REMATCH[1]}
else
label="$label (ignoring force result for ticket which is not in tracker \"$force_result_tracker\")"
fi
fi
label-on-issue "$id" "$search" "$label" "${after//*\":retry*/1}" "$force_result" && break
fi
done
)
}
label_on_issues_without_tickets() {
if label-on-issue "$id" '([dD]ownload.*failed.*404)[\S\s]*Result: setup failure' 'label:non_existing asset, candidate for removal or wrong settings'; then
return
elif label-on-issue "$id" 'File .*\.yaml.* does not exist at .*scheduler.pm' 'label:missing_schedule_file'; then
return
elif label-on-issue "$id" 'Compilation failed in require at .*isotovideo line 28.' 'label:schedule_compilation_error'; then
return
elif label-on-issue "$id" 'qemu-img: Could not open .*: No such file or directory' 'label:missing_asset'; then
return
elif label-on-issue "$id" 'fatal: Remote branch .* not found' 'label:remote_branch_not_found, probably wrong custom git URL specified with branch'; then
return
elif label-on-issue "$id" 'fatal: repository not found' 'label:remote_repo_not_found, probably wrong custom git URL specified'; then
return
elif label-on-issue "$id" '(?s)Cloning git URL.*to use as test distribution.*(No scripts in|needledir not found)' 'label:remote_repo_invalid, probably wrong custom git URL specified'; then
return
elif label-on-issue "$id" '(?s)Cloning git URL.*to use as test distribution.*(SCHEDULE.*not set|loadtest needs a script below)' 'label:remote_repo_schedule_not_found, probably wrong custom git URL + PRODUCTDIR specified'; then
return
elif label-on-issue "$id" '\[error\] Failed to download' 'label:download_error potentially out-of-space worker?' 1; then
return
fi
false
}
investigate_issue() {
local testurl=$1
local id="${testurl##*/}"
local reason
local curl_output
echo "Requesting jobs/${id} via openqa-cli"
job_data=$(openqa-cli "${client_args[@]}" jobs/"$id")
state="$(echo "$job_data" | runjq -r '.job.state')"
result="$(echo "$job_data" | runjq -r '.job.result')"
# Skip all unfinished or passed jobs for now
if [[ "$state" != 'done' || "$result" == passed ]]; then
return
fi
reason=$(echo "$job_data" | runjq -r '.job.reason')
group_id=$(echo "$job_data" | runjq -r '.job.group_id')
curl_output=$(curl "${curl_args[@]}" -sS -w "%{http_code}" "$testurl/file/autoinst-log.txt" -o "$out")
# combine both the reason and autoinst-log.txt to check known issues
# against even in case when autoinst-log.txt is missing the details, e.g.
# see https://progress.opensuse.org/issues/69178
echo "$reason" >> "$out"
if [[ "$curl_output" != "200" ]] && [[ "$curl_output" != "301" ]]; then
# if we can not even access the page it is something more critical
handle_unreachable "$testurl" "$out" || return 0
[[ $curl_output != 404 ]] && return
# not unreachable, no log, no reason, not too old
if [[ -z $reason ]] || [[ $reason = null ]]; then
echoerr "'$testurl' does not have autoinst-log.txt or reason, cannot label"
return
fi
fi
label_on_issues_from_issue_tracker "$id" && return
## Issues without tickets, e.g. potential singular, manual debug jobs,
# wrong user settings, etc.
# could create an issue automatically with
# $client_prefix curl -s -H "Content-Type: application/json" -X POST -H "X-Redmine-API-Key: $(sed -n 's/redmine-token = //p' ~/.query_redminerc)" --data '{"issue": {"project_id": 36, "category_id": 152, priority_id: 5, "subject": "test from command line"}}' https://progress.opensuse.org/issues.json
# but we should check if the issue already exists, e.g. same
# subject line
label_on_issues_without_tickets "$id" && return
handle_unreviewed "$testurl" "$out" "$reason" "$group_id" "$email_unreviewed" "$from_email" "$notification_address" "$job_data" "$dry_run"
}
label_issue() {
local testurl="${1:?"Need 'testurl'"}"
[[ "$dry_run" = "1" ]] && client_prefix="echo"
if [[ -z "$client_call" ]]; then
client_call=(openqa-cli "${client_args[@]}")
if [[ -n "$client_prefix" ]]; then
client_call=("$client_prefix" "${client_call[@]}")
fi
fi
# search for issues with a subject search term
issues=${issues:-$(exp_retries=12 runcurl "${curl_args[@]}" -sS "$issue_query" | runjq -r '.issues | .[] | (.id,.subject,.tracker.name)')}
investigate_issue "$testurl"
}
caller 0 > /dev/null || label_issue "$@"