Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow for recurring maintenance events #246

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
17 changes: 17 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
"nodemailer": "^6.5.0",
"notifme-sdk": "^1.10.0",
"prettier": "^2.2.1",
"rrule": "^2.8.1",
"shelljs": "^0.8.4",
"tcp-ping": "^0.1.1",
"ws": "^7.4.6"
Expand Down
130 changes: 100 additions & 30 deletions src/update.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import dns from "dns";
import { isIP, isIPv6 } from "net";
import slugify from "@sindresorhus/slugify";
import dayjs from "dayjs";
import utc from "dayjs/plugin/utc";
import { mkdirp, readFile, writeFile } from "fs-extra";
import { load } from "js-yaml";
import { join } from "path";
Expand All @@ -17,6 +18,10 @@ import { curl } from "./helpers/request";
import { getOwnerRepo, getSecret } from "./helpers/secrets";
import { SiteHistory } from "./interfaces";
import { generateSummary } from "./summary";
import { rrulestr } from 'rrule';
import { Octokit } from "@octokit/rest";

dayjs.extend(utc);

const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

Expand All @@ -41,6 +46,45 @@ function getHumanReadableTimeDifference(startTime: Date): string {
return result.join(", ");
}

function getDurationMinutes(duration: string): number {
const unit = duration.slice(-1).toUpperCase();

try {
if (unit === "M") {
return parseInt(duration.slice(0, -1));
} else if (unit === "H") {
return parseInt(duration.slice(0, -1)) * 60;
} else if (unit === "D") {
return parseInt(duration.slice(0, -1)) * 60 * 24;
} else {
return 0;
}
} catch (error) {
console.error("Error parsing duration", duration, error);
return 0;
}
}

async function closeMaintenanceIssue(octokit: Octokit, owner: string, repo: string, incidentNumber: number) {
await octokit.issues.unlock({
owner,
repo,
issue_number: incidentNumber,
});
await octokit.issues.update({
owner,
repo,
issue_number: incidentNumber,
state: "closed",
});
await octokit.issues.lock({
owner,
repo,
issue_number: incidentNumber,
});
console.log("Closed maintenance completed event", incidentNumber);
}

export const update = async (shouldCommit = false) => {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I observed that if an error is thrown at all throughout this process, we fail the GitHub Action and dont raise any alerts.

If someone misconfigures a maintenance issue, they could inadvertently disable all of their uptime checks. This probably warrants a separate issue and fix, regardless of how this PR goes.

if (!(await shouldContinue())) return;
await mkdirp("history");
Expand Down Expand Up @@ -77,38 +121,63 @@ export const update = async (shouldCommit = false) => {
metadata[i.split(/:(.+)/)[0].trim()] = i.split(/:(.+)/)[1].trim();
});
}
if (metadata.start && metadata.end) {
let expectedDown: string[] = [];
let expectedDegraded: string[] = [];
if (metadata.expectedDown)
expectedDown = metadata.expectedDown
.split(",")
.map((i) => i.trim())
.filter((i) => i.length);
if (metadata.expectedDown)
expectedDegraded = metadata.expectedDown
.split(",")
.map((i) => i.trim())
.filter((i) => i.length);
Comment on lines -88 to -92
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Additionally this PR I fixed the expectedDegraded array. You can see here we were actually just checking expectedDown twice, and assigning it to expectedDegraded


if (dayjs(metadata.end).isBefore(dayjs())) {
await octokit.issues.unlock({
owner,
repo,
issue_number: incident.number,
});
await octokit.issues.update({
owner,
repo,
issue_number: incident.number,
state: "closed",
});
await octokit.issues.lock({
owner,
repo,
issue_number: incident.number,
let expectedDown: string[] = [];
let expectedDegraded: string[] = [];
if (metadata.expectedDown)
expectedDown = metadata.expectedDown
.split(",")
.map((i) => i.trim())
.filter((i) => i.length);
if (metadata.expectedDegraded)
expectedDegraded = metadata.expectedDegraded
.split(",")
.map((i) => i.trim())
.filter((i) => i.length);

if (metadata.rrule && metadata.duration && metadata.start) {
// The DTSTART and UNTIL params in RRules should be in UTC format without colons and dashes
if (!metadata.rrule.includes("DTSTART")) {
const cleanStartTime = dayjs(metadata.start)
.utc()
.format()
.replaceAll(":", "")
.replaceAll("-", "");
metadata.rrule += `;DTSTART=${cleanStartTime}`;
}
if (!metadata.rrule.includes("UNTIL") && metadata.end) {
const cleanEndTime = dayjs(metadata.end)
.utc()
.format()
.replaceAll(":", "")
.replaceAll("-", "");
metadata.rrule += `;UNTIL=${cleanEndTime}`;
}
const rule = rrulestr(metadata.rrule);

if (metadata.end && dayjs(metadata.end).isBefore(dayjs())) {
await closeMaintenanceIssue(octokit, owner, repo, incident.number);
} else {
// Get all potentially valid occurrences of this rule (started up to `duration` ago)
// Limit to 1000 results to avoid any potential long-running operations
const durationMinutes = getDurationMinutes(metadata.duration);
const after = dayjs().subtract(durationMinutes, "minutes").toDate();
rule.between(after, new Date(), true, (_, i) => i < 1000).forEach((startDate) => {
const endDate = dayjs(startDate).add(durationMinutes, "minutes").toDate();

const start = startDate.toISOString();
const end = endDate.toISOString();

ongoingMaintenanceEvents.push({
issueNumber: incident.number,
metadata: { start, end, expectedDegraded, expectedDown },
});
});
console.log("Closed maintenance completed event", incident.number);
}

} else if (metadata.start && metadata.end) {
if (dayjs(metadata.end).isBefore(dayjs())) {
await closeMaintenanceIssue(octokit, owner, repo, incident.number);
} else if (dayjs(metadata.start).isBefore(dayjs())) {
ongoingMaintenanceEvents.push({
issueNumber: incident.number,
Expand Down Expand Up @@ -519,3 +588,4 @@ generator: Upptime <https://github.com/upptime/upptime>
};

const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));