first version checkly bot

src/checkly/checklyclient.spec.ts

@@ -13,6 +13,12 @@ describe("ChecklyService", () => {
     const activated = result.filter((r) => r.activated);
     expect(activated).toBeDefined();
   });
+
+  it("can download all check groups", async () => {
+    const groups = await client.getCheckGroups();
+    expect(groups).toBeDefined();
+  });
+
   it("can find activated checks", async () => {
     const result = await client.getActivatedChecks();
     expect(result).toBeDefined();
@@ -37,6 +43,83 @@ describe("ChecklyService", () => {
     expect(result).toBeDefined();
   });
 
+  it("can retrieve check metrics", async () => {
+    const result = await client.getCheckMetrics("BROWSER");
+
+    expect(result).toBeDefined();
+  });
+
+  it("can retrieve check statuses", async () => {
+    const result = await client.getStatuses();
+
+    expect(result).toBeDefined();
+  });
+
+  it("can retrieve dashboards", async () => {
+    const result = await client.getDashboards();
+
+    expect(result).toBeDefined();
+  });
+
+  it("can retrieve dashboard by id", async () => {
+    const id = "77b9895d";
+    const result = await client.getDashboard(id);
+
+    expect(result).toBeDefined();
+  });
+
+  it.skip("can run a check", async () => {
+    const result = await client.runCheck(
+      "e7608d1a-c013-4194-9da0-dec05d2fbabc",
+    );
+
+    expect(result).toBeDefined();
+  });
+
+  it("can retrieve reportings", async () => {
+    const result = await client.getReporting();
+
+    expect(result).toBeDefined();
+  });
+
+  it("can merge checks and groups", async () => {
+    const result = await client.getPrometheusCheckStatus();
+
+    const failingSummary = Object.entries(
+      result.failing.reduce(
+        (acc, curr) => {
+          console.log(acc);
+          if (!acc[curr.labels.group]) {
+            acc[curr.labels.group] = [];
+          }
+
+          acc[curr.labels.group].push(curr.labels.name);
+
+          return acc;
+        },
+        {} as Record<string, string[]>,
+      ),
+    ).reduce((acc, [group, tests]) => {
+      return (
+        acc +
+        "  " +
+        group +
+        "\n" +
+        tests.reduce((acc, curr) => acc + "     " + curr + "\n", "")
+      );
+    }, "Failing tests:\n");
+
+    console.log(`
+Passing: ${result.passing.length}
+Failing: ${result.failing.length}
+Degraded: ${result.degraded.length}
+
+${failingSummary}
+    `);
+
+    expect(result).toBeDefined();
+  });
+
   /*  it('should be defined', async () => {
       const result = await client.getCheckResult(bcheckid, bcheckresult);
       expect(result).toBeDefined();

src/checkly/checklyclient.ts

@@ -1,7 +1,7 @@
 import { plainToClass, plainToInstance } from "class-transformer";
 import * as fs from "fs";
 import fetch from "node-fetch";
-import { Check, CheckGroup, CheckResult } from "./models";
+import { Check, CheckGroup, CheckResult, Reporting, Status } from "./models";
 import { PrometheusParser } from "./PrometheusParser";
 
 interface ChecklyClientOptions {
@@ -53,10 +53,14 @@ export class ChecklyClient {
     return this.getPaginatedDownload("checks", Check);
   }
 
+  async getCheckGroups(): Promise<CheckGroup[]> {
+    return this.getPaginatedDownload("check-groups", CheckGroup);
+  }
+
   async getActivatedChecks(): Promise<Check[]> {
     const results = await Promise.all([
-      this.getPaginatedDownload("checks", Check),
-      this.getPaginatedDownload("check-groups", CheckGroup),
+      this.getChecks(),
+      this.getCheckGroups(),
     ]);
     const groups = results[1];
     const groupMap = new Map<number, CheckGroup>();
@@ -104,10 +108,46 @@ export class ChecklyClient {
     return this.makeRequest(url, CheckResult) as Promise<CheckResult>;
   }
 
-  async makeRequest<T>(url: string, type: { new (): T }): Promise<T | T[]> {
+  async getDashboards() {
+    const url = `${this.checklyApiUrl}dashboards`;
+    return this.makeRequest(url, Object) as Promise<Object>;
+  }
+
+  async getDashboard(id: string) {
+    const url = `${this.checklyApiUrl}dashboards/${id}`;
+    return this.makeRequest(url, Object) as Promise<Object>;
+  }
+
+  async getCheckMetrics(
+    checkType: "HEARTBEAT" | "BROWSER" | "API" | "MULTI_STEP" | "TCP",
+  ) {
+    const url = `${this.checklyApiUrl}analytics/metrics?checkType=${checkType}`;
+    return this.makeRequest(url, Object) as Promise<Object>;
+  }
+
+  async getReporting(options?: { quickRange: "last24Hrs" | "last7Days" }) {
+    const url = `${this.checklyApiUrl}reporting`;
+    return this.makeRequest(url, Reporting) as Promise<Reporting[]>;
+  }
+
+  async getStatuses() {
+    const url = `${this.checklyApiUrl}check-statuses`;
+    return this.makeRequest(url, Status) as Promise<Status[]>;
+  }
+
+  async runCheck(checkId: string) {
+    const url = `${this.checklyApiUrl}triggers/checks/${checkId}`;
+    return this.makeRequest(url, Object, { method: "POST" }) as Promise<Object>;
+  }
+
+  async makeRequest<T>(
+    url: string,
+    type: { new (): T },
+    options?: { method: "GET" | "POST" | undefined },
+  ): Promise<T | T[]> {
     try {
       const response = await fetch(url, {
-        method: "GET", // Optional, default is 'GET'
+        method: options?.method || "GET", // Optional, default is 'GET'
         headers: {
           Authorization: `Bearer ${this.apiKey}`, // Add Authorization header
           "X-Checkly-Account": this.accountId, // Add custom X-Checkly-Account header

src/checkly/models.ts

@@ -174,8 +174,10 @@ export class CheckResult {
     jobAssets: unknown | null;
   } | null;
   browserCheckResult: {
+    errors: Array<ErrorMessage>;
     jobLog: Array<LogEntry>;
     playwrightTestTraces: Array<string>;
+    playwrightTestJsonReportFile: string;
   } | null;
   multiStepCheckResult: {
     errors: Array<ErrorMessage>;
@@ -206,3 +208,71 @@ export class CheckResult {
       .join("\n");
   }
 }
+
+export class PrometheusCheckMetric {
+  name: string;
+  checkType: string;
+  muted: boolean;
+  activated: boolean;
+  checkId: string;
+  tags: string[];
+  group: string;
+  status: "passing" | "failing" | "degraded";
+  value: number;
+
+  static fromJson(json: {
+    labels: {
+      name: string;
+      check_type: string;
+      muted: string;
+      activated: string;
+      check_id: string;
+      tags: string;
+      group: string;
+      status: string;
+    };
+    value: number;
+  }): PrometheusCheckMetric {
+    const metric = {
+      ...json.labels,
+      tags: json.labels.tags.split(","),
+      checkId: json.labels.check_id,
+      checkType: json.labels.check_type,
+      muted: json.labels.muted === "true",
+      activated: json.labels.activated === "true",
+      status: json.labels.status as "passing" | "failing" | "degraded",
+      value: json.value,
+    };
+
+    return metric;
+  }
+}
+
+export class Reporting {
+  name: string;
+  checkId: string;
+  checkType: string;
+  deactivated: boolean;
+  tags: string[];
+  aggregate: {
+    successRatio: number;
+    avg: number;
+    p95: number;
+    p99: number;
+  };
+}
+
+export class Status {
+  name: string;
+  hasErrors: boolean;
+  hasFailures: boolean;
+  longestRun: number | null;
+  shortestRun: number | null;
+  checkId: string;
+  created_at: string;
+  updated_at: string;
+  lastRunLocation: string | null;
+  lastCheckRunId: string | null;
+  sslDaysRemaining: number | null;
+  isDegraded: boolean;
+}

src/prompts/checkly.eval.spec.ts

@@ -1,9 +1,11 @@
+import fs from "fs";
+
 import { generateText } from "ai";
 import dotenv from "dotenv";
 import { CheckContext } from "../aggregator/ContextAggregator";
 import { getOpenaiSDKClient } from "../ai/openai";
 import { startLangfuseTelemetrySDK } from "../langfuse";
-import { contextAnalysisSummaryPrompt } from "./checkly";
+import { contextAnalysisSummaryPrompt, featureCoveragePrompt } from "./checkly";
 import { expect } from "@jest/globals";
 import { Possible, Factuality, Battle, Summary } from "./toScoreMatcher";
 startLangfuseTelemetrySDK();
@@ -166,4 +168,67 @@ test('visit page and take screenshot', async ({ page }) => {
       ),
     ]);
   });
+
+  it("should generate a feature coverage prompt", async () => {
+    const { name, script, scriptPath } = {
+      name: "Create Browser Check",
+      script:
+        "// import { test } from '@playwright/test'\nimport { expect, test } from '../../../../../../__checks__/helpers/checklyTest'\nimport { invokeFnAndWaitForResponse } from '../../../../../../__checks__/helpers/invokeFnAndWaitForResponse'\nimport { randomString } from '../../../../../../__checks__/helpers/randomString'\nimport { CheckBuilderHeaderPom } from '../../../../../components/checks/check-builder/__checks__/CheckBuilderHeaderPom'\nimport { BrowserCheckBuilderPom } from './pom/BrowserCheckBuilderPom'\n\ntest('should create browser check', async ({ page, webapp, api }) => {\n  await webapp.login()\n\n  const browserCheckBuilder = new BrowserCheckBuilderPom({ page, webapp })\n\n  await browserCheckBuilder.navigateToUsingSidebarCreateButton()\n\n  const checkName = `Check E2E test ${randomString()}`\n\n  const checkBuilderHeaderPom = new CheckBuilderHeaderPom({ page, webapp })\n  await checkBuilderHeaderPom.nameInput.fill(checkName)\n  await checkBuilderHeaderPom.activateCheckbox.uncheck({ force: true })\n\n  const createdCheck = await invokeFnAndWaitForResponse({\n    page,\n    urlMatcher: (url: string) => url.endsWith('/checks'),\n    method: 'POST',\n    status: 200,\n    fn: () => checkBuilderHeaderPom.saveButton.click(),\n  })\n\n  /**\n   * Wait until the browser check builder has loaded the check, otherwise it might get deleted via the API\n   * before the UI has a chance to refresh the create screen into the edit screen.\n   *\n   * The assertion here is a bit arbitrary (i.e. it could be any element on the edit screen).\n   */\n  await expect(page.getByText('Export to code')).toBeVisible()\n\n  api.checks.addToCleanupQueue(createdCheck.id)\n})\n",
+      scriptPath:
+        "src/pages/checks/browser/create/__checks__/create_check.spec.ts",
+    };
+
+    const errors = [
+      'Error: Timed out 30000ms waiting for expect(locator).toHaveTitle(expected)\n\nLocator: locator(\':root\')\nExpected string: "New browser check"\nReceived string: "Create from scratch"\nCall log:\n  - expect.toHaveTitle with timeout 30000ms\n  - waiting for locator(\':root\')\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Dashboard"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Dashboard"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Dashboard"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n  -   locator resolved to <html lang="en">…</html>\n  -   unexpected value "Create from scratch"\n\n    at BrowserCheckBuilderPom.navigateToUsingSidebarCreateButton (/check/569270bb-6b06-4c38-9e4f-f2294953745f/src/pages/checks/browser/create/__checks__/pom/module.ts:38:25)\n    at /check/569270bb-6b06-4c38-9e4f-f2294953745f/src/pages/checks/browser/create/__checks__/create_check.spec.ts:13:31',
+    ];
+
+    const [prompt, config] = featureCoveragePrompt(
+      name,
+      scriptPath,
+      script,
+      errors,
+    );
+    const { text: summary } = await generateText({
+      ...config,
+      prompt,
+    });
+
+    const expected = `
+        1. User logs into the application.
+        2. Navigate to create new check.
+        3. Enter name for browser check.
+        4. Deselect activate for new check.
+        5. Save the new browser check.
+
+    **Failure Occurred At:** Step 2: Navigate to create new check. The error happened while trying to navigate, as evident by the unresolved title "New browser check". This indicates a navigation issue, possibly remaining on the "Create from scratch" page instead.`;
+    const input = JSON.stringify({
+      name,
+      scriptPath,
+      script,
+      errors,
+    });
+
+    return Promise.all([
+      expect(summary).toScorePerfect(
+        Possible({
+          input,
+          expected: expected,
+        }),
+      ),
+      expect(summary).toScoreGreaterThanOrEqual(
+        Factuality({
+          input: prompt,
+          expected: expected,
+        }),
+        0.5,
+      ),
+      expect(summary).toScoreGreaterThanOrEqual(
+        Battle({
+          instructions: prompt,
+          expected: expected,
+        }),
+        0.5,
+      ),
+    ]);
+  });
 });

src/prompts/checkly.ts

@@ -3,7 +3,12 @@ import { CheckContext, ContextKey } from "../aggregator/ContextAggregator";
 import { Check } from "../checkly/models";
 import { mapCheckToContextValue } from "../checkly/utils";
 import { validObjectList, validObject } from "./validation";
-import { definePrompt, PromptDefinition } from "./common";
+import {
+  definePrompt,
+  promptConfig,
+  PromptConfig,
+  PromptDefinition,
+} from "./common";
 import { slackFormatInstructions } from "./slack";
 import { z } from "zod";
 
@@ -46,6 +51,49 @@ ${stringify(entry)}`;
   });
 }
 
+export function featureCoveragePrompt(
+  testName: string,
+  scriptName: string,
+  script: string,
+  errors: string[],
+): [string, PromptConfig] {
+  return [
+    `
+      The following details describe a test which is used to monitor an application.
+
+      Do not refer to technical details of the test, use the domain language from the application under test.
+      Test name: ${testName}
+      Script name: ${scriptName}
+      Script: ${script}
+      Error stack: ${errors}
+
+      Summarize the steps executed by the test using high level domain language. Focus on the user flow omit technical details. Use max 5 words per step.
+      Identify the step which failed by match the script code with the stack of the error. Include details about the test failure.
+
+      CONSTITUTION:
+      - Always prioritize accuracy and relevance in the summary
+      - Be concise but comprehensive in your explanations
+      - Focus on providing actionable information that can help judging user impact
+    `,
+    promptConfig("checklySummarizeFeatureCoverage", {
+      temperature: 1,
+      maxTokens: 500,
+    }),
+  ];
+}
+
+/**
+ * Generates a comprehensive analysis prompt for multiple context entries.
+ * Creates a structured prompt for analyzing check state changes and generating
+ * actionable insights for DevOps engineers. The prompt includes specific
+ * instructions for format, analysis approach, and output requirements.
+ *
+ * @param {CheckContext[]} contextRows - Array of context entries to analyze
+ * @returns {string} A formatted prompt string for comprehensive context analysis
+ *
+ * @example
+ * const summary = contextAnalysisSummaryPrompt(contextEntries);
+ */
 export function contextAnalysisSummaryPrompt(
   contextRows: CheckContext[],
 ): PromptDefinition {