Feat: Configurable Follow HTTP Redirect for Probe Requests (hyperjump…

…tech#1270)

* refactor: move global variable to context to prevent recreating in many places

* refactor: extract log running info and fetch and cache public network info function

* refactor: extract init prometheus function

* docs: add follow redirect documentation

* feat: add follow redirects json schema property

* feat: implement follow redirects on request level

* fix: adjust affected codes

* test: increase test timeout

* test: debug test

* test: remove CI env var as an indicator of running test

* refactor: move fields down to child function

* chore: remove unnecessary logs

* test: add httpbin docker compose for testing purposes

* docs: add readme for HTTPbin

README.md

@@ -63,12 +63,38 @@ See this [docs](https://turbo.build/repo/docs/handbook/workspaces#workspaces-whi
 
 ### How to Test Probe Locally
 
-If you need to test a probe locally, there are predefined services in /dev/docker-compose.yaml. You are **encouraged** to add other services that can be probed by Monika. Run `cd dev && docker compose up` to run those services.
+If you need to test a probe locally, there are predefined services in `/dev/docker-compose.yaml`. You are **encouraged** to add other services that can be probed by Monika. Run `cd dev && docker compose up` to run those services.
 
 #### Available Services
 
 Use the following Monika config to probe the service.
 
+##### HTTPBin
+
+```yaml
+probes:
+  - id: 'should not follow redirect'
+    requests:
+      - url: http://localhost:3000/status/302
+        followRedirects: 0
+    alerts:
+      - assertion: response.status != 302
+        message: You should not follow redirect
+  - id: 'should follow redirect with default config'
+    requests:
+      - url: http://localhost:3000/absolute-redirect/20
+    alerts:
+      - assertion: response.status == 302
+        message: You are not follow redirect
+  - id: 'should follow redirect with customize config'
+    requests:
+      - url: http://localhost:3000/status/302
+        followRedirects: 2
+    alerts:
+      - assertion: response.status == 302
+        message: You are not follow redirect
+```
+
 ##### MariaDB
 
 ```yaml

dev/docker-compose.yaml

@@ -1,4 +1,8 @@
 services:
+  httpbin:
+    image: kennethreitz/httpbin
+    ports:
+      - 3000:80
   mariadb:
     image: mariadb:11
     container_name: monika_mariadb

docs/src/pages/guides/probes.md

@@ -50,6 +50,7 @@ probes:
           - assertion: response.status != 200
             message: Status not 2xx
         allowUnauthorized: true
+        followRedirects: 1
     incidentThreshold: 3
     alerts:
       - assertion: response.status != 200
@@ -72,6 +73,7 @@ Details of the field are given in the table below.
 | alerts (optional)            | The condition which will trigger an alert, and the subsequent notification method to send out the alert. See below for further details on alerts and notifications. See [alerts](./alerts) section for detailed information.                                                                                                                                                                                                                                                                                                                                                                      |
 | ping (optional)              | (boolean), If set true then send a PING to the specified url instead.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | allowUnauthorized (optional) | (boolean), If set to true, will make https agent to not check for ssl certificate validity                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| followRedirects (optional)   | The request follows redirects as many times as specified here. If unspecified, it will fallback to the value set by the [follow redirects flag](https://monika.hyperjump.tech/guides/cli-options#follow-redirects)                                                                                                                                                                                                                                                                                                                                                                                |
 
 ## Request Body
 

src/commands/monika.ts

@@ -36,7 +36,7 @@ import { closeLog, openLogfile } from '../components/logger/history'
 import { logStartupMessage } from '../components/logger/startup-message'
 import { scheduleSummaryNotification } from '../components/notification/schedule-notification'
 import { sendMonikaStartMessage } from '../components/notification/start-message'
-import { setContext } from '../context'
+import { getContext, setContext } from '../context'
 import events from '../events'
 import {
   type MonikaFlags,
@@ -410,7 +410,7 @@ export default class Monika extends Command {
           signal,
         })
 
-        if (process.env.NODE_ENV === 'test') {
+        if (getContext().isTest) {
           break
         }
 

src/components/config/__tests__/expected.sitemap-oneprobe.yml

@@ -6,22 +6,27 @@ probes:
         method: GET
         timeout: 10000
         body: ''
+        followRedirects: 21
       - url: https://de.wiktionary.org/wiki/hyperjump
         method: GET
         timeout: 10000
         body: ''
+        followRedirects: 21
       - url: https://id.wiktionary.org/wiki/hyperjump
         method: GET
         timeout: 10000
         body: ''
+        followRedirects: 21
       - url: https://en.wiktionary.org/wiki/hyperjump
         method: GET
         timeout: 10000
         body: ''
+        followRedirects: 21
       - url: https://monika.hyperjump.tech/index
         method: GET
         timeout: 10000
         body: ''
+        followRedirects: 21
     interval: 900
     alerts:
       - assertion: response.status < 200 or response.status > 299

src/components/config/__tests__/expected.textfile.yml

@@ -6,6 +6,7 @@ probes:
         method: GET
         timeout: 10000
         body: {}
+        followRedirects: 21
     interval: 900
     alerts:
       - assertion: response.status < 200 or response.status > 299
@@ -19,6 +20,7 @@ probes:
         method: GET
         timeout: 10000
         body: {}
+        followRedirects: 21
     interval: 900
     alerts:
       - assertion: response.status < 200 or response.status > 299

src/components/config/index.test.ts

@@ -147,7 +147,14 @@ describe('updateConfig', () => {
           id: '1',
           name: '',
           interval: 1000,
-          requests: [{ url: 'https://example.com', body: '', timeout: 1000 }],
+          requests: [
+            {
+              url: 'https://example.com',
+              body: '',
+              followRedirects: 21,
+              timeout: 1000,
+            },
+          ],
           alerts: [],
         },
       ],
@@ -181,7 +188,14 @@ describe('updateConfig', () => {
           id: '1',
           name: '',
           interval: 1000,
-          requests: [{ url: 'https://example.com', body: '', timeout: 1000 }],
+          requests: [
+            {
+              url: 'https://example.com',
+              body: '',
+              followRedirects: 21,
+              timeout: 1000,
+            },
+          ],
           alerts: [],
         },
       ],

src/components/config/index.ts

@@ -62,7 +62,6 @@ type WatchConfigFileParams = {
   path: string
 }
 
-const isTestEnvironment = process.env.CI || process.env.NODE_ENV === 'test'
 const emitter = getEventEmitter()
 
 const defaultConfigs: Partial<Config>[] = []
@@ -103,7 +102,8 @@ export const updateConfig = async (config: Config): Promise<void> => {
     log.info('Config file update detected')
   } catch (error: unknown) {
     const message = getErrorMessage(error)
-    if (isTestEnvironment) {
+
+    if (getContext().isTest) {
       // return error during tests
       throw new Error(message)
     }
@@ -220,7 +220,7 @@ function scheduleRemoteConfigFetcher({
 }
 
 function watchConfigFile({ flags, path }: WatchConfigFileParams) {
-  const isWatchConfigFile = !(isTestEnvironment || flags.repeat !== 0)
+  const isWatchConfigFile = !(getContext().isTest || flags.repeat !== 0)
   if (isWatchConfigFile) {
     const watcher = watch(path)
     watcher.on('change', async () => {

src/components/config/parse-sitemap.ts

@@ -24,11 +24,12 @@
 
 import type { Config } from '../../interfaces/config'
 import { XMLParser } from 'fast-xml-parser'
+import { getContext } from '../../context'
 import { monikaFlagsDefaultValue } from '../../flag'
 import type { MonikaFlags } from '../../flag'
 import type { Probe, ProbeAlert } from '../../interfaces/probe'
+import type { RequestConfig } from '../../interfaces/request'
 import Joi from 'joi'
-import { RequestConfig } from 'src/interfaces/request'
 
 const sitemapValidator = Joi.object({
   config: Joi.object({
@@ -95,6 +96,7 @@ const generateProbesFromXmlOneProbe = (parseResult: unknown) => {
         method: 'GET',
         timeout: 10_000,
         body: '',
+        followRedirects: getContext().flags['follow-redirects'],
       },
     ]
     if (item['xhtml:link']) {
@@ -106,6 +108,7 @@ const generateProbesFromXmlOneProbe = (parseResult: unknown) => {
             method: 'GET',
             timeout: 10_000,
             body: '',
+            followRedirects: getContext().flags['follow-redirects'],
           },
         ]
       }

src/components/config/parse-text.ts

@@ -46,6 +46,7 @@
  * SOFTWARE.                                                                      *
  **********************************************************************************/
 
+import { getContext } from '../../context'
 import type { Config } from '../../interfaces/config'
 import { monikaFlagsDefaultValue } from '../../flag'
 import type { Probe, ProbeAlert } from '../../interfaces/probe'
@@ -67,6 +68,7 @@ export const parseConfigFromText = (configString: string): Config => {
               method: 'GET',
               timeout: 10_000,
               body: {} as JSON,
+              followRedirects: getContext().flags['follow-redirects'],
             },
           ],
           interval: monikaFlagsDefaultValue['config-interval'],

src/components/config/validate.test.ts

@@ -0,0 +1,97 @@
+/**********************************************************************************
+ * MIT License                                                                    *
+ *                                                                                *
+ * Copyright (c) 2021 Hyperjump Technology                                        *
+ *                                                                                *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy   *
+ * of this software and associated documentation files (the "Software"), to deal  *
+ * in the Software without restriction, including without limitation the rights   *
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      *
+ * copies of the Software, and to permit persons to whom the Software is          *
+ * furnished to do so, subject to the following conditions:                       *
+ *                                                                                *
+ * The above copyright notice and this permission notice shall be included in all *
+ * copies or substantial portions of the Software.                                *
+ *                                                                                *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     *
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       *
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    *
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         *
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  *
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  *
+ * SOFTWARE.                                                                      *
+ **********************************************************************************/
+
+import { expect } from '@oclif/test'
+
+import { getContext, setContext } from '../../context'
+import type { Probe } from '../../interfaces/probe'
+import { validateProbes } from './validation'
+
+describe('Configuration validation', () => {
+  it('should use default follow redirect from flag', async () => {
+    // arrange
+    const probes: Probe[] = [
+      {
+        id: 'hrf6g',
+        requests: [
+          {
+            body: '',
+            url: 'https://example.com',
+            timeout: 1000,
+          },
+        ],
+      } as Probe,
+    ]
+    // act
+    const validatedProbes = await validateProbes(probes)
+
+    // assert
+    expect(validatedProbes[0].requests![0].followRedirects).eq(21)
+  })
+
+  it('should use follow redirect from flag', async () => {
+    // arrange
+    setContext({ flags: { ...getContext().flags, 'follow-redirects': 0 } })
+    const probes: Probe[] = [
+      {
+        id: 'hrf6g',
+        requests: [
+          {
+            body: '',
+            url: 'https://example.com',
+            timeout: 1000,
+          },
+        ],
+      } as Probe,
+    ]
+    // act
+    const validatedProbes = await validateProbes(probes)
+
+    // assert
+    expect(validatedProbes[0].requests![0].followRedirects).eq(0)
+  })
+
+  it('should use follow redirect from config', async () => {
+    // arrange
+    setContext({ flags: { ...getContext().flags, 'follow-redirects': 0 } })
+    const probes: Probe[] = [
+      {
+        id: 'hrf6g',
+        requests: [
+          {
+            body: '',
+            followRedirects: 1,
+            url: 'https://example.com',
+            timeout: 1000,
+          },
+        ],
+      } as Probe,
+    ]
+    // act
+    const validatedProbes = await validateProbes(probes)
+
+    // assert
+    expect(validatedProbes[0].requests![0].followRedirects).eq(1)
+  })
+})

src/components/config/validation/validator/probe.test.ts

@@ -28,7 +28,7 @@ import type { Probe } from '../../../../interfaces/probe'
 
 import { FAILED_REQUEST_ASSERTION } from '../../../../looper'
 import { validateProbes } from './probe'
-import { resetContext, setContext } from '../../../../context'
+import { getContext, resetContext, setContext } from '../../../../context'
 import type { MonikaFlags } from '../../../../flag'
 
 describe('Probe validation', () => {
@@ -175,6 +175,7 @@ describe('Probe validation', () => {
       // arrange
       setContext({
         flags: {
+          ...getContext().flags,
           symonKey: 'bDF8j',
           symonUrl: 'https://example.com',
         } as MonikaFlags,

src/components/config/validation/validator/probe.ts

@@ -259,6 +259,10 @@ export async function validateProbes(probes: Probe[]): Promise<Probe[]> {
                     joi.number(),
                     joi.bool()
                   ),
+                followRedirects: joi
+                  .number()
+                  .min(0)
+                  .default(getContext().flags['follow-redirects']),
                 headers: joi.object().allow(null),
                 id: joi.string().allow(''),
                 interval: joi.number().min(1),

src/components/logger/startup-message.test.ts

@@ -37,7 +37,13 @@ const defaultConfig: Config = {
       name: 'Acme Inc.',
       interval: 3000,
       requests: [
-        { url: 'https://example.com', headers: {}, body: '', timeout: 0 },
+        {
+          url: 'https://example.com',
+          headers: {},
+          body: '',
+          followRedirects: 21,
+          timeout: 0,
+        },
       ],
       alerts: [
         {
@@ -240,6 +246,7 @@ describe('Startup message', () => {
                     url: 'https://example.com',
                     headers: {},
                     body: '',
+                    followRedirects: 21,
                     timeout: 0,
                   },
                 ],

src/components/notification/alert-message.ts

@@ -32,14 +32,15 @@ import { getContext } from '../../context'
 import type { NotificationMessage } from '@hyperjumptech/monika-notification'
 import { ProbeRequestResponse } from '../../interfaces/request'
 import { ProbeAlert } from '../../interfaces/probe'
-import { publicIpAddress, publicNetworkInfo } from '../../utils/public-ip'
+import { getPublicNetworkInfo, publicIpAddress } from '../../utils/public-ip'
 import { getIncidents } from '../incident'
 
 const getLinuxDistro = promisify(getos)
 
 export const getMonikaInstance = async (ipAddress: string): Promise<string> => {
   const osHostname = hostname()
 
+  const publicNetworkInfo = getPublicNetworkInfo()
   if (publicNetworkInfo) {
     const { city, isp } = publicNetworkInfo