feat(#425): E2E test actions (#620)

* feat(#425): Create e2e test setup

* Setup mocha hooks and add to npm run command

* orchestrate with mocha hooks

* test run in the ci

* test run in the ci

* move session-token test to test/integration folder

* add trace logs

* add trace logs

* remove traces

* split logic if we have an existing project configuration file

* refactor hooks and extract utils functions to deal with cht-docker-helper

* refactor `spinUpCht` to fix eslint warning about promise executor cannot be async

* finally get the first e2e test out, working as expected

* trace errors in CI

* check for existence of project configuration before running teardown

* add more traces

* do we have the latest version of the script?

* what's wrong with `docker exec`?

* create make parent directories as needed same as `mkdir -p`

* organize todos, remove debugging logs

* replace 4 spaces => 2 spaces to follow coding style in the repo

* oops forgot these

* extract utils functions to reuse in other tests

* logs

* pass project name to `runChtConf` as expected

* await getProjectUrl()

* sonar :)

* clean up

* more clean up

* clearer test title

* - switch back to cht-docker-compose.sh from cht-core master
- hardcode local-ip IP address in the CI job

* add trace

* clean up trace

* replace hardcoded package.json name

* add comments to explain the rationale behind the `stdio` option when running the docker helper script

* remove linting before running e2e tests

* increase timeout to prevent frequent failures due to CHT instance taking too long to be ready

* remove `.cht-docker-helper` in teardown

* fix import of `DEFAULT_PROJECT_NAME`

* dedup code in `initProject`

* remove unnecessary `structuredClone`

* add assertions about `baseSettings.language`

* throw error early when config file doesn't exist

* extract `readCompiledAppSettings` & `writeBaseAppSettings` cht conf utils

* touch a word about e2e tests in readme

* it's better with the right npm script...

* format

---------

Co-authored-by: Sugat Bajracharya <[email protected]>

.github/workflows/build.yml

@@ -30,3 +30,22 @@ jobs:
           coverage
           .nyc_output
       if: ${{ failure() }}
+
+  e2e:
+    name: E2E tests
+    runs-on: ubuntu-22.04
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18.x
+    - name: Install dependencies
+      run: |
+        pip install git+https://github.com/medic/[email protected]#egg=pyxform-medic
+        npm ci
+    - name: Hard code local-ip IP in /etc/hosts per https://github.com/medic/medic-infrastructure/issues/571#issuecomment-2209120441
+      run: |
+        echo "15.188.129.97 local-ip.medicmobile.org" | sudo tee -a /etc/hosts
+    - name: Run E2E tests
+      run: npm run test-e2e

.gitignore

@@ -10,3 +10,4 @@ coverage
 .nyc_output
 .DS_Store
 test/.DS_Store
+test/e2e/.cht-docker-helper

README.md

@@ -303,8 +303,16 @@ To develop a new action or improve an existing one, check the ["Actions" doc](sr
 
 ## Testing
 
+### Unit tests
+
 Execute `npm test` to run static analysis checks and the test suite. Requires Docker to run integration tests against a CouchDB instance. 
 
+### End-to-end tests
+
+Run `npm run test-e2e` to run the end-to-end test suite against an actual CHT instance locally. These tests rely on [CHT Docker Helper](https://docs.communityhealthtoolkit.org/hosting/4.x/app-developer/#cht-docker-helper-for-4x) to spin up and tear down an instance locally.
+
+The code interfacing with CHT Docker Helper lives in [`test/e2e/cht-docker-utils.js`](./test/e2e/cht-docker-utils.js). You should rely on the API exposed by this file to orchestrate CHT instances for testing purposes. It is preferable to keep the number of CHT instances orchestrated in E2E tests low as it takes a non-negligible amount of time to spin up an instance and can quickly lead to timeouts.
+
 ## Executing your local branch
 
 1. Clone the project locally

package.json

@@ -11,7 +11,8 @@
     "eslint": "eslint 'src/**/*.js' test/*.js 'test/**/*.js'",
     "docker-start-couchdb": "npm run docker-stop-couchdb && docker run -d -p 6984:5984 --rm --name cht-conf-couchdb couchdb:2.3.1 && sh test/scripts/wait_for_response_code.sh 6984 200 CouchDB",
     "docker-stop-couchdb": "docker stop cht-conf-couchdb || true",
-    "test": "npm run eslint && npm run docker-start-couchdb && npm run clean && mkdir -p build/test && cp -r test/data build/test/data && cd build/test && nyc --reporter=html mocha --forbid-only \"../../test/**/*.spec.js\" && cd ../.. && npm run docker-stop-couchdb",
+    "test": "npm run eslint && npm run docker-start-couchdb && npm run clean && mkdir -p build/test && cp -r test/data build/test/data && cd build/test && nyc --reporter=html mocha --forbid-only \"../../test/**/*.spec.js\" --exclude \"../../test/e2e/**/*.spec.js\" && cd ../.. && npm run docker-stop-couchdb",
+    "test-e2e": "mocha --config test/e2e/.mocharc.js",
     "semantic-release": "semantic-release"
   },
   "bin": {

test/e2e/.mocharc.js

@@ -0,0 +1,14 @@
+module.exports = {
+  allowUncaught: false,
+  color: true,
+  checkLeaks: true,
+  fullTrace: true,
+  asyncOnly: false,
+  spec: ['test/e2e/**/*.spec.js'],
+  timeout: 120_000, // spinning up a CHT instance takes a little long
+  reporter: 'spec',
+  file: ['test/e2e/hooks.js'],
+  captureFile: 'test/e2e/results.txt',
+  exit: true,
+  recursive: true,
+};

test/e2e/cht-conf-utils.js

@@ -0,0 +1,76 @@
+const path = require('path');
+const { exec } = require('child_process');
+const fs = require('fs');
+const fse = require('fs-extra');
+
+const log = require('../../src/lib/log');
+const { getProjectUrl } = require('./cht-docker-utils');
+
+const getProjectDirectory = (projectName) => path.resolve(__dirname, `../../build/${projectName}`);
+
+const runChtConf = (projectName, command) => new Promise((resolve, reject) => {
+  getProjectUrl(projectName).then(url => {
+    const projectDirectory = getProjectDirectory(projectName);
+    const cliPath = path.join(__dirname, '../../src/bin/index.js');
+    exec(`node ${cliPath} --url=${url} ${command}`, { cwd: projectDirectory }, (error, stdout, stderr) => {
+      if (!error) {
+        return resolve(stdout);
+      }
+
+      log.error(stderr);
+      reject(new Error(stdout.toString('utf8')));
+    });
+  });
+});
+
+const cleanupProject = (projectName) => {
+  const projectDirectory = getProjectDirectory(projectName);
+  if (fs.existsSync(projectDirectory)) {
+    fse.removeSync(projectDirectory);
+  }
+};
+
+const initProject = async (projectName) => {
+  const projectDirectory = getProjectDirectory(projectName);
+  cleanupProject(projectName);
+
+  fse.mkdirpSync(projectDirectory);
+  fs.writeFileSync(
+    path.join(projectDirectory, 'package.json'),
+    JSON.stringify({
+      name: projectName,
+      version: '1.0.0',
+      dependencies: {
+        'cht-conf': 'file:../..',
+      },
+    }, null, 4),
+  );
+
+  await runChtConf(projectName, 'initialise-project-layout');
+};
+
+const writeBaseAppSettings = async (projectName, baseSettings) => {
+  const projectDirectory = getProjectDirectory(projectName);
+
+  return await fs.promises.writeFile(
+    path.join(projectDirectory, 'app_settings/base_settings.json'),
+    JSON.stringify(baseSettings, null, 2),
+  );
+};
+
+const readCompiledAppSettings = async (projectName) => {
+  const projectDirectory = getProjectDirectory(projectName);
+
+  return JSON.parse(
+    await fs.promises.readFile(path.join(projectDirectory, 'app_settings.json'), 'utf8')
+  );
+};
+
+module.exports = {
+  cleanupProject,
+  getProjectDirectory,
+  initProject,
+  runChtConf,
+  readCompiledAppSettings,
+  writeBaseAppSettings,
+};

test/e2e/cht-docker-utils.js

@@ -0,0 +1,129 @@
+const path = require('path');
+const fs = require('fs');
+const https = require('https');
+const { spawn } = require('child_process');
+const fse = require('fs-extra');
+const request = require('request-promise-native');
+
+const log = require('../../src/lib/log');
+
+const DEFAULT_PROJECT_NAME = 'cht_conf_e2e_tests';
+const dockerHelperDirectory = path.resolve(__dirname, '.cht-docker-helper');
+const dockerHelperScript = path.resolve(dockerHelperDirectory, './cht-docker-compose.sh');
+
+const downloadDockerHelperScript = () => new Promise((resolve, reject) => {
+  const file = fs.createWriteStream(dockerHelperScript, { mode: 0o755 });
+  https
+    .get('https://raw.githubusercontent.com/medic/cht-core/master/scripts/docker-helper-4.x/cht-docker-compose.sh', (response) => {
+      response.pipe(file);
+      file.on('finish', () => file.close(resolve));
+      file.on('error', () => file.close(reject));
+    })
+    .on('error', () => {
+      fs.unlinkSync(file.path);
+      file.close(() => reject('Failed to download CHT Docker Helper script "cht-docker-compose.sh"'));
+    });
+});
+
+const ensureScriptExists = async () => {
+  if (!fs.existsSync(dockerHelperDirectory)) {
+    await fs.promises.mkdir(dockerHelperDirectory);
+  }
+
+  if (!fs.existsSync(dockerHelperScript)) {
+    await downloadDockerHelperScript();
+  }
+};
+
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+const getProjectConfig = async (projectName) => {
+  const configFilePath = path.resolve(dockerHelperDirectory, `${projectName}.env`);
+  if (!fs.existsSync(configFilePath)) {
+    throw new Error(`Unexpected error: config file not found at ${configFilePath}`);
+  }
+
+  const configFile = await fs.promises.readFile(configFilePath, 'utf8');
+  return Object.fromEntries(
+    configFile.toString()
+      .split('\n')
+      .map(line => line.split('='))
+      .filter(entry => entry.length === 2),
+  );
+};
+
+const getProjectUrl = async (projectName = DEFAULT_PROJECT_NAME) => {
+  const config = await getProjectConfig(projectName);
+  const { COUCHDB_USER, COUCHDB_PASSWORD, NGINX_HTTPS_PORT } = config;
+  return `https://${COUCHDB_USER}:${COUCHDB_PASSWORD}@127-0-0-1.local-ip.medicmobile.org:${NGINX_HTTPS_PORT}`;
+};
+
+const isProjectReady = async (projectName, attempt = 1) => {
+  log.info(`Checking if CHT is ready, attempt ${attempt}.`);
+  const url = await getProjectUrl(projectName);
+  await request({ uri: `${url}/api/v2/monitoring`, json: true })
+    .catch(async (error) => {
+      if (
+        error.error.code !== 'DEPTH_ZERO_SELF_SIGNED_CERT' ||
+        ![502, 503].includes(error.statusCode)
+      ) {
+        // unexpected error, log it to keep a trace,
+        // but we'll keep retrying until the instance is up, or we hit the timeout limit
+        log.trace(error);
+      }
+
+      await sleep(1000);
+      return isProjectReady(projectName, attempt + 1);
+    });
+};
+
+const startProject = (projectName) => new Promise((resolve, reject) => {
+  log.info(`Starting CHT instance "${projectName}"`);
+
+  // stdio: 'pipe' to answer the prompts to initialize a project by writing to stdin
+  const childProcess = spawn(dockerHelperScript, { stdio: 'pipe', cwd: dockerHelperDirectory });
+  childProcess.on('error', reject);
+  childProcess.on('close', async () => {
+    await isProjectReady(projectName);
+    resolve();
+  });
+
+  childProcess.stdin.write('y\n');
+  childProcess.stdin.write('y\n');
+  childProcess.stdin.write(`${projectName}\n`);
+});
+
+const destroyProject = (projectName) => new Promise((resolve, reject) => {
+  // stdio: 'inherit' to see the script's logs and understand why it requests elevated permissions when cleaning up project files
+  const childProcess = spawn(dockerHelperScript, [`${projectName}.env`, 'destroy'], {
+    stdio: 'inherit',
+    cwd: dockerHelperDirectory,
+  });
+  childProcess.on('error', reject);
+  childProcess.on('close', resolve);
+});
+
+const spinUpCht = async (projectName = DEFAULT_PROJECT_NAME) => {
+  await ensureScriptExists();
+  await startProject(projectName);
+};
+
+const tearDownCht = async (projectName = DEFAULT_PROJECT_NAME) => {
+  if (!fs.existsSync(dockerHelperDirectory)) {
+    return;
+  }
+
+  if (fs.existsSync(path.resolve(dockerHelperDirectory, `${projectName}.env`))) {
+    await ensureScriptExists();
+    await destroyProject(projectName);
+  }
+
+  fse.removeSync(dockerHelperDirectory);
+};
+
+module.exports = {
+  DEFAULT_PROJECT_NAME,
+  getProjectUrl,
+  spinUpCht,
+  tearDownCht,
+};

test/e2e/edit-app-settings.spec.js

@@ -0,0 +1,60 @@
+const { expect } = require('chai');
+const request = require('request-promise-native');
+
+const { DEFAULT_PROJECT_NAME, getProjectUrl } = require('./cht-docker-utils');
+const {
+  cleanupProject,
+  initProject,
+  runChtConf,
+  readCompiledAppSettings,
+  writeBaseAppSettings,
+} = require('./cht-conf-utils');
+
+describe('edit-app-settings', () => {
+  const projectName = DEFAULT_PROJECT_NAME;
+
+  before(async () => {
+    await initProject(projectName);
+  });
+
+  after(async () => {
+    await cleanupProject(projectName);
+  });
+
+  it('disables a language, recompile, and push app settings', async () => {
+    const url = await getProjectUrl(projectName);
+    const baseSettings = await request.get({ url: `${url}/api/v1/settings`, json: true });
+    baseSettings.languages.forEach(language => expect(language.enabled).to.be.true);
+    expect(baseSettings.locale).to.equal('en');
+    expect(baseSettings.locale_outgoing).to.equal('en');
+
+    baseSettings.languages = baseSettings.languages.map(language => {
+      if (language.locale === 'en') {
+        language.enabled = false;
+      }
+
+      return language;
+    });
+    baseSettings.locale = 'fr';
+    baseSettings.locale_outgoing = 'fr';
+    await writeBaseAppSettings(projectName, baseSettings);
+
+    await runChtConf(projectName, 'compile-app-settings');
+    const compiledSettings = await readCompiledAppSettings(projectName);
+    expect(compiledSettings.languages.find(language => language.locale === 'en')).to.deep.equal({
+      locale: 'en',
+      enabled: false,
+    });
+    expect(compiledSettings.locale).to.equal('fr');
+    expect(compiledSettings.locale_outgoing).to.equal('fr');
+
+    await runChtConf(projectName, 'upload-app-settings');
+    const newSettings = await request.get({ url: `${url}/api/v1/settings`, json: true });
+    expect(newSettings.languages.find(language => language.locale === 'en')).to.deep.equal({
+      locale: 'en',
+      enabled: false,
+    });
+    expect(newSettings.locale).to.equal('fr');
+    expect(newSettings.locale_outgoing).to.equal('fr');
+  });
+});

test/e2e/hooks.js

@@ -0,0 +1,11 @@
+const { spinUpCht, tearDownCht } = require('./cht-docker-utils');
+
+before(async () => {
+  // cleanup eventual leftovers before starting
+  await tearDownCht();
+  await spinUpCht();
+});
+
+after(async () => {
+  await tearDownCht();
+});

test/e2e/session-token.spec.js → test/integration/session-token.spec.js

@@ -56,7 +56,7 @@ const runCliCommand = (command) => {
   });
 };
 
-describe('e2e/session-token', function() {
+describe('integration/session-token', function() {
   this.timeout(15000);
 
   let sessionToken;
@@ -149,4 +149,4 @@ describe('e2e/session-token', function() {
       // Bad Request: Malformed AuthSession cookie
       .that.contains('INFO Error: Received error code 400');
   });
-});
+});