Transform Security's Cypress Junit reports to have the same format as…

… those produced by FTR and Jest (elastic#162279)

## Summary

Cypress produced Junit reports, but the failed-test-reporter and various
github and kibanamachine workflows rely on a specifically formatted
junit report that includes an encoded version of the spec file path in
each testcase.

For FTR and Jest, these specially formatted junit reports are created by
a custom reporter. Due to the architecture of Cypress, re-using those
would be difficult. Instead this PR adds a script that reads,
transforms, and updates all the junit reports created by Cypress.

### TODO
Some work is not covered in this PR. I need to merge this change to test
that flaky test triaging works in buildkite and kibana machine (note: if
you know how to validate this without merging it, please reach out!)
After I'm confident that this works, I'll open follow up PRs to do the
following:

```[tasklist]
### Follow up work
- [ ] Enable this script for test_serverless cypress tests
- [ ] Enable this script for threat intelligence cypress tests (optional)
- [ ] Enable this script for fleet (optional)
```

### For maintainers

- [ ] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: kibanamachine <[email protected]>

x-pack/plugins/security_solution/jest.config.dev.js

@@ -12,5 +12,6 @@ module.exports = {
     '<rootDir>/x-pack/plugins/security_solution/common/*/jest.config.js',
     '<rootDir>/x-pack/plugins/security_solution/server/*/jest.config.js',
     '<rootDir>/x-pack/plugins/security_solution/public/*/jest.config.js',
+    '<rootDir>/x-pack/plugins/security_solution/scripts/junit_transformer/*/jest.config.js',
   ],
 };

x-pack/plugins/security_solution/package.json

@@ -25,9 +25,10 @@
     "cypress:dw:endpoint:open": "node ./scripts/start_cypress_parallel open --config-file ./public/management/cypress_endpoint.config.ts ts --ftr-config-file ../../../../../../x-pack/test/defend_workflows_cypress/endpoint_config",
     "cypress:investigations:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/investigations/**/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
     "cypress:explore:run": "yarn cypress:run:reporter --browser chrome --spec './cypress/e2e/explore/**/*.cy.ts' --ftr-config-file ../../../../../../x-pack/test/security_solution_cypress/cli_config; status=$?; yarn junit:merge && exit $status",
-    "junit:merge": "../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-security-solution/cypress/results/mochawesome*.json > ../../../target/kibana-security-solution/cypress/results/output.json && ../../../node_modules/.bin/marge ../../../target/kibana-security-solution/cypress/results/output.json --reportDir ../../../target/kibana-security-solution/cypress/results && mkdir -p ../../../target/junit && cp ../../../target/kibana-security-solution/cypress/results/*.xml ../../../target/junit/",
+    "junit:merge": "../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-security-solution/cypress/results/mochawesome*.json > ../../../target/kibana-security-solution/cypress/results/output.json && ../../../node_modules/.bin/marge ../../../target/kibana-security-solution/cypress/results/output.json --reportDir ../../../target/kibana-security-solution/cypress/results && yarn junit:transform && mkdir -p ../../../target/junit && cp ../../../target/kibana-security-solution/cypress/results/*.xml ../../../target/junit/",
     "test:generate": "node scripts/endpoint/resolver_generator",
     "mappings:generate": "node scripts/mappings/mappings_generator",
-    "mappings:load": "node scripts/mappings/mappings_loader"
+    "mappings:load": "node scripts/mappings/mappings_loader",
+    "junit:transform": "node scripts/junit_transformer --pathPattern '../../../target/kibana-security-solution/cypress/results/*.xml' --rootDirectory ../../../ --reportName 'Security Solution Cypress' --writeInPlace"
   }
 }

x-pack/plugins/security_solution/scripts/jest.config.js

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../..',
+  roots: ['<rootDir>/x-pack/plugins/security_solution/scripts'],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/security_solution/scripts',
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: ['<rootDir>/x-pack/plugins/security_solution/scripts/**/*.{ts,tsx}'],
+};

x-pack/plugins/security_solution/scripts/junit_transformer/README.md

@@ -0,0 +1,9 @@
+The failed test reporter creates github issues based on junit reports. Github workflows, and kibanamachine workflows, allow the Kibana Operations team to track and triage flaky tests. These workflows rely on those github issues, specifically their titles, to work. The titles of the github issues contain an encoded version of the file path that contains the failing test. 
+
+This process is facilitated by custom mocha/junit reporters written for the functional test runner and jest. These reporters encode the file name of each spec file and include it in an attribute on elements in the junit report.
+
+There is no such custom mocha reporter for Cypress, and due to the architecture of Cypress, reusing the existing custom mocha reports, or any of their existing code, is not feasible. Cypress runs in its own process, with its own version of node, and that environment is incompatible with running babel-register. This means we cannot easily interpret the code that implements the existing custom mocha reporters from within Cypress.
+
+We could compile a library using the code from those custom junit reporters, but there is no established pattern or tooling for doing that.
+
+For there reasons, our approach is to transform the junit report created by Cypress into a format consumable by the failed test reporter and the kibana operations triage scripts. This script does that.

x-pack/plugins/security_solution/scripts/junit_transformer/fixtures/suite_with_failing_test.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<testsuites name="Mocha Tests" time="350.7660" tests="8" failures="1">
+  <testsuite name="Root Suite" timestamp="2023-07-20T18:55:26" tests="0" file="cypress/e2e/urls/compatibility.cy.ts" time="0.0000" failures="0"/>
+  <testsuite name="URL compatibility" timestamp="2023-07-20T18:55:26" tests="8" time="350.7620" failures="1">
+    <testcase name="URL compatibility Redirects to alerts from old siem Detections URL Redirects to alerts from old siem Detections URL" time="10.4490" classname="Security Solution Cypress.x-pack/plugins/security_solution/cypress/e2e/urls/compatibility·cy·ts"/>
+    <testcase name="URL compatibility Redirects to alerts from old Detections URL Redirects to alerts from old Detections URL" time="5.9170" classname="Security Solution Cypress.x-pack/plugins/security_solution/cypress/e2e/urls/compatibility·cy·ts"/>
+    <testcase name="URL compatibility Redirects to rules from old Detections rules URL Redirects to rules from old Detections rules URL" time="5.1660" classname="Security Solution Cypress.x-pack/plugins/security_solution/cypress/e2e/urls/compatibility·cy·ts"/>
+    <testcase name="URL compatibility Redirects to rules creation from old Detections rules creation URL Redirects to rules creation from old Detections rules creation URL" time="152.5400" classname="Security Solution Cypress.x-pack/plugins/security_solution/cypress/e2e/urls/compatibility·cy·ts">
+      <failure message="Timed out retrying after 150000ms: expected 'http://localhost:5647/app/security/rules/create' to include 'app/security/rules/create1'" type="AssertionError">AssertionError: Timed out retrying after 150000ms: expected 'http://localhost:5647/app/security/rules/create' to include 'app/security/rules/create1'
+    at Context.eval (webpack:///./e2e/urls/compatibility.cy.ts:65:13)</failure>
+    </testcase>
+    <testcase name="URL compatibility Redirects to rule details from old Detections rule details URL Redirects to rule details from old Detections rule details URL" time="5.6040" classname="Security Solution Cypress.x-pack/plugins/security_solution/cypress/e2e/urls/compatibility·cy·ts"/>
+    <testcase name="URL compatibility Redirects to rule details alerts tab from old Detections rule details URL Redirects to rule details alerts tab from old Detections rule details URL" time="6.0160" classname="Security Solution Cypress.x-pack/plugins/security_solution/cypress/e2e/urls/compatibility·cy·ts"/>
+    <testcase name="URL compatibility Redirects to rule edit from old Detections rule edit URL Redirects to rule edit from old Detections rule edit URL" time="5.5470" classname="Security Solution Cypress.x-pack/plugins/security_solution/cypress/e2e/urls/compatibility·cy·ts"/>
+    <testcase name="URL compatibility sets the global start and end dates from the url with timestamps sets the global start and end dates from the url with timestamps" time="5.8890" classname="Security Solution Cypress.x-pack/plugins/security_solution/cypress/e2e/urls/compatibility·cy·ts"/>
+  </testsuite>
+</testsuites>

x-pack/plugins/security_solution/scripts/junit_transformer/index.js

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+require('../../../../../src/setup_node_env');
+require('./junit_transformer');

x-pack/plugins/security_solution/scripts/junit_transformer/junit_transformer.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { promises as fs } from 'fs';
+import { mkdtemp } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import type { CommandArgs } from './lib';
+import { command } from './lib';
+
+describe('junit_transformer', () => {
+  const junitFileName = 'junit.xml';
+  let pathPattern: string;
+  let path: string;
+  let mockCommandArgs: CommandArgs;
+
+  beforeEach(async () => {
+    // get a temporary directory
+    const directory = await mkdtemp(join(tmpdir(), 'junit-transformer-test-'));
+
+    // define a glob pattern that will match the fixture
+    pathPattern = `${directory}/*`;
+
+    // determine the path for the fixture
+    path = join(directory, junitFileName);
+
+    // read the fixture and write it to the temporary file
+    await fs.writeFile(
+      path,
+      await fs.readFile(join(__dirname, './fixtures/suite_with_failing_test.xml'), {
+        encoding: 'utf8',
+      })
+    );
+
+    mockCommandArgs = {
+      // define the flags that will be passed to the command
+      flags: {
+        pathPattern,
+        // use the directory as the root directory. This lets us test the relative file path functionality without having a tree of temp files.
+        rootDirectory: directory,
+        reportName: 'Test',
+        writeInPlace: true,
+      },
+
+      log: {
+        info: jest.fn(),
+        write: jest.fn(),
+        error: jest.fn(),
+        success: jest.fn(),
+        warning: jest.fn(),
+      },
+    };
+  });
+  it('updates the file in place, applying the expected transformation', async () => {
+    await command(mockCommandArgs);
+    expect(await fs.readFile(path, { encoding: 'utf8' })).toMatchSnapshot();
+  });
+});

x-pack/plugins/security_solution/scripts/junit_transformer/junit_transformer.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { run } from '@kbn/dev-cli-runner';
+import { command } from './lib';
+
+/**
+ * This script processes all junit reports matching a glob pattern. It reads each report, parses it into json, validates that it is a report from Cypress, then transforms the report to a form that can be processed by Kibana Operations workflows and the failed-test-reporter, it then optionally writes the report back, in xml format, to the original file path.
+ */
+run(command, {
+  description: `
+      Transform junit reports to match the style required by the Kibana Operations flaky test triage workflows such as '/skip'.
+    `,
+  flags: {
+    string: ['pathPattern', 'rootDirectory', 'reportName'],
+    boolean: ['writeInPlace'],
+    help: `
+        --pathPattern      Required, glob passed to globby to select files to operate on
+        --rootDirectory    Required, path of the kibana repo. Used to calcuate the file path of each spec file relative to the Kibana repo
+        --reportName       Required, used as a prefix for the classname. Eventually shows up in the title of flaky test Github issues
+        --writeInPlace     Defaults to false. If passed, rewrite the file in place with transformations. If false, the script will pass the transformed XML as a string to stdout
+
+      If an error is encountered when processing one file, the script will still attempt to process other files.
+      `,
+  },
+});