[Reporting] revive "existing indices" functional test

x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+/**
+ * This file tests the situation when a reporting index spans releases. By default reporting indexes are created
+ * on a weekly basis, but this is configurable so it is possible a user has this set to yearly. In that event, it
+ * is possible report data is getting posted to an index that was created by a very old version. We don't have a
+ * reporting index migration plan, so this test is important to ensure BWC, or that in the event we decide to make
+ * a major change in a major release, we handle it properly.
+ */
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ getService }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const reportingAPI = getService('reportingAPI');
+
+  describe('BWC report generation into existing indexes', () => {
+    let cleanupIndexAlias: () => Promise<void>;
+
+    describe('existing 6_2 index', () => {
+      before('load data and add index alias', async () => {
+        await reportingAPI.deleteAllReports();
+        // data to report on
+        await esArchiver.load('test/functional/fixtures/es_archiver/logstash_functional');
+        await esArchiver.load('test/functional/fixtures/es_archiver/discover'); // includes index pattern for logstash_functional
+        // archive with reporting index mappings v6.2
+        await esArchiver.load('x-pack/test/functional/es_archives/reporting/bwc/6_2');
+
+        // The index name in the reporting/bwc/6_2 archive.
+        const ARCHIVED_REPORTING_INDEX = '.reporting-2018.03.11';
+        // causes reporting to assume the v6.2 index is the one to use for new jobs posted
+        cleanupIndexAlias = await reportingAPI.coerceReportsIntoExistingIndex(
+          ARCHIVED_REPORTING_INDEX
+        );
+      });
+
+      after('remove index alias', async () => {
+        await esArchiver.load('test/functional/fixtures/es_archiver/logstash_functional');
+        await esArchiver.load('test/functional/fixtures/es_archiver/discover');
+
+        await cleanupIndexAlias();
+        await esArchiver.unload('x-pack/test/functional/es_archives/reporting/bwc/6_2');
+      });
+
+      it('single job posted can complete in an index created with an older version', async () => {
+        const reportPaths = [];
+        reportPaths.push(
+          await reportingAPI.postJob(
+            '/api/reporting/generate/csv_searchsource?jobParams=%28browserTimezone%3AAmerica%2FPhoenix%2Ccolumns%3A%21%28%29%2CobjectType%3Asearch%2CsearchSource%3A%28fields%3A%21%28%28field%3A%27%2A%27%2Cinclude_unmapped%3Atrue%29%29%2Cfilter%3A%21%28%28meta%3A%28index%3A%27logstash-%2A%27%2Cparams%3A%28%29%29%2Crange%3A%28%27%40timestamp%27%3A%28format%3Astrict_date_optional_time%2Cgte%3A%272015-09-20T16%3A00%3A56.290Z%27%2Clte%3A%272015-09-21T10%3A37%3A45.066Z%27%29%29%29%29%2Cindex%3A%27logstash-%2A%27%2Cparent%3A%28filter%3A%21%28%29%2Cindex%3A%27logstash-%2A%27%2Cquery%3A%28language%3Akuery%2Cquery%3A%27%27%29%29%2Csort%3A%21%28%28%27%40timestamp%27%3Adesc%29%29%2CtrackTotalHits%3A%21t%2Cversion%3A%21t%29%2Ctitle%3A%27Discover%20search%20%5B2021-07-30T11%3A47%3A03.731-07%3A00%5D%27%29'
+          )
+        );
+        await reportingAPI.expectAllJobsToFinishSuccessfully(reportPaths);
+      }).timeout(1540000);
+    });
+  });
+}

x-pack/test/reporting_api_integration/reporting_and_security/index.ts

@@ -20,6 +20,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
       await reportingAPI.createTestReportingUser();
     });
 
+    loadTestFile(require.resolve('./bwc_existing_indexes'));
     loadTestFile(require.resolve('./security_roles_privileges'));
     loadTestFile(require.resolve('./download_csv_dashboard'));
     loadTestFile(require.resolve('./generate_csv_discover'));

x-pack/test/reporting_api_integration/services/usage.ts

@@ -6,6 +6,7 @@
  */
 
 import expect from '@kbn/expect';
+import { indexTimestamp } from '../../../plugins/reporting/server/lib/store/index_timestamp';
 import { FtrProviderContext } from '../ftr_provider_context';
 
 interface PDFAppCounts {
@@ -37,6 +38,7 @@ export interface UsageStats {
 
 export function createUsageServices({ getService }: FtrProviderContext) {
   const log = getService('log');
+  const esSupertest = getService('esSupertest');
   const supertest = getService('supertest');
 
   return {
@@ -67,6 +69,42 @@ export function createUsageServices({ getService }: FtrProviderContext) {
       expect(statusCode).to.be(200);
     },
 
+    /**
+     *
+     * @return {Promise<Function>} A function to call to clean up the index alias that was added.
+     */
+    async coerceReportsIntoExistingIndex(indexName: string) {
+      log.debug(`ReportingAPI.coerceReportsIntoExistingIndex(${indexName})`);
+
+      // Adding an index alias coerces the report to be generated on an existing index which means any new
+      // index schema won't be applied. This is important if a point release updated the schema. Reports may still
+      // be inserted into an existing index before the new schema is applied.
+      const timestampForIndex = indexTimestamp('week', '.');
+      await esSupertest
+        .post('/_aliases')
+        .send({
+          actions: [
+            {
+              add: { index: indexName, alias: `.reporting-${timestampForIndex}` },
+            },
+          ],
+        })
+        .expect(200);
+
+      return async () => {
+        await esSupertest
+          .post('/_aliases')
+          .send({
+            actions: [
+              {
+                remove: { index: indexName, alias: `.reporting-${timestampForIndex}` },
+              },
+            ],
+          })
+          .expect(200);
+      };
+    },
+
     async expectAllJobsToFinishSuccessfully(jobPaths: string[]) {
       await Promise.all(
         jobPaths.map(async (path) => {