Separated feature tests on ci into batches

.circleci/template.yml

@@ -161,7 +161,7 @@ jobs:
           at: .out
       - run:
           name: Merge the code coverage of the framework and features
-          command: cat .out/combined_framework.info .out/combined_features.info > .out/combined_lcov.info
+          command: cat .out/combined_framework.info .out/combined_features_batch_n.info > .out/combined_lcov.info
       - run:
           name: Preparing the environment variables
           command: |
@@ -263,7 +263,7 @@ jobs:
 
 workflows:
   version: 2
-  # `cke5_tests_framework` and `cke5_tests_features` jobs are generated by the `scripts/ci/generate-config-tests.js` script.
+  # `cke5_tests_framework` and `cke5_tests_features_batch_n` jobs are generated by the `scripts/ci/generate-config-tests.js` script.
   main:
     when:
       and:
@@ -275,15 +275,15 @@ workflows:
             branches:
               ignore:
                 - stable
-      - cke5_tests_features:
+      - cke5_tests_features_batch_n:
           filters:
             branches:
               ignore:
                 - stable
       - cke5_coverage:
           requires:
             - cke5_tests_framework
-            - cke5_tests_features
+            - cke5_tests_features_batch_n
           filters:
             branches:
               only:
@@ -304,7 +304,7 @@ workflows:
       - cke5_trigger_uber_ci:
           requires:
             - cke5_tests_framework
-            - cke5_tests_features
+            - cke5_tests_features_batch_n
             - cke5_coverage
             - cke5_validators
             - cke5_manual
@@ -316,7 +316,7 @@ workflows:
       - cke5_trigger_release_process:
           requires:
             - cke5_tests_framework
-            - cke5_tests_features
+            - cke5_tests_features_batch_n
             - cke5_validators
             - cke5_manual
             - release_prepare
@@ -336,7 +336,7 @@ workflows:
     when: << pipeline.parameters.isNightly >>
     jobs:
       - cke5_tests_framework
-      - cke5_tests_features
+      - cke5_tests_features_batch_n
       - cke5_validators
       - cke5_manual
       - release_prepare

scripts/ci/generate-circleci-configuration.js

@@ -24,6 +24,19 @@ const IS_COMMUNITY_PR = require( './is-community-pr' );
 const CKEDITOR5_ROOT_DIRECTORY = upath.join( __dirname, '..', '..' );
 const CIRCLECI_CONFIGURATION_DIRECTORY = upath.join( CKEDITOR5_ROOT_DIRECTORY, '.circleci' );
 
+const FEATURE_TEST_BATCH_NAME_PLACEHOLDER = 'cke5_tests_features_batch_n';
+const FEATURE_COVERAGE_BATCH_FILENAME_PLACEHOLDER = '.out/combined_features_batch_n.info';
+
+/**
+ * This variable determines amount and size of feature test batches.
+ *
+ * If there are more feature packages than the sum of all batches defined here,
+ * one batch will be automatically added to cover remaining tests.
+ */
+const FEATURE_BATCH_SIZES = [
+	30
+];
+
 const NON_FULL_COVERAGE_PACKAGES = [
 	'ckeditor5-minimap'
 ];
@@ -43,6 +56,22 @@ const prepareCodeCoverageDirectories = () => ( {
 	}
 } );
 
+const listBatchPackages = packageNames => {
+	const text = [
+		`A total of ${ packageNames.length } packages will be tested in this batch:`,
+		packageNames.map( packageName => ` - ${ packageName }` ).join( '\\n' ),
+		''
+	].join( '\\n\\n' );
+
+	return {
+		run: {
+			when: 'always',
+			name: 'List packages tested in this batch',
+			command: `printf "${ text }"`
+		}
+	};
+};
+
 const persistToWorkspace = fileName => ( {
 	persist_to_workspace: {
 		root: '.out',
@@ -67,6 +96,32 @@ const persistToWorkspace = fileName => ( {
 		await fs.readFile( upath.join( CIRCLECI_CONFIGURATION_DIRECTORY, 'template.yml' ) )
 	);
 
+	const featureTestBatches = featurePackages.reduce( ( output, packageName, packageIndex ) => {
+		let currentBatch = FEATURE_BATCH_SIZES.findIndex( ( batchSize, batchIndex, allBatches ) => {
+			return packageIndex < allBatches.slice( 0, batchIndex + 1 ).reduce( ( a, b ) => a + b );
+		} );
+
+		// Additional batch for the remaining tests not included in defined batch sizes.
+		if ( currentBatch === -1 ) {
+			currentBatch = FEATURE_BATCH_SIZES.length;
+		}
+
+		if ( !output[ currentBatch ] ) {
+			output[ currentBatch ] = [];
+		}
+
+		output[ currentBatch ].push( packageName );
+
+		return output;
+	}, [] );
+
+	const featureTestBatchNames = featureTestBatches.map( ( batch, batchIndex ) => {
+		return FEATURE_TEST_BATCH_NAME_PLACEHOLDER.replace( /(?<=_)n$/, batchIndex + 1 );
+	} );
+	const featureCoverageBatchFilenames = featureTestBatches.map( ( batch, batchIndex ) => {
+		return FEATURE_COVERAGE_BATCH_FILENAME_PLACEHOLDER.replace( /(?<=_)n(?=\.info$)/, batchIndex + 1 );
+	} );
+
 	config.jobs.cke5_tests_framework = {
 		machine: true,
 		steps: [
@@ -81,19 +136,77 @@ const persistToWorkspace = fileName => ( {
 		]
 	};
 
-	config.jobs.cke5_tests_features = {
-		machine: true,
-		steps: [
-			...bootstrapCommands(),
-			prepareCodeCoverageDirectories(),
-			...generateTestSteps( featurePackages, {
-				checkCoverage: true,
-				coverageFile: '.out/combined_features.info'
-			} ),
-			'community_verification_command',
-			persistToWorkspace( 'combined_features.info' )
-		]
-	};
+	// Adding batches to the root `jobs`.
+	featureTestBatches.forEach( ( batch, batchIndex ) => {
+		config.jobs[ featureTestBatchNames[ batchIndex ] ] = {
+			machine: true,
+			steps: [
+				...bootstrapCommands(),
+				prepareCodeCoverageDirectories(),
+				listBatchPackages( batch ),
+				...generateTestSteps( batch, {
+					checkCoverage: true,
+					coverageFile: featureCoverageBatchFilenames[ batchIndex ]
+				} ),
+				'community_verification_command',
+				persistToWorkspace( featureCoverageBatchFilenames[ batchIndex ].replace( /^\.out\//, '' ) )
+			]
+		};
+	} );
+
+	Object.values( config.workflows ).forEach( workflow => {
+		if ( !( workflow instanceof Object ) ) {
+			return;
+		}
+
+		if ( !workflow.jobs ) {
+			return;
+		}
+
+		// Replacing the placeholder batch names in `requires` arrays in `workflows`.
+		workflow.jobs.forEach( job => {
+			const { requires } = Object.values( job )[ 0 ];
+
+			if ( requires ) {
+				replacePlaceholderBatchNameInArray( requires, featureTestBatchNames );
+			}
+		} );
+
+		// Replacing the placeholder batch names in `jobs` arrays in `workflows`.
+		replacePlaceholderBatchNameInArray( workflow.jobs, featureTestBatchNames );
+
+		// Replacing the placeholder batch objects in `jobs` arrays in `workflows`.
+		const placeholderJobIndex = workflow.jobs.findIndex( job => job[ FEATURE_TEST_BATCH_NAME_PLACEHOLDER ] );
+
+		if ( placeholderJobIndex === -1 ) {
+			return;
+		}
+
+		const placeholderJobContent = workflow.jobs[ placeholderJobIndex ][ FEATURE_TEST_BATCH_NAME_PLACEHOLDER ];
+		const newBatchJobs = featureTestBatchNames.map( featureTestBatchName => {
+			return {
+				[ featureTestBatchName ]: placeholderJobContent
+			};
+		} );
+
+		workflow.jobs.splice( placeholderJobIndex, 1, ...newBatchJobs );
+	} );
+
+	// Replacing the coverage filename placeholder in coverage steps.
+	Object.values( config.jobs.cke5_coverage.steps ).forEach( step => {
+		if ( !( step instanceof Object ) ) {
+			return;
+		}
+
+		if ( !step.run || !step.run.command ) {
+			return;
+		}
+
+		step.run.command = step.run.command.replace(
+			FEATURE_COVERAGE_BATCH_FILENAME_PLACEHOLDER,
+			featureCoverageBatchFilenames.join( ' ' )
+		);
+	} );
 
 	if ( IS_COMMUNITY_PR ) {
 		// CircleCI does not understand custom cloning when a PR comes from the community.
@@ -158,6 +271,16 @@ function replaceShortCheckout( config, jobName ) {
 	} );
 }
 
+function replacePlaceholderBatchNameInArray( array, featureTestBatchNames ) {
+	const placeholderIndex = array.findIndex( item => item === FEATURE_TEST_BATCH_NAME_PLACEHOLDER );
+
+	if ( placeholderIndex === -1 ) {
+		return;
+	}
+
+	array.splice( placeholderIndex, 1, ...featureTestBatchNames );
+}
+
 /**
  * This type partially covers supported options on CircleCI.
  * To see the complete guide, follow: https://circleci.com/docs/configuration-reference.