From 519f0174b08871ede8080697d4f705692dc888b1 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Mon, 11 Nov 2024 14:17:16 -0800 Subject: [PATCH] Run OSD integ test ci-groups in parallel Signed-off-by: Rishabh Singh --- .../integ-test.jenkinsfile | 133 ++++++++++-------- .../integ_test/integ_test_runner.py | 19 +-- src/test_workflow/test_args.py | 3 + .../test_integ_test_runner_opensearch.py | 1 + ...integ_test_runner_opensearch_dashboards.py | 6 +- 5 files changed, 84 insertions(+), 78 deletions(-) diff --git a/jenkins/opensearch-dashboards/integ-test.jenkinsfile b/jenkins/opensearch-dashboards/integ-test.jenkinsfile index 812b733039..cb5617e336 100644 --- a/jenkins/opensearch-dashboards/integ-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/integ-test.jenkinsfile @@ -7,7 +7,7 @@ * compatible open source license. */ -lib = library(identifier: 'jenkins@7.3.0', retriever: modernSCM([ +lib = library(identifier: 'jenkins@7.4.0', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', ])) @@ -230,73 +230,82 @@ pipeline { // Must use local variable due to groovy for loop and closure scope // Or the stage will be fixed to the last item in return when new stages are triggered here // https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/ - def local_component = component.trim() - def local_component_index = componentList.indexOf(local_component) - def wait_seconds = local_component_index * 20 - echo "Adding Component: ${local_component}" - componentTests["Run Integtest ${local_component}"] = { - // Using scripted pipelines to trigger dynamic parallel stages - timeout(time: 6, unit: 'HOURS') { + // Changes to run OpenSearch-Dashboards ci-groups in parallel. + // There are total 9 ci-groups + if (component == "OpenSearch-Dashboards") { + for (int i = 1; i <= 9; i++) { + def local_component = 'OpenSearch-Dashboards' + def wait_seconds = i * 20 + def ciNum = i.toString() + + echo "Adding Component: ${local_component}-ci-group-${ciNum}" + + componentTests["Run Integtest ${local_component}-ci-group-${ciNum}"] = { + timeout(time: 3, unit: 'HOURS') { node(AGENT_LABEL) { - docker.withRegistry('https://public.ecr.aws/') { - docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) { - try { - stage("${local_component}") { - // Jenkins tend to not clean up workspace at times even though ws clean is called - // Since docker is mounted on the agent node directly so it can communicated with the agent - // This sometimes causes the workspace to retain last run test-results and ends with build failures - // https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99 - sh("echo ${local_component} with index ${local_component_index} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}") - unstash "integtest-opensearch-dashboards-$BUILD_NUMBER" - if (env.platform == 'windows') { - echo "On Windows Platform, unstash repository and download the artifacts" - echo "Downloading from S3: ${artifactPathOpenSearch}" - downloadFromS3( - assumedRoleName: 'opensearch-bundle', - roleAccountNumberCred: 'jenkins-aws-account-public', - downloadPath: "${artifactPathOpenSearch}/", - bucketName: "${ARTIFACT_BUCKET_NAME}", - localPath: "${WORKSPACE}/artifacts", - force: true - ) - sh("cp -a $WORKSPACE/artifacts/${artifactPathOpenSearch} $WORKSPACE") + docker.withRegistry('https://public.ecr.aws/') { + docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) { + try { + stage("${local_component}-ci-group-${ciNum}") { + sh("echo ${local_component}-ci-group-${ciNum} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}") + processAndRunOsdIntegTest( + localComponent: local_component, + switchUserNonRoot: switch_user_non_root, + ciGroup: ciNum + ) + } + } catch (e) { + throw new Exception("Error running integtest for component ${local_component}-ci-group-${ciNum}", e) + } finally { + echo "Completed running integtest for component ${local_component}-ci-group-${ciNum}" + uploadTestResults( + buildManifestFileName: BUILD_MANIFEST, + jobName: JOB_NAME + ) + postCleanup() + } + } + } + } + } + } + } + } else { + def local_component = component.trim() + def local_component_index = componentList.indexOf(local_component) + def wait_seconds = local_component_index * 20 - echo "Downloading from S3: ${artifactPath}" - downloadFromS3( - assumedRoleName: 'opensearch-bundle', - roleAccountNumberCred: 'jenkins-aws-account-public', - downloadPath: "${artifactPath}/", - bucketName: "${ARTIFACT_BUCKET_NAME}", - localPath: "${WORKSPACE}/artifacts", - force: true - ) - sh("cp -a $WORKSPACE/artifacts/${artifactPath} $WORKSPACE") - sh("rm -rf $WORKSPACE/artifacts") + echo "Adding Component: ${local_component}" + componentTests["Run Integtest ${local_component}"] = { + // Using scripted pipelines to trigger dynamic parallel stages + timeout(time: 6, unit: 'HOURS') { + node(AGENT_LABEL) { + docker.withRegistry('https://public.ecr.aws/') { + docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) { + try { + stage("${local_component}") { + // Jenkins tend to not clean up workspace at times even though ws clean is called + // Since docker is mounted on the agent node directly so it can communicated with the agent + // This sometimes causes the workspace to retain last run test-results and ends with build failures + // https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99 + sh("echo ${local_component} with index ${local_component_index} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}") + processAndRunOsdIntegTest( + localComponent: local_component, + switchUserNonRoot: switch_user_non_root, + ciGroup: "" + ) } - else { - echo "Not on Windows, unstash repository+artifacts" - } - - sh("rm -rf test-results") - runIntegTestScript( - jobName: "$BUILD_JOB_NAME", - componentName: "${local_component}", - buildManifest: "$BUILD_MANIFEST", - testManifest: "manifests/${TEST_MANIFEST}", - localPath: "${WORKSPACE}/${distribution}", - switchUserNonRoot: "${switch_user_non_root}" + } catch (e) { + throw new Exception("Error running integtest for component ${local_component}", e) + } finally { + echo "Completed running integtest for component ${local_component}" + uploadTestResults( + buildManifestFileName: BUILD_MANIFEST, + jobName: JOB_NAME ) + postCleanup() } - } catch (e) { - throw new Exception("Error running integtest for component ${local_component}", e) - } finally { - echo "Completed running integtest for component ${local_component}" - uploadTestResults( - buildManifestFileName: BUILD_MANIFEST, - jobName: JOB_NAME - ) - postCleanup() } } } diff --git a/src/test_workflow/integ_test/integ_test_runner.py b/src/test_workflow/integ_test/integ_test_runner.py index a9f410057b..4caad01dd5 100644 --- a/src/test_workflow/integ_test/integ_test_runner.py +++ b/src/test_workflow/integ_test/integ_test_runner.py @@ -43,19 +43,12 @@ def run(self) -> TestSuiteResults: if component.name in self.test_manifest.components: test_config = self.test_manifest.components[component.name] if test_config.integ_test: - if 'ci-groups' in test_config.integ_test.keys(): - orig_component_name = component.name - for i in range(1, test_config.integ_test['ci-groups'] + 1): - component.name = f"{orig_component_name}-ci-group-{i}" - test_suite = self.__create_test_suite__(component, test_config, work_dir.path) - test_results = test_suite.execute_tests() - [self.test_recorder.test_results_logs.generate_component_yml(result_data) for result_data in test_suite.result_data] - all_results.append(component.name, test_results) - else: - test_suite = self.__create_test_suite__(component, test_config, work_dir.path) - test_results = test_suite.execute_tests() - [self.test_recorder.test_results_logs.generate_component_yml(result_data) for result_data in test_suite.result_data] - all_results.append(component.name, test_results) + if self.args.ci_group: + component.name = f"{component.name}-ci-group-{self.args.ci_group}" + test_suite = self.__create_test_suite__(component, test_config, work_dir.path) + test_results = test_suite.execute_tests() + [self.test_recorder.test_results_logs.generate_component_yml(result_data) for result_data in test_suite.result_data] + all_results.append(component.name, test_results) else: logging.info(f"Skipping integ-tests for {component.name}, as it is currently not supported") else: diff --git a/src/test_workflow/test_args.py b/src/test_workflow/test_args.py index 2a9a15de6a..c734c76cb1 100644 --- a/src/test_workflow/test_args.py +++ b/src/test_workflow/test_args.py @@ -23,6 +23,7 @@ class TestArgs: logging_level: int test_manifest_path: str paths: dict + ci_group: str def __init__(self) -> None: parser = argparse.ArgumentParser(description="Test an OpenSearch Bundle") @@ -36,6 +37,7 @@ def __init__(self) -> None: parser.add_argument( "-v", "--verbose", help="Show more verbose output.", action="store_const", default=logging.INFO, const=logging.DEBUG, dest="logging_level" ) + parser.add_argument("--ci-group", type=str, default=None, help="ci group number.") args = parser.parse_args() self.test_run_id = args.test_run_id or uuid.uuid4().hex self.components = args.components @@ -44,6 +46,7 @@ def __init__(self) -> None: self.test_manifest_path = args.test_manifest_path self.paths = args.paths self.base_path = args.base_path + self.ci_group = args.ci_group TestArgs.__test__ = False # type:ignore diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py index 2d86aac3f5..dcfe192919 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch.py @@ -24,6 +24,7 @@ def test_with_integ_test(self, mock_temp: Mock, mock_test_recorder: Mock, mock_s self.args.paths = {"opensearch": "test-path"} self.args.component = "sql" self.args.test_run_id = "12345" + self.args.ci_group = None mock_test_config = MagicMock() mock_test_config.integ_test = MagicMock() diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py index de8a4346a2..0a18fff24b 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_integ_test_runner_opensearch_dashboards.py @@ -25,6 +25,7 @@ def test_with_integ_test(self, mock_temp: Mock, mock_test_recorder: Mock, mock_s self.args.paths = {"opensearch-dashboards": "test-path"} self.args.component = "sql" self.args.test_run_id = "12345" + self.args.ci_group = None mock_test_config = MagicMock() mock_test_config.integ_test = MagicMock() @@ -100,6 +101,7 @@ def test_with_integ_test_ci_groups(self, mock_temp: Mock, mock_test_recorder: Mo self.args.paths = {"opensearch-dashboards": "test-path"} self.args.component = "sql" self.args.test_run_id = "12345" + self.args.ci_group = '1' mock_test_config = MagicMock() mock_test_config.integ_test = {'test-configs': ['with-security'], 'ci-groups': 3} @@ -149,8 +151,6 @@ def test_with_integ_test_ci_groups(self, mock_temp: Mock, mock_test_recorder: Mo results = runner.run() self.assertEqual(results["sql-ci-group-1"], mock_test_results) - self.assertEqual(results["sql-ci-group-2"], mock_test_results) - self.assertEqual(results["sql-ci-group-3"], mock_test_results) mock_suite_object.result_data.__iter__.assert_called() mock_test_recorder_object.test_results_logs.generate_component_yml.assert_called() @@ -167,4 +167,4 @@ def test_with_integ_test_ci_groups(self, mock_temp: Mock, mock_test_recorder: Mo mock_path, mock_test_recorder_object ) - mock_suite.assert_has_calls([expected_call, expected_call, expected_call], any_order=True) + mock_suite.assert_has_calls([expected_call])