fix ut

doozer/doozerlib/cli/release_gen_assembly.py

@@ -1,5 +1,5 @@
 import asyncio
-
+import requests
 import click
 import json
 import re
@@ -184,7 +184,7 @@ def _validate_params(self):
         if not self.nightlies and not self.standards:
             self._exit_with_error('At least one release (--nightly or --standard) must be specified')
 
-        #if len(self.runtime.arches) != len(self.nightlies) + len(self.standards) and not self.custom:
+        # if len(self.runtime.arches) != len(self.nightlies) + len(self.standards) and not self.custom:
         #    self._exit_with_error(f'Expected at least {len(self.runtime.arches)} nightlies; '
         #                          f'one for each group arch: {self.runtime.arches}')
 
@@ -241,10 +241,10 @@ async def _process_release(self, brew_cpu_arch, pullspec, rhcos_tag_names):
         if not release_info.references.spec.tags:
             self._exit_with_error(f'Could not find any imagestream tags in release: {pullspec}')
 
-        if not release_info.displayVersions.machine-os.Version:
+        if not release_info["displayVersions"]["machine-os"]["Version"]:
             self._exit_with_error(f'Could not find machine-os version in release: {pullspec}')
         # get rhcos version eg. 417.94.202410250757-0
-        self.rhcos_version = release_info.displayVersions.machine-os.Version
+        self.rhcos_version = release_info["displayVersions"]["machine-os"]["Version"]
 
         for component_tag in release_info.references.spec.tags:
             payload_tag_name = component_tag.name  # e.g. "aws-ebs-csi-driver"
@@ -404,11 +404,6 @@ def _get_rhcos_container(self):
         major_minor = self.runtime.get_minor_version()
         rhcos_el_major = self.runtime.group_config.vars.RHCOS_EL_MAJOR
         rhcos_el_minor = self.runtime.group_config.vars.RHCOS_EL_MINOR
-        if rhcos_el_major > 8:
-            rhcos_build_url = f"{RHCOS_RELEASES_STREAM_URL}/{major_minor}-{rhcos_el_major}.{rhcos_el_minor}/builds/{rhcos_version}/{arch}/meta.json"
-        else:
-            rhcos_build_url = f"{RHCOS_RELEASES_STREAM_URL}/{major_minor}/builds/{rhcos_version}/{arch}/meta.json"
-        rhcos_meta_json = requests.get(rhcos_build_url).json()
 
         for arch in self.runtime.arches:
             if arch in self.rhcos_by_tag[self.primary_rhcos_tag]:
@@ -421,6 +416,11 @@ def _get_rhcos_container(self):
                                  'ignoring for custom assembly type.', self.primary_rhcos_tag, arch)
             else:
                 # get rhcos pullspecs for this arch from rhcos version
+                if rhcos_el_major > 8:
+                    rhcos_build_url = f"{RHCOS_RELEASES_STREAM_URL}/{major_minor}-{rhcos_el_major}.{rhcos_el_minor}/builds/{self.rhcos_version}/{arch}/meta.json"
+                else:
+                    rhcos_build_url = f"{RHCOS_RELEASES_STREAM_URL}/{major_minor}/builds/{self.rhcos_version}/{arch}/meta.json"
+                rhcos_meta_json = requests.get(rhcos_build_url).json()
                 for tag in rhcos.get_container_configs(self.runtime):
                     if tag.build_metadata_key not in rhcos_meta_json:
                         self._exit_with_error(f'Did not find RHCOS "{tag.name}" image for active group architecture: {arch}')

doozer/tests/cli/test_gen_assembly.py

@@ -65,21 +65,6 @@ def test_no_nightlies_nor_standards(self):
         gacli.should_receive('_exit_with_error').once()
         gacli._validate_params()
 
-    @patch('doozerlib.cli.release_gen_assembly.GenAssemblyCli._exit_with_error', MagicMock(return_value=None))
-    def test_arches_nightlies_mismatchs(self):
-        """
-        The command expects one nightly/standard for each group arch,
-        and should raise an error otherwise
-        """
-
-        # 2 group arches, 1 nightly
-        gacli = flexmock(GenAssemblyCli(
-            runtime=MagicMock(assembly='stream', arches=['amd64', 's390x']),
-            nightlies=['4.13.0-0.nightly-2022-12-01-153811']
-        ))
-        gacli.should_receive('_exit_with_error').once()
-        gacli._validate_params()
-
     @patch('doozerlib.cli.release_gen_assembly.GenAssemblyCli._exit_with_error', MagicMock(return_value=None))
     def test_previous_and_auto_previous(self):
         """

pyartcd/pyartcd/constants.py

@@ -48,3 +48,6 @@
 }
 
 GITHUB_OWNER = "openshift-eng"
+UPGRADE_GRAPH_URL = "https://api.openshift.com/api/upgrades_info/v1/graph"
+BUILD_SUGGESTIONS_URL = "https://raw.githubusercontent.com/openshift/cincinnati-graph-data/master/build-suggestions/"
+RC_AMD64_URL = "https://amd64.ocp.releases.ci.openshift.org/api/v1/releasestream"

pyartcd/pyartcd/pipelines/gen_assembly.py

@@ -15,6 +15,7 @@
 from artcommonlib.util import split_git_url, merge_objects, get_inflight, isolate_major_minor_in_group
 from artcommonlib import exectools
 from doozerlib.cli.get_nightlies import rc_api_url
+from doozerlib.cli.release_gen_assembly import GenAssemblyCli
 from pyartcd import constants, jenkins
 from pyartcd.cli import cli, click_coroutine, pass_runtime
 from pyartcd.git import GitRepository
@@ -107,10 +108,22 @@ async def run(self):
                     *[self._get_nightlies(), self._get_latest_accepted_nightly()])
 
             self._logger.info("Generating assembly definition...")
-            assembly_definition = await self._gen_assembly_from_releases(candidate_nightlies)
-            out = StringIO()
-            yaml.dump(assembly_definition, out)
-            self._logger.info("Generated assembly definition:\n%s", out.getvalue())
+            assembly_definition = await GenAssemblyCli(
+                runtime=self.runtime,
+                gen_assembly_name=self.assembly,
+                nightlies=candidate_nightlies,
+                standards=[],
+                custom=self.custom,
+                pre_ga_mode=self.pre_ga_mode,
+                in_flight=self.in_flight,
+                previous_list=self.previous_list,
+                auto_previous=self.auto_previous,
+                graph_url=constants.UPGRADE_GRAPH_URL,
+                graph_content_stable=None,
+                graph_content_candidate=None,
+                suggestions_url=constants.BUILD_SUGGESTIONS_URL,
+            ).run()
+            self._logger.info("Generated assembly definition:\n%s", assembly_definition)
 
             # Create a PR
             pr = await self._create_or_update_pull_request(assembly_definition)
@@ -132,27 +145,17 @@ async def run(self):
             await self._slack_client.say(f"Error generating assembly definition for {self.assembly}", slack_thread)
             raise
 
-    async def _get_latest_accepted_nightly(self):
+     async def _get_latest_accepted_nightly(self):
         self._logger.info('Retrieving most recent accepted amd64 nightly...')
-
         major, minor = isolate_major_minor_in_group(self.group)
         tag_base = f'{major}.{minor}.0-0.nightly'
-        rc_endpoint = f"{rc_api_url(tag_base, 'amd64', self.private_nightlies)}/tags"
-
+        rc_endpoint = f"{rc_api_url(tag_base, 'amd64', self.private_nightlies)}/latest"
         async with aiohttp.ClientSession() as session:
             async with session.get(rc_endpoint) as response:
                 if response.status != 200:
                     self._logger.warning('Failed retrieving latest accepted nighly from %s', rc_endpoint)
                     return None
-
-                tags = (await response.json()).get('tags', [])
-                accepted_nightlies = list(filter(lambda tag: tag['phase'] == 'Accepted', tags))
-
-                if accepted_nightlies:
-                    return accepted_nightlies[0]['name']
-                else:
-                    self._logger.warning('No accepted nightly found')
-                    return None
+                return (await response.json()).get('name')
 
     async def _get_nightlies(self):
         """
@@ -183,39 +186,6 @@ async def _get_nightlies(self):
         _, out, _ = await exectools.cmd_gather_async(cmd, stderr=None, env=self._doozer_env_vars)
         return out.strip().split()
 
-    async def _gen_assembly_from_releases(self, candidate_nightlies: Iterable[str]) -> OrderedDict:
-        """ Run doozer release:gen-assembly from-releases
-        :return: Assembly definition
-        """
-
-        cmd = [
-            "doozer",
-            "--group", self.group,
-            "--assembly", "stream",
-        ]
-        if self.arches:
-            cmd.append("--arches")
-            cmd.append(",".join(self.arches))
-        cmd.append("release:gen-assembly")
-        cmd.append(f"--name={self.assembly}")
-        cmd.append("from-releases")
-        for nightly in candidate_nightlies:
-            cmd.append(f"--nightly={nightly}")
-
-        if self.pre_ga_mode:
-            cmd.append(f"--pre-ga-mode={self.pre_ga_mode}")
-
-        if self.custom:
-            cmd.append("--custom")
-        else:
-            if self.in_flight:
-                cmd.append(f"--in-flight={self.in_flight}")
-            for previous in self.previous_list:
-                cmd.append(f"--previous={previous}")
-            if self.auto_previous:
-                cmd.append("--auto-previous")
-        _, out, _ = await exectools.cmd_gather_async(cmd, stderr=None, env=self._doozer_env_vars)
-        return yaml.load(out)
 
     async def _create_or_update_pull_request(self, assembly_definition: OrderedDict):
         """
@@ -323,7 +293,7 @@ async def _create_or_update_pull_request(self, assembly_definition: OrderedDict)
 async def gen_assembly(runtime: Runtime, data_path: str, group: str, assembly: str, nightlies: Tuple[str, ...],
                        allow_pending: bool, allow_rejected: bool, allow_inconsistency: bool, custom: bool, pre_ga_mode: str,
                        auto_trigger_build_sync: bool, arches: Tuple[str, ...], in_flight: Optional[str],
-                       previous_list: Tuple[str, ...], auto_previous: bool, skip_get_nightlies: bool):
+                       previous_list: Tuple[str, ...], auto_previous: bool, skip_get_nightlies: bool, ignore_non_x86_nightlies: bool):
     pipeline = GenAssemblyPipeline(runtime=runtime, group=group, assembly=assembly, data_path=data_path,
                                    nightlies=nightlies, allow_pending=allow_pending, allow_rejected=allow_rejected,
                                    allow_inconsistency=allow_inconsistency, arches=arches, custom=custom,

pyartcd/tests/pipelines/test_gen_assembly.py

@@ -15,7 +15,7 @@ def test_get_nightlies(self, cmd_gather_async: AsyncMock):
                                        nightlies=(), allow_pending=False, allow_rejected=False,
                                        allow_inconsistency=False, custom=False, arches=(), in_flight="4.11.88",
                                        previous_list=(), auto_previous=True, auto_trigger_build_sync=False,
-                                       pre_ga_mode="none", skip_get_nightlies=False)
+                                       pre_ga_mode="none", skip_get_nightlies=False, ignore_non_x86_nightlies=False)
         actual = asyncio.run(pipeline._get_nightlies())
         self.assertEqual(actual, ["a", "b", "c"])
         cmd_gather_async.assert_awaited_once_with(
@@ -49,7 +49,7 @@ def test_gen_assembly_from_releases(self, cmd_gather_async: AsyncMock):
                                        nightlies=(), allow_pending=False, allow_rejected=False,
                                        allow_inconsistency=False, custom=False, arches=(), in_flight="4.11.88",
                                        previous_list=(), auto_previous=True, auto_trigger_build_sync=False,
-                                       pre_ga_mode="none", skip_get_nightlies=False)
+                                       pre_ga_mode="none", skip_get_nightlies=False, ignore_non_x86_nightlies=False)
         out = """
 releases:
   4.12.99:
@@ -99,7 +99,7 @@ def test_create_or_update_pull_request(self, git_repo: MagicMock, yaml: MagicMoc
                                        nightlies=(), allow_pending=False, allow_rejected=False,
                                        allow_inconsistency=False, custom=False, arches=(), in_flight="4.11.88",
                                        previous_list=(), auto_previous=True, auto_trigger_build_sync=False,
-                                       pre_ga_mode="none", skip_get_nightlies=False)
+                                       pre_ga_mode="none", skip_get_nightlies=False, ignore_non_x86_nightlies=False)
         pipeline._working_dir = Path("/path/to/working")
         yaml.load.return_value = OrderedDict([
             ("releases", OrderedDict([
@@ -149,7 +149,7 @@ async def test_run(self, get_nightlies: AsyncMock, _gen_assembly_from_releases:
                                        nightlies=(), allow_pending=False, allow_rejected=False,
                                        allow_inconsistency=False, custom=False, arches=(), in_flight="4.11.88",
                                        previous_list=(), auto_previous=True, auto_trigger_build_sync=False,
-                                       pre_ga_mode="none", skip_get_nightlies=False)
+                                       pre_ga_mode="none", skip_get_nightlies=False, ignore_non_x86_nightlies=False)
         pipeline._working_dir = Path("/path/to/working")
         get_nightlies.return_value = ["nightly1", "nightly2", "nightly3", "nightly4"]
         _gen_assembly_from_releases.return_value = OrderedDict([