Skip to content

Commit

Permalink
[Internal] Add script for checking OCI images (#1408)
Browse files Browse the repository at this point in the history
This new command in `oci_image_tools.py` checks
that images are published in the OCI Marketplace
and completed the OCI review successfully. This
command is supposed to be used manually. Using it
as part of the images build CI workflow is not
viable, as reviews can take indefinite time.
  • Loading branch information
jvstme authored Jul 10, 2024
1 parent 6aed1cb commit bdf3ecc
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 15 deletions.
65 changes: 59 additions & 6 deletions scripts/oci_image_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,28 @@ def from_namespace(args: Namespace) -> "PublishCommandArgs":
)


@dataclass
class CheckCommandArgs:
image_name: str
regions: List[str]
compartment_id: str

@classmethod
def setup_parser(cls, parser: ArgumentParser) -> None:
parser.add_argument("--image", dest="image_name", required=True)
parser.add_argument("--regions", metavar="REGION_NAME", nargs="*")
parser.add_argument("--compartment", dest="compartment_id", required=True)
parser.set_defaults(to_struct=cls.from_namespace, run_command=check_command)

@staticmethod
def from_namespace(args: Namespace) -> "PublishCommandArgs":
return CheckCommandArgs(
image_name=args.image_name,
regions=args.regions or [],
compartment_id=args.compartment_id,
)


def main() -> None:
parser = ArgumentParser(description="Tools for delivering OCI images")
subparsers = parser.add_subparsers()
Expand All @@ -130,6 +152,15 @@ def main() -> None:
)
PublishCommandArgs.setup_parser(publish_parser)

check_parser = subparsers.add_parser(
name="check",
description=(
"Check that Custom Image is published in OCI Marketplace in all subscribed "
"regions or in specified regions."
),
)
CheckCommandArgs.setup_parser(check_parser)

args = parser.parse_args()
args.run_command(args.to_struct(args))

Expand Down Expand Up @@ -177,17 +208,39 @@ def publish_command(args: PublishCommandArgs) -> None:
compartment_id=args.compartment_id,
client=region_clients[region_name].marketplace_client,
)
logging.info("Published in %s (%d/%d)", region_name, i, len(regions_for_publishing))
logging.info("Submitted in %s (%d/%d)", region_name, i, len(regions_for_publishing))

logging.info("Published image %s in regions: %s", args.image_name, regions_for_publishing)
logging.info("Submitted image %s in regions: %s", args.image_name, regions_for_publishing)
logging.info(
"Note that the publications need to go through OCI's review process "
"that may take a few hours. The publications will be unavailable until then. "
"See the review statuses at https://cloud.oracle.com/marketplace/community-images. "
"Do not forget to choose the region and the compartment."
"The publications will now go through OCI's review process that may take a few hours. "
"The publications will be unavailable until the review finishes. "
"Use `python oci_image_tools.py check` to check publication statuses."
)


def check_command(args: CheckCommandArgs) -> None:
region_clients = get_region_clients(required_regions=args.regions)
regions_to_check = args.regions or list(region_clients)
some_not_published = False

for region in sorted(regions_to_check):
if not resources.list_marketplace_listings(
args.image_name, region_clients[region].marketplace_client
):
some_not_published = True
status = "Not published"
else:
status = "Published"
logging.info("%24s: %s", region, status)

if some_not_published:
raise ScriptError(
f"Image {args.image_name} is not published or is still under review in some regions. "
"Check the review status by choosing the correct region and compartment here: "
"https://cloud.oracle.com/marketplace/community-images"
)


def get_region_clients(
required_regions: Iterable[str] = frozenset(),
) -> Dict[str, OCIRegionClient]:
Expand Down
23 changes: 14 additions & 9 deletions src/dstack/_internal/core/backends/oci/resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,20 @@ def get_marketplace_listing_and_package(
if cuda:
listing_name = f"dstack-cuda-{version.base_image}"

listing_summaries: List[oci.marketplace.models.ListingSummary] = [
listing_summaries = list_marketplace_listings(listing_name, client)
if len(listing_summaries) != 1:
msg = f"Expected to find 1 listing by name {listing_name}, found {len(listing_summaries)}"
raise BackendError(msg)

listing: oci.marketplace.models.Listing = client.get_listing(listing_summaries[0].id).data
package = client.get_package(listing.id, listing.default_package_version).data
return listing, package


def list_marketplace_listings(
listing_name: str, client: oci.marketplace.MarketplaceClient
) -> List[oci.marketplace.models.ListingSummary]:
return [
listing
for listing in chain_paginated_responses(
client.list_listings,
Expand All @@ -371,14 +384,6 @@ def get_marketplace_listing_and_package(
if listing.name == listing_name # list_listings returns inexact matches
]

if len(listing_summaries) != 1:
msg = f"Expected to find 1 listing by name {listing_name}, found {len(listing_summaries)}"
raise BackendError(msg)

listing: oci.marketplace.models.Listing = client.get_listing(listing_summaries[0].id).data
package = client.get_package(listing.id, listing.default_package_version).data
return listing, package


def accept_marketplace_listing_agreements(
listing: oci.marketplace.models.Listing,
Expand Down

0 comments on commit bdf3ecc

Please sign in to comment.