From e215da81e8080aa3e958a534c6820c5be59ec47e Mon Sep 17 00:00:00 2001 From: Matheus Tosta Date: Thu, 13 Feb 2025 12:26:25 -0400 Subject: [PATCH] chore: implement CloudFormation stack for deploying the install script on the vantagecompute.ai domain. (#32) This commit implements a CloudFormation stack whose purpose is to publish the `deploy-democluster.sh` script on the `vantagecompute.ai` domain. As well as, a new job is added in both the `Linter` and `Build and Publish democluster to s3` workflows. --- .../workflows/build-and-publish-image.yaml | 54 +++- .github/workflows/ci.yaml | 46 ++++ .gitignore | 4 +- cdk.json | 6 + poetry.lock | 238 +++++++++++++++++- pyproject.toml | 4 + website.py | 130 ++++++++++ 7 files changed, 471 insertions(+), 11 deletions(-) create mode 100644 cdk.json create mode 100644 website.py diff --git a/.github/workflows/build-and-publish-image.yaml b/.github/workflows/build-and-publish-image.yaml index d47ffd0..0123287 100644 --- a/.github/workflows/build-and-publish-image.yaml +++ b/.github/workflows/build-and-publish-image.yaml @@ -5,12 +5,6 @@ on: branches: - main workflow_dispatch: - inputs: - jobbergate_version: - description: The Jobbergate agent version - required: false - type: string - default: 4.3.1 jobs: build-image-in-lxd: @@ -26,8 +20,6 @@ jobs: poetry install - name: Build democluster - env: - JG_VERSION: ${{ github.event.inputs.jobbergate_version }} run: | poetry run image-factory build democluster @@ -40,3 +32,49 @@ jobs: aws s3 sync democluster/final \ s3://omnivector-public-assets/cloud-images/democluster/latest \ --acl public-read --follow-symlinks --delete 1> /dev/null + + install-script-cd: + name: install-script-cd + runs-on: ubuntu-24.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + + - uses: actions/setup-node@v4 + with: + node-version: '22.8.0' + + - name: Install Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: 1.8.3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + cache: 'poetry' + + - name: Install CDK + run: | + npm install -g aws-cdk@2.178.2 + + - name: Cache cdk.out + uses: actions/cache@v4 + with: + path: cdk.out + key: cdk-out + + - name: Setup AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Install Python packages + run: | + poetry install --with=website + + - name: Publish the deploy script + run: | + cdk deploy --path-metadata false --version-reporting false --require-approval never DemoclusterWebsite diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fbcf66e..58e0a58 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,3 +50,49 @@ jobs: run: | poetry run mypy image_factory --pretty poetry run ruff check image_factory + + synth-stack: + name: synth-stack + runs-on: ubuntu-24.04 + steps: + - name: Check out code + uses: actions/checkout@v3 + + - uses: actions/setup-node@v4 + with: + node-version: '22.8.0' + + - name: Install Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: 1.8.3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + cache: 'poetry' + + - name: Install CDK + run: | + npm install -g aws-cdk@2.178.2 + + - name: Cache cdk.out + uses: actions/cache@v4 + with: + path: cdk.out + key: cdk-out + + - name: Setup AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Install Python packages + run: | + poetry install --with=website + + - name: Publish the deploy script + run: | + cdk synth --path-metadata false --version-reporting false diff --git a/.gitignore b/.gitignore index 1eb951e..b3b0d18 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ output-* *.fd seeds-cloudimg.iso *.tar.gz -*.img \ No newline at end of file +*.img + +cdk.out/ \ No newline at end of file diff --git a/cdk.json b/cdk.json new file mode 100644 index 0000000..15ff96b --- /dev/null +++ b/cdk.json @@ -0,0 +1,6 @@ +{ + "app":"poetry run python3 ./website.py", + "context":{ + "@aws-cdk/core:newStyleStackSynthesis":true + } +} \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 6fe8bc6..8adf046 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,108 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "attrs" +version = "24.3.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.8" +files = [ + {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, + {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, +] + +[package.extras] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] + +[[package]] +name = "aws-cdk-asset-awscli-v1" +version = "2.2.223" +description = "A library that contains the AWS CLI for use in Lambda Layers" +optional = false +python-versions = "~=3.8" +files = [ + {file = "aws_cdk.asset_awscli_v1-2.2.223-py3-none-any.whl", hash = "sha256:2695a3356b97437f1f73678978deb73b7b2158ddbb5989cca5d4734126e5d595"}, + {file = "aws_cdk_asset_awscli_v1-2.2.223.tar.gz", hash = "sha256:130803633287b0c01e387ef43b6a298dcae1b9c985ade5ba81e398c15ec62296"}, +] + +[package.dependencies] +jsii = ">=1.106.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<4.3.0" + +[[package]] +name = "aws-cdk-asset-kubectl-v20" +version = "2.1.4" +description = "A Lambda Layer that contains kubectl v1.20" +optional = false +python-versions = "~=3.8" +files = [ + {file = "aws_cdk.asset_kubectl_v20-2.1.4-py3-none-any.whl", hash = "sha256:14a194e14adbf3868a8105b07e8714e10e6621a22beca9b4859a82a9cfbe68f6"}, + {file = "aws_cdk_asset_kubectl_v20-2.1.4.tar.gz", hash = "sha256:c723c94c6c89283efef779ca44bea8e2e312d49b07bf5beaf6f27340e1fecff4"}, +] + +[package.dependencies] +jsii = ">=1.106.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<4.3.0" + +[[package]] +name = "aws-cdk-asset-node-proxy-agent-v6" +version = "2.1.0" +description = "@aws-cdk/asset-node-proxy-agent-v6" +optional = false +python-versions = "~=3.8" +files = [ + {file = "aws_cdk.asset_node_proxy_agent_v6-2.1.0-py3-none-any.whl", hash = "sha256:24a388b69a44d03bae6dbf864c4e25ba650d4b61c008b4568b94ffbb9a69e40e"}, + {file = "aws_cdk_asset_node_proxy_agent_v6-2.1.0.tar.gz", hash = "sha256:1f292c0631f86708ba4ee328b3a2b229f7e46ea1c79fbde567ee9eb119c2b0e2"}, +] + +[package.dependencies] +jsii = ">=1.103.1,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<5.0.0" + +[[package]] +name = "aws-cdk-cloud-assembly-schema" +version = "39.2.20" +description = "Cloud Assembly Schema" +optional = false +python-versions = "~=3.8" +files = [ + {file = "aws_cdk.cloud_assembly_schema-39.2.20-py3-none-any.whl", hash = "sha256:7b1ab0593fbfaac4ff0d5d0aa6a3b54185e0286f4aa68376557cac2d50c59183"}, + {file = "aws_cdk_cloud_assembly_schema-39.2.20.tar.gz", hash = "sha256:e110b22f961d15c25a9099590375280c0b45637c8325ade9b570a0ef11e6e907"}, +] + +[package.dependencies] +jsii = ">=1.106.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<4.3.0" + +[[package]] +name = "aws-cdk-lib" +version = "2.178.2" +description = "Version 2 of the AWS Cloud Development Kit library" +optional = false +python-versions = "~=3.8" +files = [ + {file = "aws_cdk_lib-2.178.2-py3-none-any.whl", hash = "sha256:624383e57fe2b32f7d0fc098b78b4cd21d19ae3af3f24b01f32ec4795baaee25"}, + {file = "aws_cdk_lib-2.178.2.tar.gz", hash = "sha256:c00757885b74023350bb34f388f6447155e802ecf827e595bda917098a4925fe"}, +] + +[package.dependencies] +"aws-cdk.asset-awscli-v1" = ">=2.2.208,<3.0.0" +"aws-cdk.asset-kubectl-v20" = ">=2.1.3,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v6" = ">=2.1.0,<3.0.0" +"aws-cdk.cloud-assembly-schema" = ">=39.2.0,<40.0.0" +constructs = ">=10.0.0,<11.0.0" +jsii = ">=1.106.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<4.3.0" [[package]] name = "black" @@ -44,6 +148,30 @@ d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "cattrs" +version = "24.1.2" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-24.1.2-py3-none-any.whl", hash = "sha256:67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0"}, + {file = "cattrs-24.1.2.tar.gz", hash = "sha256:8028cfe1ff5382df59dd36474a86e02d817b06eaf8af84555441bac915d2ef85"}, +] + +[package.dependencies] +attrs = ">=23.1.0" + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +msgspec = ["msgspec (>=0.18.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + [[package]] name = "certifi" version = "2023.11.17" @@ -179,6 +307,22 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "constructs" +version = "10.4.2" +description = "A programming model for software-defined state" +optional = false +python-versions = "~=3.8" +files = [ + {file = "constructs-10.4.2-py3-none-any.whl", hash = "sha256:1f0f59b004edebfde0f826340698b8c34611f57848139b7954904c61645f13c1"}, + {file = "constructs-10.4.2.tar.gz", hash = "sha256:ce54724360fffe10bab27d8a081844eb81f5ace7d7c62c84b719c49f164d5307"}, +] + +[package.dependencies] +jsii = ">=1.102.0,<2.0.0" +publication = ">=0.0.3" +typeguard = ">=2.13.3,<2.14.0" + [[package]] name = "craft-providers" version = "1.21.0" @@ -214,6 +358,45 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] +[[package]] +name = "importlib-resources" +version = "6.5.2" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.9" +files = [ + {file = "importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec"}, + {file = "importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] +type = ["pytest-mypy"] + +[[package]] +name = "jsii" +version = "1.106.0" +description = "Python client for jsii runtime" +optional = false +python-versions = "~=3.8" +files = [ + {file = "jsii-1.106.0-py3-none-any.whl", hash = "sha256:5a44d7c3a5a326fa3d9befdb3770b380057e0a61e3804e7c4907f70d76afaaa2"}, + {file = "jsii-1.106.0.tar.gz", hash = "sha256:c79c47899f53a7c3c4b20f80d3cd306628fe9ed1852eee970324c71eba1d974e"}, +] + +[package.dependencies] +attrs = ">=21.2,<25.0" +cattrs = ">=1.8,<24.2" +importlib-resources = ">=5.2.0" +publication = ">=0.0.3" +python-dateutil = "*" +typeguard = ">=2.13.3,<4.5.0" +typing-extensions = ">=3.8,<5.0" + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -343,6 +526,17 @@ files = [ docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] +[[package]] +name = "publication" +version = "0.0.3" +description = "Publication helps you maintain public-api-friendly modules by preventing unintentional access to private implementation details via introspection." +optional = false +python-versions = "*" +files = [ + {file = "publication-0.0.3-py2.py3-none-any.whl", hash = "sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6"}, + {file = "publication-0.0.3.tar.gz", hash = "sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4"}, +] + [[package]] name = "pydantic" version = "1.10.14" @@ -410,6 +604,20 @@ files = [ plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + [[package]] name = "pyyaml" version = "6.0.1" @@ -549,6 +757,32 @@ files = [ {file = "ruff-0.0.282.tar.gz", hash = "sha256:ef677c26bae756e4c98af6d8972da83caea550bc92ffef97a6e939ca5b24ad06"}, ] +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + +[[package]] +name = "typeguard" +version = "2.13.3" +description = "Run-time type checker for Python" +optional = false +python-versions = ">=3.5.3" +files = [ + {file = "typeguard-2.13.3-py3-none-any.whl", hash = "sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1"}, + {file = "typeguard-2.13.3.tar.gz", hash = "sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4"}, +] + +[package.extras] +doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["mypy", "pytest", "typing-extensions"] + [[package]] name = "typer" version = "0.9.0" @@ -611,4 +845,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "e9e5d526333211475788e77b41d26ea4b120b4f700bbbf14918886864f327bfe" +content-hash = "ba550deef6be30a118e279f996cb7b3cf9bebf003679c0b20b61e996d9dcab2c" diff --git a/pyproject.toml b/pyproject.toml index 13530e3..aa986e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,10 @@ black = "^24.3.0" [tool.poetry.group.dev.dependencies] types-pyyaml = "^6.0.12.11" + +[tool.poetry.group.website.dependencies] +aws-cdk-lib = "^2" + [tool.black] line-length = 110 src = ["image_factory"] diff --git a/website.py b/website.py new file mode 100644 index 0000000..a413b9e --- /dev/null +++ b/website.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 + + +import aws_cdk as cdk +from aws_cdk import App, Stack +from aws_cdk import aws_certificatemanager as cm +from aws_cdk import aws_cloudfront as cloudfront +from aws_cdk import aws_route53 as route53 +from aws_cdk import aws_route53_targets as targets +from aws_cdk import aws_s3 as s3 +from aws_cdk import aws_s3_deployment as s3_deploy + + +class WebsiteStack(Stack): + """CloudFormation stack for hosting the Democluster installation script.""" + + def __init__(self, app: App, id: str, **kwargs): # noqa: D107 + super().__init__(app, id, **kwargs) + + site_domain = "install-demo-cluster.vantagecompute.ai" + + hosted_zone = route53.HostedZone.from_hosted_zone_attributes( + self, + "HostedZone", + hosted_zone_id="Z076740924E27W77EXSVN", + zone_name="vantagecompute.ai", + ) + + distribution_oai = cloudfront.OriginAccessIdentity( + self, + "CloudFrontOAI", + comment=f"OAI for stack {id}" + ) + + bucket = s3.Bucket( + self, + "HostBucket", + bucket_name=site_domain, + website_index_document="index.html", + website_error_document="index.html", + removal_policy=cdk.RemovalPolicy.DESTROY, + cors=[ + s3.CorsRule( + allowed_origins=["*"], + allowed_methods=[s3.HttpMethods.GET, s3.HttpMethods.HEAD], + allowed_headers=["*"], + ) + ], + ) + + bucket.grant_read(distribution_oai) + + certificate = cm.Certificate( + self, + "Certificate", + domain_name=site_domain, + validation=cm.CertificateValidation.from_dns(hosted_zone=hosted_zone), + ) + + distribution = cloudfront.CloudFrontWebDistribution( + self, + "Distribution", + default_root_object="deploy-democluster.sh", + error_configurations=[ + cloudfront.CfnDistribution.CustomErrorResponseProperty( + error_code=404, + error_caching_min_ttl=0, + response_code=200, + response_page_path="/deploy-democluster.sh", + ) + ], + origin_configs=[ + cloudfront.SourceConfiguration( + behaviors=[ + cloudfront.Behavior( + is_default_behavior=True, + max_ttl=cdk.Duration.seconds(0), + min_ttl=cdk.Duration.seconds(0), + default_ttl=cdk.Duration.seconds(0), + ) + ], + s3_origin_source=cloudfront.S3OriginConfig( + s3_bucket_source=bucket, + origin_access_identity=distribution_oai, + ), + ), + ], + price_class=cloudfront.PriceClass.PRICE_CLASS_ALL, + viewer_certificate=cloudfront.ViewerCertificate.from_acm_certificate( + certificate=certificate, + aliases=[site_domain], + security_policy=cloudfront.SecurityPolicyProtocol.TLS_V1_2_2021, + ssl_method=cloudfront.SSLMethod.SNI, + ), + viewer_protocol_policy=cloudfront.ViewerProtocolPolicy.REDIRECT_TO_HTTPS, + ) + + route53.ARecord( + self, + "ARecord", + target=route53.RecordTarget.from_alias( + targets.CloudFrontTarget(distribution=distribution) + ), + zone=hosted_zone, + record_name=site_domain, + ) + + with open("public-scripts/deploy-democluster.sh", "r") as f: + script = f.read() + + s3_deploy.BucketDeployment( + self, + "BucketDeployment", + sources=[s3_deploy.Source.data("deploy-democluster.sh", script)], + destination_bucket=bucket, + distribution=distribution, + memory_limit=256, + content_type="text/plain", + ) + +common_tags = {"project": "Vantage", "Application": "democluster"} + +env = cdk.Environment( + account = '266735843730', + region = 'us-east-1' +) + +app = cdk.App() +WebsiteStack(app, "DemoclusterWebsite", env=env, tags=common_tags) +app.synth()