ci

.devcontainer/Dockerfile

@@ -0,0 +1,18 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/codespaces-linux/.devcontainer/base.Dockerfile
+
+FROM mcr.microsoft.com/vscode/devcontainers/universal:2-focal
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+RUN apt-get update && apt-get -y install --no-install-recommends \
+   python3.8-venv \
+   gcc
+
+ARG USER="codespace"
+ARG VENV_PATH="/home/${USER}/venv"
+COPY requirements.txt /tmp/
+COPY Makefile /tmp/
+RUN su $USER -c "/usr/bin/python3 -m venv /home/${USER}/venv" \
+   && su $USER -c "${VENV_PATH}/bin/pip --disable-pip-version-check --no-cache-dir install -r /tmp/requirements.txt" \
+   && rm -rf /tmp/requirements.txt

.devcontainer/devcontainer.json

@@ -0,0 +1,79 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/codespaces-linux
+{
+  "name": "GitHub Codespaces (Default)",
+
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  "features": {
+    "ghcr.io/devcontainers/features/nvidia-cuda:1": {
+      "installCudnn": true
+    }
+  },
+
+  // Configure tool-specific properties.
+  "customizations": {
+    // Configure properties specific to VS Code.
+    "vscode": {
+      // Set *default* container specific settings.json values on container create.
+      "settings": {
+        "go.toolsManagement.checkForUpdates": "local",
+        "go.useLanguageServer": true,
+        "go.gopath": "/go",
+        "python.defaultInterpreterPath": "/home/codespace/venv/bin/python",
+        "python.linting.enabled": true,
+        "python.linting.pylintEnabled": true,
+        "python.formatting.autopep8Path": "/home/codespace/venv/bin/autopep8",
+        "python.formatting.blackPath": "/home/codespace/venv/bin/black",
+        "python.formatting.yapfPath": "/home/codespace/venv/bin/yapf",
+        "python.linting.banditPath": "/home/codespace/venv/bin/bandit",
+        "python.linting.flake8Path": "/home/codespace/venv/bin/flake8",
+        "python.linting.mypyPath": "/home/codespace/venv/bin/mypy",
+        "python.linting.pycodestylePath": "/home/codespace/venv/bin/pycodestyle",
+        "python.linting.pydocstylePath": "/home/codespace/venv/bin/pydocstyle",
+        "python.linting.pylintPath": "/home/codespace/venv/bin/pylint",
+        "lldb.executable": "/usr/bin/lldb",
+        "files.watcherExclude": {
+          "**/target/**": true
+        }
+      },
+
+      // Add the IDs of extensions you want installed when the container is created.
+      "extensions": [
+        "GitHub.vscode-pull-request-github",
+        "GitHub.copilot-nightly",
+        "GitHub.copilot-labs",
+        "ms-azuretools.vscode-docker",
+        "ms-toolsai.jupyter",
+        "ms-toolsai.jupyter-keymap",
+        "ms-toolsai.jupyter-renderers",
+        "ms-python.vscode-pylance",
+        "ms-python.python",
+        "ms-vscode.makefile-tools"
+      ]
+    }
+  },
+
+  "remoteUser": "codespace",
+
+  "overrideCommand": false,
+
+  "mounts": ["source=codespaces-linux-var-lib-docker,target=/var/lib/docker,type=volume"],
+
+  "runArgs": [
+    "--cap-add=SYS_PTRACE",
+    "--security-opt",
+    "seccomp=unconfined",
+    "--privileged",
+    "--init"
+  ],
+
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+
+  // "oryx build" will automatically install your dependencies and attempt to build your project
+  //"postCreateCommand": "oryx build -p virtualenv_name=.venv --log-file /tmp/oryx-build.log --manifest-dir /tmp || echo 'Could not auto-build. Skipping.'"
+  "postCreateCommand": "bash setup.sh"
+}

.github/workflows/format.yml

@@ -0,0 +1,18 @@
+name: Format
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: install packages
+        run: make install
+      - name: format
+        run: make format

.github/workflows/install.yml

@@ -0,0 +1,16 @@
+name: Install
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: install packages
+        run: make install

.github/workflows/lint.yml

@@ -0,0 +1,18 @@
+name: Lint
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: install packages
+        run: make install
+      - name: lint
+        run: make lint

.github/workflows/test.yml

@@ -0,0 +1,18 @@
+name: Test
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: install packages
+        run: make install
+      - name: test
+        run: make test

Makefile

@@ -0,0 +1,17 @@
+install:
+	pip install --upgrade pip &&\
+		pip install -r requirements.txt
+
+test:
+	python3 -m pytest -vv --cov=services/connector services/connector/test.py &&\
+	python3 -m pytest -vv --cov=services/gateway services/gateway/test.py
+
+format:
+	black services/**/*.py
+
+lint:
+	ruff check services/**/*.py
+
+refactor: format lint
+
+all: install lint test format

cloudformation.yaml

@@ -0,0 +1,27 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: Deploy Flask App on AWS App Runner using CloudFormation
+
+Resources:
+  FlaskAppRunnerService:
+    Type: AWS::AppRunner::Service
+    Properties:
+      ServiceName: "apptest1"
+      SourceConfiguration:
+        ImageRepository:
+          ImageIdentifier: "381492212823.dkr.ecr.us-east-2.amazonaws.com/xh/final01:latest" 
+          ImageRepositoryType: ECR
+        AutoDeploymentsEnabled: true
+      InstanceConfiguration:
+        Cpu: "1024" # 1 vCPU
+        Memory: "2048" # 2 GB RAM
+      HealthCheckConfiguration:
+        Protocol: HTTP
+        Path: "/"
+      NetworkConfiguration:
+        EgressConfiguration:
+          EgressType: DEFAULT
+
+Outputs:
+  AppRunnerServiceURL:
+    Description: "URL of the deployed Flask app on App Runner"
+    Value: !GetAtt FlaskAppRunnerService.ServiceUrl

data/busiest_airports.py

@@ -42,7 +42,7 @@ def create_busiest_airports_by_delays_table():
     ORDER BY Total_Avg_Delay DESC
     """
     busiest_airports = spark.sql(query)
-    busiest_airports.write.format("delta").mode("overwrite").saveAsTable("busiest_airports_by_delays4")
+    busiest_airports.write.format("delta").mode("overwrite").saveAsTable("busiest_airports_by_delays")
     print("busiest_airports_by_delays table created successfully.")
 
 

requirements.txt

@@ -51,3 +51,7 @@ uvicorn==0.32.1
 uvloop==0.21.0
 watchfiles==1.0.3
 websockets==14.1
+ruff
+black
+pytest
+pytest-cov