diff --git a/public-cloud/gcp/datalake/.gitignore b/public-cloud/gcp/datalake/.gitignore new file mode 100644 index 0000000..dbf25ad --- /dev/null +++ b/public-cloud/gcp/datalake/.gitignore @@ -0,0 +1,17 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator.log +runs +context diff --git a/public-cloud/gcp/datalake/README.md b/public-cloud/gcp/datalake/README.md new file mode 100644 index 0000000..152d20e --- /dev/null +++ b/public-cloud/gcp/datalake/README.md @@ -0,0 +1,64 @@ +# CDP Public Cloud - Environment and Datalake on GCP Base Example + +> Constructs a CDP Public Cloud Environment and Datalake. Generates via Ansible the GCP infrastructure and CDP artifacts, including SSH key, cross-account Service Account, GCS buckets, etc. + +## Requirements + +To run, you need: + +* Docker (or a Docker alternative) +* GCP Service Account provisioning credentials (set via `GCP_SERVICE_ACCOUNT_FILE`) +* CDP credentials (set via `CDP_PROFILE`) + +## Set Up + +First, set up your `ansible-navigator` aka `cdp-navigator` environment -- follow the instructions in the [NAVIGATOR document](https://github.com/cloudera-labs/cldr-runner/blob/main/NAVIGATOR.md) in `cloudera-labs/cldr-runner`. + +Then, clone this project and change your working directory. + +```bash +git clone https://github.com/cloudera-labs/cloudera-deploy.git; cd cloudera-deploy/public-cloud/gcp/datalake +``` + +## Configure + +Set the required environment variables: + +```bash +export GCP_SERVICE_ACCOUNT_FILE=absolute-path-to-service-account-file +export CDP_PROFILE=your-cdp-profile +``` + +Tweak the `definition.yml` parameters to your liking. Notably, you should add and/or change: + +```yaml +name_prefix: ex01 # Keep this short (4-7 characters) +admin_password: "BadPass@1" # 1 upper, 1 special, 1 number, 8-64 chars. +infra_region: us-east1 +gcp_project_id: gcp-project-id # GCP Project ID +``` + +> [!NOTE] +> You can override these parameters with any typical Ansible _extra variables_ flags, i.e. `-e admin_password=my_password`. See the [cldr-runner FAQ](https://github.com/cloudera-labs/cldr-runner/blob/main/FAQ.md#how-do-i-add-extra-variables-and-tags-to-ansible-navigator) for details. + +### SSH Keys + +This definition will create a new SSH keypair on the host in your `~/.ssh` directory if you do not specify a SSH public key. + +If you wish to use an existing SSH key, set `public_key_file` to the key's local path. + +## Execute + +Then set up the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run main.yml +``` + +## Tear Down + +Tear down the CDP Public Cloud by running the playbook: + +```bash +ansible-navigator run teardown.yml +``` diff --git a/public-cloud/gcp/datalake/ansible-navigator.yml b/public-cloud/gcp/datalake/ansible-navigator.yml new file mode 100644 index 0000000..c2e26f3 --- /dev/null +++ b/public-cloud/gcp/datalake/ansible-navigator.yml @@ -0,0 +1,61 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ansible-navigator: + playbook-artifact: + save-as: "runs/{playbook_name}-{time_stamp}.json" + + ansible-runner: + artifact-dir: runs + rotate-artifacts-count: 3 + + logging: + level: debug + append: False + + ansible: + inventory: + entries: + - inventory.ini + + execution-environment: + container-engine: docker + enabled: True + environment-variables: + pass: + - GCP_SERVICE_ACCOUNT_FILE + - CDP_PROFILE + set: + ANSIBLE_CALLBACK_WHITELIST: "ansible.posix.profile_tasks" + ANSIBLE_GATHERING: "smart" + ANSIBLE_DEPRECATION_WARNINGS: False + ANSIBLE_HOST_KEY_CHECKING: False + ANSIBLE_SSH_RETRIES: 10 + GCP_AUTH_KIND: "serviceaccount" + image: ghcr.io/cloudera-labs/cldr-runner:gcp-latest + pull: + policy: missing + volume-mounts: + - src: "${GCP_SERVICE_ACCOUNT_FILE}" + dest: "${GCP_SERVICE_ACCOUNT_FILE}" + - src: "~/.cdp" + dest: "/runner/.cdp" + options: "Z" + - src: "~/.ssh" + dest: "/runner/.ssh" + options: "Z" + container-options: + - "--network=host" diff --git a/public-cloud/gcp/datalake/definition.yml b/public-cloud/gcp/datalake/definition.yml new file mode 100644 index 0000000..043dfc7 --- /dev/null +++ b/public-cloud/gcp/datalake/definition.yml @@ -0,0 +1,44 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################ +# Global variables +################################################################################ +# Either define here or override using _extra variables_ in the CLI or AWX. +# For example, '-e name_prefix=basex' +# name_prefix: # You must specify a name prefix +# admin_password: # You must specify an admin password +infra_region: us-east1 # CSP region for infra +infra_type: gcp # CSP + +#gcp_project_id: # You must specify a GCP Project ID + +# Limit to the caller/controller +allowed_cidrs: "{{ lookup('ansible.builtin.url', 'https://api.ipify.org', wantlist=True) | product(['32']) | map('join', '/') | list }}" + +################################################################################ +# CDP Environment and Datalake variables +################################################################################ +env: + tunnel: no + public_endpoint_access: yes + +infra: + gcp: + project: "{{ gcp_project_id }}" + vpc: + extra_cidr: "{{ allowed_cidrs }}" + extra_ports: [22, 443] diff --git a/public-cloud/gcp/datalake/inventory.ini b/public-cloud/gcp/datalake/inventory.ini new file mode 100644 index 0000000..fef0f4a --- /dev/null +++ b/public-cloud/gcp/datalake/inventory.ini @@ -0,0 +1,15 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" \ No newline at end of file diff --git a/public-cloud/gcp/datalake/main.yml b/public-cloud/gcp/datalake/main.yml new file mode 100644 index 0000000..19daeb9 --- /dev/null +++ b/public-cloud/gcp/datalake/main.yml @@ -0,0 +1,45 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Initialization of GCP deployment + block: + - name: GCloud Auth using the Service Account + command: > + gcloud auth activate-service-account + --key-file={{ lookup('env', 'GCP_SERVICE_ACCOUNT_FILE') }} + + - name: Set the GCP project for GCloud + command: > + gcloud config set project {{ gcp_project_id }} + tags: + - always + +- name: Set up CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_setup.yml + +- name: Set up CDP Public Cloud (Env and DL example) + ansible.builtin.import_playbook: cloudera.exe.pbc_setup.yml diff --git a/public-cloud/gcp/datalake/teardown.yml b/public-cloud/gcp/datalake/teardown.yml new file mode 100644 index 0000000..082fa13 --- /dev/null +++ b/public-cloud/gcp/datalake/teardown.yml @@ -0,0 +1,45 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set up the cloudera-deploy variables + hosts: localhost + connection: local + gather_facts: yes + tasks: + - name: Read definition variables + ansible.builtin.include_role: + name: cloudera.exe.init_deployment + public: yes + when: init__completed is undefined + + - name: Initialization of GCP deployment + block: + - name: GCloud Auth using the Service Account + command: > + gcloud auth activate-service-account + --key-file={{ lookup('env', 'GCP_SERVICE_ACCOUNT_FILE') }} + + - name: Set the GCP project for GCloud + command: > + gcloud config set project {{ gcp_project_id }} + tags: + - always + +- name: Tear down CDP Public Cloud (Env and DL example) + ansible.builtin.import_playbook: cloudera.exe.pbc_teardown.yml + +- name: Tear down CDP Public Cloud infrastructure (Ansible-based) + ansible.builtin.import_playbook: cloudera.exe.pbc_infra_teardown.yml