diff --git a/topic/autoscaling/autoscaling_script.ipynb b/topic/autoscaling/autoscaling_script.ipynb deleted file mode 100644 index b811bcea..00000000 --- a/topic/autoscaling/autoscaling_script.ipynb +++ /dev/null @@ -1,126 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 7, - "id": "4c9711ff-fe70-45d1-a79f-d71df86e64cd", - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from requests.auth import HTTPBasicAuth\n", - "from datetime import datetime\n", - "\n", - "auth_data = HTTPBasicAuth('API-KEY', 'API-SECRET')\n", - "\n", - "date_format = '%Y-%m-%dT%H:%M:%S.%f'\n", - "\n", - "def get_metrics(organization_id):\n", - " api_url = f'https://console.cratedb.cloud/api/v2/organizations/{organization_id}/metrics/prometheus/'\n", - " headers = {'accept': 'plain/text'}\n", - " response = requests.get(api_url, auth=auth_data)\n", - " return response.text\n", - "\n", - "def num_shards(metrics):\n", - " for row in metrics.split('\\n'):\n", - " line = row.split(' ')\n", - " if line[0].startswith('crate_node{name=\"shard_stats\"') and 'property=\"total\"' in line[0] and 'crate-data-hot-97ebb337-b44e-446f-adca-a092fd1df0aa' in line[0]:\n", - " return float(line[1])\n", - "\n", - "def get_cluster_status(cluster_id):\n", - " api_url = f'https://console.cratedb.cloud/api/v2/clusters/{cluster_id}/'\n", - " return requests.get(api_url, auth=auth_data).json()\n", - "\n", - "def get_cluster_running_op(cluster_status, cluster_id):\n", - " if (datetime.now() - datetime.strptime(cluster_status['health']['last_seen'], date_format)).seconds > 30:\n", - " cluster_status = get_cluster_status(cluster_id)\n", - " return cluster_status['health']['running_operation']\n", - "\n", - "def get_cluster_num_nodes(cluster_status, cluster_id):\n", - " if (datetime.now() - datetime.strptime(cluster_status['health']['last_seen'], date_format)).seconds > 30:\n", - " cluster_status = get_cluster_status(cluster_id)\n", - " return cluster_status['num_nodes'] - 1\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "c14c779c-1809-4e19-b995-e850cfa667ba", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Current number of shards: 25.0\n", - "Start scaling out from 3 to 4\n" - ] - } - ], - "source": [ - "import time\n", - "\n", - "delay_seconds = 5\n", - "max_num_shard = 30\n", - "\n", - "organization_id = 'Use your organisation ID here'\n", - "cluster_id = 'Use your Cluster ID here'\n", - "\n", - "api_url = f'https://console.cratedb.cloud/api/v2/clusters/{cluster_id}/scale/'\n", - "\n", - "while True:\n", - " metrics = get_metrics(organization_id) # Pass organization_id to get_metrics function\n", - " cluster_status = get_cluster_status(cluster_id)\n", - " num = num_shards(metrics)\n", - " if num is not None: # Check if num is not None\n", - " print(f'Current number of shards: {num}')\n", - " if num > (0.8 * max_num_shard):\n", - " num_nodes = get_cluster_num_nodes(cluster_status, cluster_id) # Pass cluster_id to get_cluster_num_nodes function\n", - " print(f'Start scaling out from {num_nodes + 1} to {num_nodes + 2}')\n", - " requests.put(api_url, json={'product_unit':num_nodes + 1}, auth=auth_data)\n", - " time.sleep(delay_seconds)\n", - " while get_cluster_running_op(cluster_status, cluster_id) != '':\n", - " cluster_status = get_cluster_status(cluster_id)\n", - " continue\n", - " print('Scaled up successfully!')\n", - " \n", - " elif num < (0.5 * max_num_shard):\n", - " num_nodes = get_cluster_num_nodes(cluster_status, cluster_id) # Pass cluster_id to get_cluster_num_nodes function\n", - " print(f'Start scaling down from {num_nodes + 1} to {num_nodes}')\n", - " requests.put(api_url, json={'product_unit':num_nodes - 1}, auth=auth_data)\n", - " time.sleep(delay_seconds)\n", - " while get_cluster_running_op(cluster_status, cluster_id) != '':\n", - " cluster_status = get_cluster_status(cluster_id)\n", - " continue\n", - " print('Scaled down successfully!')\n", - " else:\n", - " print('Nothing to do!')\n", - " else:\n", - " print('Failed to retrieve shard metrics.')\n", - " time.sleep(delay_seconds)\n", - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}