diff --git a/omg/cmd/parser/__init__.py b/omg/cmd/parser/__init__.py index ec0e506..f4f13a9 100644 --- a/omg/cmd/parser/__init__.py +++ b/omg/cmd/parser/__init__.py @@ -1,13 +1,18 @@ import os +import json from tabulate import tabulate from omg.common.config import Config -from .etcd_out import etcd_member_list -from .etcd_out import etcd_endpoint_health -from .etcd_out import etcd_endpoint_status -from .etcd_out import etcd_show_all -from .alerts_out import alerts_summary -from .alerts_out import alerts_firing +from .etcd_out import ( + etcd_member_list, + etcd_endpoint_health, + etcd_endpoint_status, + etcd_show_all +) +from .alerts_out import ( + alerts_summary, alerts_firing +) +from . import prometheus_out as prom_out parser_map = { @@ -53,6 +58,30 @@ "helper": "Parser alerts firing exported by must-gather monitoring/alerts.json", "file_in": "monitoring/alerts.json", "fn_out": alerts_firing + }, + "prometheus-status-tsdb": + { + "command": "prometheus-status-tsdb", + "helper": "Parser TSDB status exported by must-gather monitoring/prometheus/status/tsdb.json", + "file_in": "", + "ignore_err": True, + "fn_out": prom_out.prom_status_tsdb + }, + "prometheus-runtime-build-info": + { + "command": "prometheus-runtime-build-info", + "helper": "Parser Runtime and Build info exported by must-gather monitoring/prometheus/status/{runtime,buildinfo}.json", + "file_in": "", + "ignore_err": True, + "fn_out": prom_out.prom_status_runtime_buildinfo + }, + "prometheus-status-config-diff": + { + "command": "prometheus-status-config-diff", + "helper": "Compare the configuration from both replicas exported by must-gather monitoring/prometheus/status/config.json", + "file_in": "", + "ignore_err": True, + "fn_out": prom_out.prom_status_config_diff } } @@ -76,25 +105,6 @@ def help(): print(tabulate(output_res, tablefmt="plain")) -def file_reader(path): - """ - Read a file to be parsed and return raw buffer. - """ - try: - full_path = os.path.join(Config().path, path) - with open(full_path, 'r') as f: - return f.read(), False - except IsADirectoryError as e: - print("WANING: ignoring file reader; Is a directory") - return "", True - except FileNotFoundError as e: - print(f"ERROR: file [{path}] not found") - return "", True - except Exception as e: - print(f"ERROR: Unknow error opening file {path}") - return "", True - - def print_table(data=None, headers=[], rows=[], fmt="psql"): """ Print a generic table. When headers and rows are defined, it will @@ -136,7 +146,7 @@ def parser_main(command=None, show=None): try: cmd = command[0] - buffer, err = file_reader(parser_map[cmd]['file_in']) + buffer, err = load_file(parser_map[cmd]['file_in']) if err: if 'ignore_err' in parser_map[cmd]: if not (parser_map[cmd]['ignore_err']): diff --git a/omg/cmd/parser/etcd_out.py b/omg/cmd/parser/etcd_out.py index 60e3489..f2125d1 100644 --- a/omg/cmd/parser/etcd_out.py +++ b/omg/cmd/parser/etcd_out.py @@ -1,18 +1,8 @@ import json - - -def _load_buffer_as_json(buffer): - """ - wrapper function to open a json from a given buffer. - Return json object and error - """ - try: - data = json.loads(buffer) - return data, False - except json.decoder.JSONDecodeError : - return "JSONDecodeError", True - except Exception as e: - return e, True +from omg.common.helper import ( + load_file, + load_json_file +) def etcd_member_list(buffer=None): @@ -21,7 +11,7 @@ def etcd_member_list(buffer=None): """ from . import print_table - data, err = _load_buffer_as_json(buffer) + data, err = load_json_buffer(buffer) h = data['header'] print(f"\nClusterID: {h['cluster_id']}, MemberID: {h['member_id']}, RaftTerm: {h['raft_term']}") @@ -34,7 +24,7 @@ def etcd_endpoint_health(buffer=None): """ from . import print_table - data, err = _load_buffer_as_json(buffer) + data, err = load_json_buffer(buffer) print_table(data=data) @@ -51,7 +41,7 @@ def sizeof_fmt(num, suffix='B'): num /= 1024.0 return ("%.1f %s%s" % (num, 'Yi', suffix)) - data, err = _load_buffer_as_json(buffer) + data, err = load_json_buffer(buffer) headers_map = [ "ENDPOINT", "ID", "VERSION", "DB SIZE", "IS LEADER", "IS LEARNER", "RAFT TERM", "RAFT INDEX", "RAFT APPLIED INDEX", @@ -93,7 +83,7 @@ def etcd_show_all(buffer=None): Show all etcd commands. """ from . import ( - parser_map, file_reader + parser_map ) etcd_cmds = [] @@ -105,7 +95,7 @@ def etcd_show_all(buffer=None): etcd_cmds.append(parser_map[cmd]) for cmd in etcd_cmds: - buffer, err = file_reader(cmd['file_in']) + buffer, err = load_file(cmd['file_in']) parser_map[cmd['command']]["fn_out"](buffer) return diff --git a/omg/cmd/parser/prometheus_out.py b/omg/cmd/parser/prometheus_out.py new file mode 100644 index 0000000..fee3381 --- /dev/null +++ b/omg/cmd/parser/prometheus_out.py @@ -0,0 +1,262 @@ +# -*- coding: utf-8 -*- +from tabulate import tabulate +from omg.common.helper import ( + fmt_sizeof, + fmt_countof, + fmt_date_from_ts, + load_json_file +) + + +def prom_replicas_reader(name, filename, output_header): + + files = [ + f"prometheus/prometheus-k8s-0/{filename}", + f"prometheus/prometheus-k8s-1/{filename}" + ] + err = False + table_fmt = "pretty" + + repl0, err = load_json_file(files[0]) + if err: + print(f"Error loading file {files[0]}") + return + + repl1, err = load_json_file(files[1]) + if err: + print(f"Error loading file {files[1]}") + return + + print(f">> {name} ({filename}) <<") + output_res = [] + + for dk in repl0["data"].keys(): + try: + output_res.append([ + dk, + repl0["data"][dk], + repl1["data"][dk] + ]) + except KeyError as e: + print(e) + + print(tabulate(output_res, + headers=output_header, + tablefmt=table_fmt) + ) + + +def prom_status_tsdb(buffer=None): + """ + Show Prometheus Status TSDB. + """ + files = [ + "prometheus/prometheus-k8s-0/status/tsdb.json", + "prometheus/prometheus-k8s-1/status/tsdb.json" + ] + err = False + table_fmt = "pretty" + + repl0, err = load_json_file(files[0]) + if err: + print(f"Error loading file {files[0]}") + return + + repl1, err = load_json_file(files[1]) + if err: + print(f"Error loading file {files[1]}") + return + + print(">> headStats <<") + output_res = [] + output_header = [ + "metric", + "prometheus-k8s-0", + "prometheus-k8s-1" + ] + + output_res.append([ + "numSeries", + fmt_countof(repl0["data"]["headStats"]["numSeries"]), + fmt_countof(repl1["data"]["headStats"]["numSeries"]) + ]) + output_res.append([ + "chunkCount", + fmt_countof(repl0["data"]["headStats"]["chunkCount"]), + fmt_countof(repl1["data"]["headStats"]["chunkCount"]) + ]) + output_res.append([ + "minTime", + fmt_date_from_ts(repl0["data"]["headStats"]["minTime"]), + fmt_date_from_ts(repl1["data"]["headStats"]["minTime"]) + ]) + output_res.append([ + "maxTime", + fmt_date_from_ts(repl0["data"]["headStats"]["maxTime"]), + fmt_date_from_ts(repl1["data"]["headStats"]["maxTime"]) + ]) + + print(tabulate(output_res, headers=output_header, tablefmt=table_fmt)) + + print("\n>> seriesCountByMetricName <<") + metric_name = "seriesCountByMetricName" + output_res = [] + output_header = [ + "Top#N", + "MetricName(prometheus-k8s-0)", + "Count(prometheus-k8s-0)", + "MetricName(prometheus-k8s-1)", + "Count(prometheus-k8s-1)" + ] + + for idx in range(0, len(repl0["data"][metric_name])): + output_res.append([ + f"#{str(idx+1)}", + repl0["data"][metric_name][idx]["name"], + fmt_countof(repl0["data"][metric_name][idx]["value"]), + repl1["data"][metric_name][idx]["name"], + fmt_countof(repl1["data"][metric_name][idx]["value"]) + ]) + + print(tabulate(output_res, headers=output_header, tablefmt=table_fmt, colalign=("right"))) + + print("\n>> labelValueCountByLabelName <<") + metric_name = "labelValueCountByLabelName" + output_res = [] + output_header = [ + "Top#N", + "MetricName(prometheus-k8s-0)", + "Count(prometheus-k8s-0)", + "MetricName(prometheus-k8s-1)", + "Count(prometheus-k8s-1)" + ] + + for idx in range(0, len(repl0["data"][metric_name])): + output_res.append([ + f"#{str(idx+1)}", + repl0["data"][metric_name][idx]["name"], + fmt_countof(repl0["data"][metric_name][idx]["value"]), + repl1["data"][metric_name][idx]["name"], + fmt_countof(repl1["data"][metric_name][idx]["value"]) + ]) + + print(tabulate(output_res, headers=output_header, tablefmt=table_fmt, colalign=("right"))) + + print("\n>> memoryInBytesByLabelName <<") + metric_name = "memoryInBytesByLabelName" + output_res = [] + output_header = [ + "Top#N", + "MetricName(prometheus-k8s-0)", + "Size(prometheus-k8s-0)", + "MetricName(prometheus-k8s-1)", + "Size(prometheus-k8s-1)" + ] + + for idx in range(0, len(repl0["data"][metric_name])): + output_res.append([ + f"#{str(idx+1)}", + repl0["data"][metric_name][idx]["name"], + fmt_sizeof(repl0["data"][metric_name][idx]["value"]), + repl1["data"][metric_name][idx]["name"], + fmt_sizeof(repl1["data"][metric_name][idx]["value"]) + ]) + + print(tabulate(output_res, headers=output_header, tablefmt=table_fmt, colalign=("right"))) + + print("\n>> seriesCountByLabelValuePair <<") + metric_name = "seriesCountByLabelValuePair" + output_res = [] + output_header = [ + "Top#N", + "MetricName(prometheus-k8s-0)", + "Count(prometheus-k8s-0)", + "MetricName(prometheus-k8s-1)", + "Count(prometheus-k8s-1)" + ] + + for idx in range(0, len(repl0["data"][metric_name])): + output_res.append([ + f"#{str(idx+1)}", + repl0["data"][metric_name][idx]["name"], + fmt_countof(repl0["data"][metric_name][idx]["value"]), + repl1["data"][metric_name][idx]["name"], + fmt_countof(repl1["data"][metric_name][idx]["value"]) + ]) + + print(tabulate(output_res, headers=output_header, tablefmt=table_fmt, colalign=("right"))) + + +def prom_status_runtimeinfo(buffer=None): + """ + Show Prometheus Status: Runtime Information (/api/v1/status/runtimeinfo). + """ + report_name = "runtimeinfo" + filename = f"status/{report_name}.json" + + output_header = [ + f"{report_name}", + "prometheus-k8s-0", + "prometheus-k8s-1" + ] + prom_replicas_reader(report_name, filename, output_header) + + +def prom_status_buildinfo(buffer=None): + """ + Show Prometheus Status: Build Information (/api/v1/status/buildinfo). + """ + report_name = "buildinfo" + filename = f"status/{report_name}.json" + + output_header = [ + f"{report_name}", + "prometheus-k8s-0", + "prometheus-k8s-1" + ] + prom_replicas_reader(report_name, filename, output_header) + + +def prom_status_flags(buffer=None): + """ + Show Prometheus Status: Build Information (/api/v1/status/flags). + """ + report_name = "flags" + filename = f"status/{report_name}.json" + + output_header = [ + f"{report_name}", + "prometheus-k8s-0", + "prometheus-k8s-1" + ] + prom_replicas_reader(report_name, filename, output_header) + + +def prom_status_runtime_buildinfo(buffer=None): + prom_status_buildinfo(buffer) + prom_status_runtimeinfo(buffer) + prom_status_flags(buffer) + return + + +def prom_status_config_diff(buffer=None): + """ + Show Prometheus Diff: Config (/api/v1/status/config). + """ + report_name = "config" + filename = f"status/{report_name}.json" + + buffer0, err0 = load_json_file(f"prometheus/prometheus-k8s-0/{filename}") + buffer1, err1 = load_json_file(f"prometheus/prometheus-k8s-1/{filename}") + + from difflib import Differ + + b0 = buffer0["data"]["yaml"].splitlines(keepends=True) + b1 = buffer1["data"]["yaml"].splitlines(keepends=True) + d = Differ() + line = -1 + for l in list(d.compare(b0, b1)): + line += 1 + if l.startswith(' '): + continue + print(l.replace('\n', ' ')) diff --git a/omg/common/helper.py b/omg/common/helper.py index c62df93..9736ea2 100644 --- a/omg/common/helper.py +++ b/omg/common/helper.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import os import sys +import json from datetime import datetime, timedelta from dateutil.parser import parse from dateutil.relativedelta import relativedelta @@ -132,3 +133,78 @@ def extract_labels(o): return "" else: return "" + + +# Helper to format outputs +def fmt_sizeof(num, suffix='B'): + for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: + if abs(num) < 1024.0: + return ("%3.1f %s%s" % (num, unit, suffix)) + num /= 1024.0 + return ("%.1f %s%s" % (num, 'Yi', suffix)) + + +def fmt_countof(num, suffix=''): + for unit in ['','K','M','G','T','P','E','Z']: + if abs(num) < 1000.0: + return ("%3.3f %s%s" % (num, unit, suffix)) + num /= 1000.0 + return ("%.3f %s%s" % (num, 'Yi', suffix)) + + +def fmt_date_from_ts(ts): + from datetime import datetime + return str(datetime.fromtimestamp(ts / 1e3).strftime("%Y-%m-%d %H:%M:%S")) + + +# File handlers +def load_file(path): + """ + Read a file to be parsed and return raw buffer. + """ + try: + full_path = os.path.join(Config().path, path) + with open(full_path, 'r') as f: + return f.read(), False + except IsADirectoryError as e: + print("WANING: ignoring file reader; Is a directory") + return "", True + except FileNotFoundError as e: + print(f"ERROR: file [{path}] not found") + return "", True + except Exception as e: + print(f"ERROR: Unknow error opening file {path}") + return "", True + + +def load_json_buffer(buffer): + """ + wrapper function to open a json from a given buffer. + Return json object and error + """ + try: + data = json.loads(buffer) + return data, False + except json.decoder.JSONDecodeError: + return "JSONDecodeError", True + except Exception as e: + return e, True + + +# File handlers +def load_json_file(path): + """ + wrapper function to open a json from a given buffer. + Return json object and error + """ + try: + buffer, err = load_file(path) + if err: + print(f"ERROR: reading file {path}") + return + data = json.loads(buffer) + return data, False + except json.decoder.JSONDecodeError: + return "JSONDecodeError", True + except Exception as e: + return e, True