From 5cac5beb7223b85db0e4496be634a9d77629767c Mon Sep 17 00:00:00 2001 From: Anshu Prateek Date: Wed, 25 Feb 2015 12:00:19 -0800 Subject: [PATCH 1/7] Adding collectinfo to asadm Features are currently at par with asmonitor's collectinfo --- lib/controller.py | 96 +++++++++++++++++++++++++++++++++++++++++++++++ lib/util.py | 27 +++++++++++++ 2 files changed, 123 insertions(+) diff --git a/lib/controller.py b/lib/controller.py index fbc81545..fc76675a 100644 --- a/lib/controller.py +++ b/lib/controller.py @@ -14,6 +14,7 @@ from lib.controllerlib import * from lib import util +import time,os,sys,platform,shutil def flip_keys(orig_data): new_data = {} @@ -44,6 +45,7 @@ def __init__(self, seed_nodes=[('127.0.0.1',3000)] , 'cluster':ClusterController , '!':ShellController , 'shell':ShellController + , 'collectinfo':CollectinfoController } @CommandHelp('Terminate session') @@ -521,3 +523,97 @@ def do_dun(self, line): def do_undun(self, line): results = self.cluster.infoUndun(self.mods['line'], nodes=self.nodes) self.view.dun(results, self.cluster, **self.mods) + +@CommandHelp('"collectinfo" is used to collect system stats on the local node.') +class CollectinfoController(CommandController): + def collect_local_file(self,src,dest_dir): + shutil.copy2(src, dest_dir) + + def write_log(self,collectedinfo,src_file='/var/log/aerospike/*log'): + aslogdir = '/tmp/as_log_' + str(time.time()) + aslogfile = aslogdir + '/ascollectinfo.log' + os.mkdir(aslogdir) + f = open(str(aslogfile), 'w') + f.write(str(collectedinfo)) + f.close() + self.collect_local_file(src_file,aslogdir) + util.shell_command(["tar -czvf " + aslogdir + ".tgz " + aslogdir]) + sys.stderr.write("\x1b[2J\x1b[H") + print "\n\n\nFiles in " + aslogdir + " and " + aslogdir + ".tgz saved. Please send tgz archive to support@aerospike.com" + print "END OF ASCOLLECTINFO" + + def do_collectinfo(self, line): + capture_stdout = util.capture_stdout + collect_output = '' + collect_output = time.strftime("%Y-%m-%d %H:%M:%S UTC\n", time.gmtime()) + info_params = ['network','service', 'namespace', 'xdr', 'sindex'] + show_params = ['config', 'distribution', 'latency', 'statistics'] + shell_cmds = ['date', + 'hostname', + 'ifconfig', + 'uname -a', + 'lsb_release -a', + 'ls /etc|grep release|xargs -I f cat /etc/f', + 'rpm -qa|grep -E "citrus|aero"', + 'dpkg -l|grep -E "citrus|aero"', + 'tail -n 10000 /var/log/aerospike/*.log', + 'tail -n 10000 /var/log/citrusleaf.log', + 'tail -n 10000 /var/log/*xdr.log', + 'netstat -pant|grep 3000', + 'top -n3 -b', + 'free -m', + 'df -h', + 'ls /sys/block/{sd*,xvd*}/queue/rotational |xargs -I f sh -c "echo f; cat f;"', + 'ls /sys/block/{sd*,xvd*}/device/model |xargs -I f sh -c "echo f; cat f;"', + 'lsof', + 'dmesg', + 'iostat -x 1 10', + 'vmstat -s', + 'vmstat -m', + 'iptables -L', + 'cat /etc/aerospike/aerospike.conf', + 'cat /etc/citrusleaf/citrusleaf.conf', + ] + + def collect_sys(self): + sys_data = '' + lsof_cmd='sudo lsof|grep `sudo ps aux|grep -v grep|grep -E \'asd|cld\'|awk \'{print $2}\'` 2>/dev/null' + print util.shell_command([lsof_cmd]) + print platform.platform() + smd_home = '/opt/aerospike/smd' + if os.path.isdir(smd_home): + smd_files = [ f for f in os.listdir(smd_home) if os.path.isfile(os.path.join(smd_home, f)) ] + for f in smd_files: + smd_f = os.path.join(smd_home, f) + print smd_f + smd_fp = open(smd_f, 'r') + print smd_fp.read() + smd_fp.close() + + for cmd in shell_cmds: + collect_output += capture_stdout(util.shell_command, [cmd]) + try: + log_location = '/var/log/aerospike/aerospike.log' + cinfo = InfoController() + for info_param in info_params: + collect_output += capture_stdout(cinfo,[info_param]) + do_show = ShowController() + for show_param in show_params: + collect_output += capture_stdout(do_show,[show_param]) + # Below is not optimum, we should query only localhost + logs = self.cluster.info('logs') + for i in logs: + log_location = logs[i].split(':')[1] + cmd = 'tail -n 10000 ' + log_location + if log_location != '/var/log/aerospike/aerospike.log': + collect_output += capture_stdout(util.shell_command, [cmd]) + + except Exception as e: + collect_output += str(e) + sys.stdout = sys.__stdout__ + + collect_output += capture_stdout(collect_sys) + self.write_log(collect_output,log_location) + + def _do_default(self, line): + self.do_collectinfo(line) diff --git a/lib/util.py b/lib/util.py index 1d8e87cc..2daa0cdf 100644 --- a/lib/util.py +++ b/lib/util.py @@ -18,6 +18,7 @@ from time import time import subprocess import pipes +import sys,StringIO def info_to_dict(value, delimiter = ';'): """ @@ -122,3 +123,29 @@ def shell_command(command): else: return out, err +# ## Many of the view functions just do a print without any valid returns +# ## This is to capture such prints so that they can be used as returns +def capture_stdout(func,line=[''],printMsg=True): + name = '' + if printMsg is True: + try: + name = func.func_name + except Exception: + pass + print "Data collection for " + name +str(line) + " in progress.." + sys.stdout.flush() + old = sys.stdout + capturer = StringIO.StringIO() + sys.stdout = capturer + print '===ASCOLLECTINFO===' + print name + str(line) + o = func(line) + if isinstance(o,object): + try: + for item in o: + print item + except Exception: + pass + output = capturer.getvalue() + sys.stdout = old + return output \ No newline at end of file From e98cfb4e557109cb6cc2c544a74b7553887aa804 Mon Sep 17 00:00:00 2001 From: Anshu Prateek Date: Fri, 27 Feb 2015 14:34:43 +0530 Subject: [PATCH 2/7] changes after review --- lib/controller.py | 95 +++++++++++++++++++++++++++++++++++++++++++++++ lib/util.py | 29 +++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/lib/controller.py b/lib/controller.py index fbc81545..7a50408c 100644 --- a/lib/controller.py +++ b/lib/controller.py @@ -14,6 +14,7 @@ from lib.controllerlib import * from lib import util +import time, os, sys, platform, shutil def flip_keys(orig_data): new_data = {} @@ -44,6 +45,7 @@ def __init__(self, seed_nodes=[('127.0.0.1',3000)] , 'cluster':ClusterController , '!':ShellController , 'shell':ShellController + , 'collectinfo':CollectinfoController } @CommandHelp('Terminate session') @@ -521,3 +523,96 @@ def do_dun(self, line): def do_undun(self, line): results = self.cluster.infoUndun(self.mods['line'], nodes=self.nodes) self.view.dun(results, self.cluster, **self.mods) + +@CommandHelp('"collectinfo" is used to collect system stats on the local node.') +class CollectinfoController(CommandController): + def collect_local_file(self,src,dest_dir): + shutil.copy2(src, dest_dir) + + def write_log(self,collectedinfo,src_file='/var/log/aerospike/*log'): + aslogdir = '/tmp/as_log_' + str(time.time()) + aslogfile = aslogdir + '/ascollectinfo.log' + os.mkdir(aslogdir) + f = open(str(aslogfile), 'w') + f.write(str(collectedinfo)) + f.close() + self.collect_local_file(src_file,aslogdir) + util.shell_command(["tar -czvf " + aslogdir + ".tgz " + aslogdir]) + sys.stderr.write("\x1b[2J\x1b[H") + print "\n\n\nFiles in " + aslogdir + " and " + aslogdir + ".tgz saved. Please send tgz archive to support@aerospike.com" + print "END OF ASCOLLECTINFO" + + def main_collectinfo(self, line): + capture_stdout = util.capture_stdout + collect_output = time.strftime("%Y-%m-%d %H:%M:%S UTC\n", time.gmtime()) + info_params = ['network','service', 'namespace', 'xdr', 'sindex'] + show_params = ['config', 'distribution', 'latency', 'statistics'] + shell_cmds = ['date', + 'hostname', + 'ifconfig', + 'uptime', + 'uname -a', + 'lsb_release -a', + 'ls /etc|grep release|xargs -I f cat /etc/f', + 'rpm -qa|grep -E "citrus|aero"', + 'dpkg -l|grep -E "citrus|aero"', + 'tail -n 10000 /var/log/aerospike/*.log', + 'tail -n 10000 /var/log/citrusleaf.log', + 'tail -n 10000 /var/log/*xdr.log', + 'netstat -pant|grep 3000', + 'top -n3 -b', + 'free -m', + 'df -h', + 'ls /sys/block/{sd*,xvd*}/queue/rotational |xargs -I f sh -c "echo f; cat f;"', + 'ls /sys/block/{sd*,xvd*}/device/model |xargs -I f sh -c "echo f; cat f;"', + 'lsof', + 'dmesg', + 'iostat -x 1 10', + 'vmstat -s', + 'vmstat -m', + 'iptables -L', + 'cat /etc/aerospike/aerospike.conf', + 'cat /etc/citrusleaf/citrusleaf.conf', + ] + + def collect_sys(self): + lsof_cmd='sudo lsof|grep `sudo ps aux|grep -v grep|grep -E \'asd|cld\'|awk \'{print $2}\'` 2>/dev/null' + print util.shell_command([lsof_cmd]) + print platform.platform() + smd_home = '/opt/aerospike/smd' + if os.path.isdir(smd_home): + smd_files = [ f for f in os.listdir(smd_home) if os.path.isfile(os.path.join(smd_home, f)) ] + for f in smd_files: + smd_f = os.path.join(smd_home, f) + print smd_f + smd_fp = open(smd_f, 'r') + print smd_fp.read() + smd_fp.close() + + for cmd in shell_cmds: + collect_output += capture_stdout(util.shell_command, [cmd]) + try: + log_location = '/var/log/aerospike/aerospike.log' + cinfo = InfoController() + for info_param in info_params: + collect_output += capture_stdout(cinfo,[info_param]) + do_show = ShowController() + for show_param in show_params: + collect_output += capture_stdout(do_show,[show_param]) + # Below is not optimum, we should query only localhost + logs = self.cluster.info('logs') + for i in logs: + log_location = logs[i].split(':')[1] + cmd = 'tail -n 10000 ' + log_location + if log_location != '/var/log/aerospike/aerospike.log': + collect_output += capture_stdout(util.shell_command, [cmd]) + + except Exception as e: + collect_output += str(e) + sys.stdout = sys.__stdout__ + + collect_output += capture_stdout(collect_sys) + self.write_log(collect_output,log_location) + + def _do_default(self, line): + self.main_collectinfo(line) \ No newline at end of file diff --git a/lib/util.py b/lib/util.py index 1d8e87cc..4d6e0854 100644 --- a/lib/util.py +++ b/lib/util.py @@ -18,6 +18,7 @@ from time import time import subprocess import pipes +import sys, StringIO def info_to_dict(value, delimiter = ';'): """ @@ -122,3 +123,31 @@ def shell_command(command): else: return out, err + # Redirecting the stdout to use the output elsewhere +def capture_stdout(func,line=[''],collectinfo_msg=True,print_msg=''): + name = '' + if collectinfo_msg is True: + try: + name = func.func_name + except Exception: + pass + print "Data collection for " + name +str(line) + " in progress.." + sys.stdout.flush() + old = sys.stdout + capturer = StringIO.StringIO() + sys.stdout = capturer + if collectinfo_msg is True: + print '===ASCOLLECTINFO===' + if print_msg != '': + print print_msg + print name + str(line) + o = func(line) + if isinstance(o, object): + try: + for item in o: + print item + except Exception: + pass + output = capturer.getvalue() + sys.stdout = old + return output \ No newline at end of file From 9db9d341432d9112618533cf57758db0ea60cbc7 Mon Sep 17 00:00:00 2001 From: Anshu Prateek Date: Fri, 6 Mar 2015 12:37:38 +0530 Subject: [PATCH 3/7] added AWS metadata collection + review changes --- lib/controller.py | 106 ++++++++++++++++++++++++++++++++++++---------- lib/util.py | 30 ++++--------- 2 files changed, 92 insertions(+), 44 deletions(-) diff --git a/lib/controller.py b/lib/controller.py index 6bd39f56..ad4e05e3 100644 --- a/lib/controller.py +++ b/lib/controller.py @@ -14,7 +14,7 @@ from lib.controllerlib import * from lib import util -import time, os, sys, platform, shutil +import time, os, sys, platform, shutil, requests def flip_keys(orig_data): new_data = {} @@ -528,11 +528,31 @@ def do_undun(self, line): class CollectinfoController(CommandController): def collect_local_file(self,src,dest_dir): shutil.copy2(src, dest_dir) + + def collectinfo_content(self,func,parm=''): + name = '' + capture_stdout = util.capture_stdout + sep = "====ASCOLLECTINFO====\n" + try: + name = func.func_name + except Exception: + pass + info_line = "[INFO] Data collection for " + name +str(parm) + " in progress.." + print info_line + sep += info_line+"\n" + + if func == 'shell': + o,e = util.shell_command(parm) + else: + o = capture_stdout(func,parm) + return sep+o def write_log(self,collectedinfo,src_file='/var/log/aerospike/*log'): aslogdir = '/tmp/as_log_' + str(time.time()) aslogfile = aslogdir + '/ascollectinfo.log' os.mkdir(aslogdir) + remove_color = re.compile(r'\x1B\[[^A-Za-z]*[A-Za-z]') + collectedinfo = remove_color.sub('', collectedinfo) f = open(str(aslogfile), 'w') f.write(str(collectedinfo)) f.close() @@ -541,9 +561,63 @@ def write_log(self,collectedinfo,src_file='/var/log/aerospike/*log'): sys.stderr.write("\x1b[2J\x1b[H") print "\n\n\nFiles in " + aslogdir + " and " + aslogdir + ".tgz saved. Please send tgz archive to support@aerospike.com" print "END OF ASCOLLECTINFO" + + + def get_metadata(self,response_str,prefix=''): + aws_c = '' + aws_timeout = 1 + aws_metadata_base_url = 'http://169.254.169.254/latest/meta-data' + prefix_o = prefix + if prefix_o == '/': + prefix = '' + for rsp in response_str.split("\n"): + if rsp[-1:] == '/': + if prefix_o == '': #First level child + rsp_p = rsp.strip('/') + else: + rsp_p = rsp + self.get_metadata(rsp_p,prefix) + else: + meta_url = aws_metadata_base_url+prefix+'/'+rsp + r = requests.get(meta_url,timeout=aws_timeout) + if r.status_code != 404: + aws_c += rsp +'\n'+r.text +"\n" + return aws_c + + def get_awsdata(self,line): + aws_rsp = '' + aws_timeout = 1 + aws_metadata_base_url = 'http://169.254.169.254/latest/meta-data' + try: + r = requests.get(aws_metadata_base_url,timeout=aws_timeout) + if r.status_code == 200: + print "This is in AWS" + rsp = r.text + aws_rsp += self.get_metadata(rsp,'/') + print aws_rsp + else: + aws_rsp = "Not in AWS" + print aws_rsp + + except Exception as e: + print e + print "Not in AWS" + + def collect_sys(self,line): + lsof_cmd='sudo lsof|grep `sudo ps aux|grep -v grep|grep -E \'asd|cld\'|awk \'{print $2}\'` 2>/dev/null' + print util.shell_command([lsof_cmd]) + print platform.platform() + smd_home = '/opt/aerospike/smd' + if os.path.isdir(smd_home): + smd_files = [ f for f in os.listdir(smd_home) if os.path.isfile(os.path.join(smd_home, f)) ] + for f in smd_files: + smd_f = os.path.join(smd_home, f) + print smd_f + smd_fp = open(smd_f, 'r') + print smd_fp.read() + smd_fp.close() def main_collectinfo(self, line): - capture_stdout = util.capture_stdout collect_output = time.strftime("%Y-%m-%d %H:%M:%S UTC\n", time.gmtime()) info_params = ['network','service', 'namespace', 'xdr', 'sindex'] show_params = ['config', 'distribution', 'latency', 'statistics'] @@ -575,43 +649,29 @@ def main_collectinfo(self, line): 'cat /etc/citrusleaf/citrusleaf.conf', ] - def collect_sys(self): - lsof_cmd='sudo lsof|grep `sudo ps aux|grep -v grep|grep -E \'asd|cld\'|awk \'{print $2}\'` 2>/dev/null' - print util.shell_command([lsof_cmd]) - print platform.platform() - smd_home = '/opt/aerospike/smd' - if os.path.isdir(smd_home): - smd_files = [ f for f in os.listdir(smd_home) if os.path.isfile(os.path.join(smd_home, f)) ] - for f in smd_files: - smd_f = os.path.join(smd_home, f) - print smd_f - smd_fp = open(smd_f, 'r') - print smd_fp.read() - smd_fp.close() - for cmd in shell_cmds: - collect_output += capture_stdout(util.shell_command, [cmd]) + collect_output += self.collectinfo_content('shell',[cmd]) try: + pass log_location = '/var/log/aerospike/aerospike.log' cinfo = InfoController() for info_param in info_params: - collect_output += capture_stdout(cinfo,[info_param]) + collect_output += self.collectinfo_content(cinfo,[info_param]) do_show = ShowController() for show_param in show_params: - collect_output += capture_stdout(do_show,[show_param]) + collect_output += self.collectinfo_content(do_show,[show_param]) # Below is not optimum, we should query only localhost logs = self.cluster.info('logs') for i in logs: log_location = logs[i].split(':')[1] cmd = 'tail -n 10000 ' + log_location if log_location != '/var/log/aerospike/aerospike.log': - collect_output += capture_stdout(util.shell_command, [cmd]) - + collect_output += self.collectinfo_content('shell',[cmd]) except Exception as e: collect_output += str(e) sys.stdout = sys.__stdout__ - - collect_output += capture_stdout(collect_sys) + collect_output += self.collectinfo_content(self.collect_sys) + collect_output += self.collectinfo_content(self.get_awsdata) self.write_log(collect_output,log_location) def _do_default(self, line): diff --git a/lib/util.py b/lib/util.py index 4d6e0854..fc589297 100644 --- a/lib/util.py +++ b/lib/util.py @@ -124,30 +124,18 @@ def shell_command(command): return out, err # Redirecting the stdout to use the output elsewhere -def capture_stdout(func,line=[''],collectinfo_msg=True,print_msg=''): - name = '' - if collectinfo_msg is True: - try: - name = func.func_name - except Exception: - pass - print "Data collection for " + name +str(line) + " in progress.." - sys.stdout.flush() +def capture_stdout(func,line=''): + """ + Redirecting the stdout to use the output elsewhere + """ + + sys.stdout.flush() old = sys.stdout capturer = StringIO.StringIO() sys.stdout = capturer - if collectinfo_msg is True: - print '===ASCOLLECTINFO===' - if print_msg != '': - print print_msg - print name + str(line) - o = func(line) - if isinstance(o, object): - try: - for item in o: - print item - except Exception: - pass + + func(line) + output = capturer.getvalue() sys.stdout = old return output \ No newline at end of file From 25b1691c31e508a4a6766fcca6f213a16596cc31 Mon Sep 17 00:00:00 2001 From: Anshu Prateek Date: Fri, 6 Mar 2015 22:01:24 +0530 Subject: [PATCH 4/7] added more info, moved asd stats before system stats like asmonitor --- lib/controller.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/controller.py b/lib/controller.py index ad4e05e3..48c4050f 100644 --- a/lib/controller.py +++ b/lib/controller.py @@ -532,7 +532,7 @@ def collect_local_file(self,src,dest_dir): def collectinfo_content(self,func,parm=''): name = '' capture_stdout = util.capture_stdout - sep = "====ASCOLLECTINFO====\n" + sep = "\n====ASCOLLECTINFO====\n" try: name = func.func_name except Exception: @@ -543,9 +543,11 @@ def collectinfo_content(self,func,parm=''): if func == 'shell': o,e = util.shell_command(parm) + elif func == 'cluster': + o = self.cluster.info(parm) else: o = capture_stdout(func,parm) - return sep+o + return sep+str(o) def write_log(self,collectedinfo,src_file='/var/log/aerospike/*log'): aslogdir = '/tmp/as_log_' + str(time.time()) @@ -621,6 +623,20 @@ def main_collectinfo(self, line): collect_output = time.strftime("%Y-%m-%d %H:%M:%S UTC\n", time.gmtime()) info_params = ['network','service', 'namespace', 'xdr', 'sindex'] show_params = ['config', 'distribution', 'latency', 'statistics'] + cluster_params = ['service', + 'services', + 'xdr-min-lastshipinfo:', + 'dump-fabric:', + 'dump-hb:', + 'dump-migrates:', + 'dump-msgs:', + 'dump-paxos:', + 'dump-smd:', + 'dump-wb:', + 'dump-wb-summary:', + 'dump-wr:', + 'sindex-dump:' + ] shell_cmds = ['date', 'hostname', 'ifconfig', @@ -648,11 +664,8 @@ def main_collectinfo(self, line): 'cat /etc/aerospike/aerospike.conf', 'cat /etc/citrusleaf/citrusleaf.conf', ] - - for cmd in shell_cmds: - collect_output += self.collectinfo_content('shell',[cmd]) + try: - pass log_location = '/var/log/aerospike/aerospike.log' cinfo = InfoController() for info_param in info_params: @@ -660,6 +673,8 @@ def main_collectinfo(self, line): do_show = ShowController() for show_param in show_params: collect_output += self.collectinfo_content(do_show,[show_param]) + for cluster_param in cluster_params: + collect_output += self.collectinfo_content('cluster',cluster_param) # Below is not optimum, we should query only localhost logs = self.cluster.info('logs') for i in logs: @@ -670,6 +685,8 @@ def main_collectinfo(self, line): except Exception as e: collect_output += str(e) sys.stdout = sys.__stdout__ + for cmd in shell_cmds: + collect_output += self.collectinfo_content('shell',[cmd]) collect_output += self.collectinfo_content(self.collect_sys) collect_output += self.collectinfo_content(self.get_awsdata) self.write_log(collect_output,log_location) From 27ef96f0b763519137e365928ec181d320f74f05 Mon Sep 17 00:00:00 2001 From: Kevin Porter Date: Wed, 11 Mar 2015 23:32:58 -0700 Subject: [PATCH 5/7] Added ability to disable/enable terminal colors --- lib/terminal.py | 205 +++++++++++++++++++++++++----------------------- 1 file changed, 108 insertions(+), 97 deletions(-) diff --git a/lib/terminal.py b/lib/terminal.py index 963a3c56..83e03073 100644 --- a/lib/terminal.py +++ b/lib/terminal.py @@ -15,104 +15,115 @@ import sys from inspect import isfunction -# Real terminal +def enable_color(is_enable): + global _add_it, _remove_it, _reset + global sclear, sbold, sdim, snormal, sunderline + global fgblack, fgred, fggreen, fgyellow, fgblue, fgmagenta, fgcyan, fgwhite + global bgblack, bgred, bggreen, bgyellow, bgblue, bgmagenta, bgcyan, bgwhite + global esc, term + global sclear_code, cur_format + + if is_enable: + sclear = '0' + sbold = '1' + sdim = '2' + snormal = '22' + sunderline = '4' + + fgblack = '30;90' + fgred = '31;91' + fggreen = '32;92' + fgyellow = '33;93' + fgblue = '34;94' + fgmagenta = '35;95' + fgcyan = '36;96' + fgwhite = '37;97' + + bgblack = '40;100' + bgred = '41;101' + bggreen = '42;102' + bgyellow = '43;103' + bgblue = '44;104' + bgmagenta = '45;105' + bgcyan = '46;106' + bgwhite = '47;107' + + esc = '\033[' + term = 'm' + + sclear_code = esc + sclear + term + cur_format = set() + + def _add_it(decoration): + if decoration in cur_format: + return '' # nothing to do + else: + cur_format.add(decoration) + return esc + ';'.join(cur_format) + term + + def _remove_it(decoration): + if decoration in cur_format: + cur_format.remove(decoration) + return esc + sclear + ';' + ';'.join(cur_format) + term + else: + return '' # nothing to do + + def _reset(): + cur_format.clear() + return esc + sclear + term + + else: + sclear = '' + sbold = '' + sdim = '' + snormal = '' + sunderline = '' + + fgblack = '' + fgred = '' + fggreen = '' + fgyellow = '' + fgblue = '' + fgmagenta = '' + fgcyan = '' + fgwhite = '' + + bgblack = '' + bgred = '' + bggreen = '' + bgyellow = '' + bgblue = '' + bgmagenta = '' + bgcyan = '' + bgwhite = '' + + sclear_code = '' + cur_format = list() + + def _add_it(decoration): + if decoration in cur_format: + return '' # nothing to do + else: + cur_format.append(decoration) + return decoration + + def _remove_it(decoration): + if decoration in cur_format: + cur_format.remove(decoration) + return decoration + else: + return '' # nothing to do + + def _reset(): + cur_format.reverse() + retval = ''.join(cur_format) + del(cur_format[:]) + return retval + +# Real terminal? isatty = sys.stdout.isatty() -if isatty: - sclear = '0' - sbold = '1' - sdim = '2' - snormal = '22' - sunderline = '4' - - fgblack = '30;90' - fgred = '31;91' - fggreen = '32;92' - fgyellow = '33;93' - fgblue = '34;94' - fgmagenta = '35;95' - fgcyan = '36;96' - fgwhite = '37;97' - - bgblack = '40;100' - bgred = '41;101' - bggreen = '42;102' - bgyellow = '43;103' - bgblue = '44;104' - bgmagenta = '45;105' - bgcyan = '46;106' - bgwhite = '47;107' - - esc = '\033[' - term = 'm' - - sclear_code = esc + sclear + term - cur_format = set() - - def _add_it(decoration): - if decoration in cur_format: - return '' # nothing to do - else: - cur_format.add(decoration) - return esc + ';'.join(cur_format) + term - - def _remove_it(decoration): - if decoration in cur_format: - cur_format.remove(decoration) - return esc + sclear + ';' + ';'.join(cur_format) + term - else: - return '' # nothing to do - - def _reset(): - cur_format.clear() - return esc + sclear + term - -else: - sclear = '' - sbold = '' - sdim = '' - snormal = '' - sunderline = '' - - fgblack = '' - fgred = '' - fggreen = '' - fgyellow = '' - fgblue = '' - fgmagenta = '' - fgcyan = '' - fgwhite = '' - - bgblack = '' - bgred = '' - bggreen = '' - bgyellow = '' - bgblue = '' - bgmagenta = '' - bgcyan = '' - bgwhite = '' - - sclear_code = '' - cur_format = list() - - def _add_it(decoration): - if decoration in cur_format: - return '' # nothing to do - else: - cur_format.append(decoration) - return decoration - - def _remove_it(decoration): - if decoration in cur_format: - cur_format.remove(decoration) - return decoration - else: - return '' # nothing to do - - def _reset(): - cur_format.reverse() - retval = ''.join(cur_format) - del(cur_format[:]) - return retval + +enable_color(isatty) def bold(): return _add_it(sbold) From 3263697e15221dffd27c149e31a992164df25613 Mon Sep 17 00:00:00 2001 From: Anshu Prateek Date: Thu, 12 Mar 2015 15:50:07 +0530 Subject: [PATCH 6/7] Moving requests to urllib2 as former is not default, some changes to write_log to capture data in stream than in one shot in end --- lib/controller.py | 81 +++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/lib/controller.py b/lib/controller.py index 48c4050f..1aa1cf5d 100644 --- a/lib/controller.py +++ b/lib/controller.py @@ -14,7 +14,7 @@ from lib.controllerlib import * from lib import util -import time, os, sys, platform, shutil, requests +import time, os, sys, platform, shutil, urllib2, socket def flip_keys(orig_data): new_data = {} @@ -527,7 +527,12 @@ def do_undun(self, line): @CommandHelp('"collectinfo" is used to collect system stats on the local node.') class CollectinfoController(CommandController): def collect_local_file(self,src,dest_dir): - shutil.copy2(src, dest_dir) + try: + shutil.copy2(src, dest_dir) + except Exception,e: + print e + pass + return def collectinfo_content(self,func,parm=''): name = '' @@ -547,27 +552,17 @@ def collectinfo_content(self,func,parm=''): o = self.cluster.info(parm) else: o = capture_stdout(func,parm) - return sep+str(o) + self.write_log(sep+str(o)) + return '' - def write_log(self,collectedinfo,src_file='/var/log/aerospike/*log'): - aslogdir = '/tmp/as_log_' + str(time.time()) - aslogfile = aslogdir + '/ascollectinfo.log' - os.mkdir(aslogdir) - remove_color = re.compile(r'\x1B\[[^A-Za-z]*[A-Za-z]') - collectedinfo = remove_color.sub('', collectedinfo) - f = open(str(aslogfile), 'w') + def write_log(self,collectedinfo): + global aslogfile + f = open(str(aslogfile), 'a') f.write(str(collectedinfo)) - f.close() - self.collect_local_file(src_file,aslogdir) - util.shell_command(["tar -czvf " + aslogdir + ".tgz " + aslogdir]) - sys.stderr.write("\x1b[2J\x1b[H") - print "\n\n\nFiles in " + aslogdir + " and " + aslogdir + ".tgz saved. Please send tgz archive to support@aerospike.com" - print "END OF ASCOLLECTINFO" - + return f.close() def get_metadata(self,response_str,prefix=''): aws_c = '' - aws_timeout = 1 aws_metadata_base_url = 'http://169.254.169.254/latest/meta-data' prefix_o = prefix if prefix_o == '/': @@ -581,20 +576,25 @@ def get_metadata(self,response_str,prefix=''): self.get_metadata(rsp_p,prefix) else: meta_url = aws_metadata_base_url+prefix+'/'+rsp - r = requests.get(meta_url,timeout=aws_timeout) - if r.status_code != 404: - aws_c += rsp +'\n'+r.text +"\n" + req = urllib2.Request(meta_url) + r = urllib2.urlopen(req) +# r = requests.get(meta_url,timeout=aws_timeout) + if r.code != 404: + aws_c += rsp +'\n'+r.read() +"\n" return aws_c def get_awsdata(self,line): aws_rsp = '' aws_timeout = 1 + socket.setdefaulttimeout(aws_timeout) aws_metadata_base_url = 'http://169.254.169.254/latest/meta-data' try: - r = requests.get(aws_metadata_base_url,timeout=aws_timeout) - if r.status_code == 200: + req = urllib2.Request(aws_metadata_base_url) + r = urllib2.urlopen(req) +# r = requests.get(aws_metadata_base_url,timeout=aws_timeout) + if r.code == 200: print "This is in AWS" - rsp = r.text + rsp = r.read() aws_rsp += self.get_metadata(rsp,'/') print aws_rsp else: @@ -618,6 +618,12 @@ def collect_sys(self,line): smd_fp = open(smd_f, 'r') print smd_fp.read() smd_fp.close() + + def archive_log(self,logdir): + util.shell_command(["tar -czvf " + logdir + ".tgz " + aslogdir]) + sys.stderr.write("\x1b[2J\x1b[H") + print "\n\n\nFiles in " + logdir + " and " + logdir + ".tgz saved. " + print "END OF ASCOLLECTINFO" def main_collectinfo(self, line): collect_output = time.strftime("%Y-%m-%d %H:%M:%S UTC\n", time.gmtime()) @@ -664,32 +670,39 @@ def main_collectinfo(self, line): 'cat /etc/aerospike/aerospike.conf', 'cat /etc/citrusleaf/citrusleaf.conf', ] - + terminal.enable_color(False) + global aslogdir,aslogfile + aslogdir = '/tmp/as_log_' + str(time.time()) + aslogfile = aslogdir + '/ascollectinfo.log' + os.mkdir(aslogdir) + self.write_log(collect_output) + log_location = '/var/log/aerospike/*.log' try: log_location = '/var/log/aerospike/aerospike.log' cinfo = InfoController() for info_param in info_params: - collect_output += self.collectinfo_content(cinfo,[info_param]) + self.collectinfo_content(cinfo,[info_param]) do_show = ShowController() for show_param in show_params: - collect_output += self.collectinfo_content(do_show,[show_param]) + self.collectinfo_content(do_show,[show_param]) for cluster_param in cluster_params: - collect_output += self.collectinfo_content('cluster',cluster_param) + self.collectinfo_content('cluster',cluster_param) # Below is not optimum, we should query only localhost logs = self.cluster.info('logs') for i in logs: log_location = logs[i].split(':')[1] cmd = 'tail -n 10000 ' + log_location if log_location != '/var/log/aerospike/aerospike.log': - collect_output += self.collectinfo_content('shell',[cmd]) + self.collectinfo_content('shell',[cmd]) except Exception as e: - collect_output += str(e) + self.write_log(str(e)) sys.stdout = sys.__stdout__ for cmd in shell_cmds: - collect_output += self.collectinfo_content('shell',[cmd]) - collect_output += self.collectinfo_content(self.collect_sys) - collect_output += self.collectinfo_content(self.get_awsdata) - self.write_log(collect_output,log_location) + self.collectinfo_content('shell',[cmd]) + self.collectinfo_content(self.collect_sys) + self.collectinfo_content(self.get_awsdata) + self.collect_local_file(log_location,aslogdir) + self.archive_log(aslogdir) def _do_default(self, line): self.main_collectinfo(line) From 0aa7f383ae9db4389e846ebaff26ab1b4e0dab19 Mon Sep 17 00:00:00 2001 From: Anshu Prateek Date: Tue, 17 Mar 2015 15:46:35 +0530 Subject: [PATCH 7/7] adding support for multiple log files --- lib/controller.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/controller.py b/lib/controller.py index 1aa1cf5d..518f5c6f 100644 --- a/lib/controller.py +++ b/lib/controller.py @@ -690,10 +690,14 @@ def main_collectinfo(self, line): # Below is not optimum, we should query only localhost logs = self.cluster.info('logs') for i in logs: - log_location = logs[i].split(':')[1] - cmd = 'tail -n 10000 ' + log_location - if log_location != '/var/log/aerospike/aerospike.log': - self.collectinfo_content('shell',[cmd]) + logs_c = logs[i].split(';') + for log in logs_c: + log_location = log.split(':')[1] + cmd = 'tail -n 10000 ' + log_location + if log_location != '/var/log/aerospike/aerospike.log': + self.collectinfo_content('shell',[cmd]) + self.collect_local_file(log_location,aslogdir) + except Exception as e: self.write_log(str(e)) sys.stdout = sys.__stdout__ @@ -701,7 +705,6 @@ def main_collectinfo(self, line): self.collectinfo_content('shell',[cmd]) self.collectinfo_content(self.collect_sys) self.collectinfo_content(self.get_awsdata) - self.collect_local_file(log_location,aslogdir) self.archive_log(aslogdir) def _do_default(self, line):