From 094c1515bd53b9877a91e38c384aa1626b031fea Mon Sep 17 00:00:00 2001 From: Ryan SVIHLA Date: Mon, 26 Apr 2021 08:28:25 +0200 Subject: [PATCH] Fix support ref (#61) * support jira ref was incorrect * added support reference link to mmap issues * updated black dependency --- CHANGELOG.txt | 4 ++++ pysper/__init__.py | 2 +- pysper/bgrep.py | 2 +- pysper/commands/core/bgrep.py | 2 +- pysper/commands/core/gc.py | 2 +- pysper/commands/sysbottle.py | 2 +- pysper/core/gcinspector.py | 12 ++++++------ pysper/core/slowquery.py | 12 ++++++------ pysper/core/statuslogger.py | 30 +++++++++++++++--------------- pysper/jarcheck.py | 4 ++-- pysper/parser/__init__.py | 2 +- pysper/parser/cases.py | 16 ++++++++-------- pysper/parser/outputlog.py | 2 +- pysper/parser/rules.py | 8 ++++---- pysper/sperf_default.py | 10 +++++++++- pysper/ttop.py | 12 ++++++------ pysper/util.py | 10 +++++----- requirements_dev.txt | 4 +++- setup.py | 2 +- tests/core/test_gc.py | 2 +- tests/test_recs.py | 2 +- tests/test_sysbottle.py | 2 +- tests/test_util.py | 2 +- 23 files changed, 80 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index cafb15b..ab1e72b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,7 @@ +sperf 0.6.8 +----------- +* sperf now links to public support topics for mmap recommendation + sperf 0.6.7 ----------- * ttop can not handle gb rate allocations diff --git a/pysper/__init__.py b/pysper/__init__.py index 7d8383f..39d2d94 100644 --- a/pysper/__init__.py +++ b/pysper/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. """top level module for sperf python port""" -VERSION = "0.6.7" +VERSION = "0.6.8" diff --git a/pysper/bgrep.py b/pysper/bgrep.py index 041ab5e..1724e91 100644 --- a/pysper/bgrep.py +++ b/pysper/bgrep.py @@ -109,7 +109,7 @@ def __setdates(self, dt): self.start = dt def print_report(self, interval=3600): - """ print bucketized result counts """ + """print bucketized result counts""" print("bucketgrep version %s" % VERSION) print("search: '%s'" % self.supplied_regex) print() diff --git a/pysper/commands/core/bgrep.py b/pysper/commands/core/bgrep.py index bd93af8..b23e355 100644 --- a/pysper/commands/core/bgrep.py +++ b/pysper/commands/core/bgrep.py @@ -65,7 +65,7 @@ def build(subparsers): def run(args): - """ run bgrep """ + """run bgrep""" files = None if args.files: files = args.files.split(",") diff --git a/pysper/commands/core/gc.py b/pysper/commands/core/gc.py index 6ef69ae..44801ea 100644 --- a/pysper/commands/core/gc.py +++ b/pysper/commands/core/gc.py @@ -79,7 +79,7 @@ def build(subparsers): def run(args): - """ run the gcinspector """ + """run the gcinspector""" files = None if args.files: files = args.files.split(",") diff --git a/pysper/commands/sysbottle.py b/pysper/commands/sysbottle.py index 93cc084..25987a4 100644 --- a/pysper/commands/sysbottle.py +++ b/pysper/commands/sysbottle.py @@ -78,7 +78,7 @@ def build(subparsers): def run(args): - """"sysbottle subcommand""" + """ "sysbottle subcommand""" conf = OrderedDict() conf["iowait_threshold"] = args.iowait conf["cpu_threshold"] = args.cpu diff --git a/pysper/core/gcinspector.py b/pysper/core/gcinspector.py index b6d4626..f8f537f 100644 --- a/pysper/core/gcinspector.py +++ b/pysper/core/gcinspector.py @@ -26,7 +26,7 @@ class GCInspector: - """ GCInspector class """ + """GCInspector class""" def __init__(self, diag_dir=None, files=None, start=None, end=None): self.diag_dir = diag_dir @@ -46,7 +46,7 @@ def __init__(self, diag_dir=None, files=None, start=None, end=None): self.end_time = date_parse(end) def analyze(self): - """ analyze files """ + """analyze files""" target = None if self.files: target = self.files @@ -69,7 +69,7 @@ def analyze(self): self.analyzed = True def __setdates(self, date, node): - """ track start/end times """ + """track start/end times""" # global if not self.start: self.start = date @@ -89,7 +89,7 @@ def __setdates(self, date, node): self.starts[node] = date def all_pauses(self): - """ get pauses for all nodes """ + """get pauses for all nodes""" pauses = OrderedDefaultDict(list) for pausedata in self.pauses.values(): for time, pause in pausedata.items(): @@ -97,7 +97,7 @@ def all_pauses(self): return pauses def print_report(self, interval=3600, by_node=False, top=3): - """ print gc report """ + """print gc report""" print("gcinspector version %s" % VERSION) print("") if not self.analyzed: @@ -151,7 +151,7 @@ def print_report(self, interval=3600, by_node=False, top=3): print("") def __print_gc(self, data): - """ print data to the user, expecting datetime keys and list(int) values """ + """print data to the user, expecting datetime keys and list(int) values""" print(". <300ms + 301-500ms ! >500ms") print("-" * 30) busiest = None diff --git a/pysper/core/slowquery.py b/pysper/core/slowquery.py index 53af4b5..8aa3335 100644 --- a/pysper/core/slowquery.py +++ b/pysper/core/slowquery.py @@ -24,7 +24,7 @@ class SlowQueryParser: - """ parses logs for slow queries """ + """parses logs for slow queries""" BEGIN = "begin" begin_match = re.compile( @@ -47,7 +47,7 @@ def __init__(self): self.state = None def parse(self, logfile): - """ parses a debug log for slow queries """ + """parses a debug log for slow queries""" ret = OrderedDict() for line in logfile: if self.state is None: @@ -78,7 +78,7 @@ def parse(self, logfile): class SlowQueryAnalyzer: - """ analyzes results from parsing slow queries """ + """analyzes results from parsing slow queries""" def __init__(self, diag_dir, files=None, start=None, end=None): self.diag_dir = diag_dir @@ -98,7 +98,7 @@ def __init__(self, diag_dir, files=None, start=None, end=None): self.end_time = date_parse(end) def analyze(self): - """ analyze slow queries """ + """analyze slow queries""" parser = SlowQueryParser() target = find_logs(self.diag_dir, "debug.log") if self.files: @@ -128,7 +128,7 @@ def analyze(self): self.analyzed = True def print_report(self, command_name, interval=3600, top=3): - """ print the report """ + """print the report""" if not self.analyzed: self.analyze() print("%s version: %s" % (command_name, VERSION)) @@ -160,7 +160,7 @@ def print_report(self, command_name, interval=3600, top=3): print("") def __print_query_times(self, data): - """ print data to the user, expecting datetime keys and list(int) values """ + """print data to the user, expecting datetime keys and list(int) values""" timings = perc.Stats([q[1] for q in self.queries]) window = timings.percentile(25) - 1 window2 = timings.percentile(50) - 1 diff --git a/pysper/core/statuslogger.py b/pysper/core/statuslogger.py index 55f034b..687c308 100644 --- a/pysper/core/statuslogger.py +++ b/pysper/core/statuslogger.py @@ -31,7 +31,7 @@ class Table: - """ represents a dse table """ + """represents a dse table""" def __init__(self, ops=0, data=0): self.ops = ops @@ -42,7 +42,7 @@ def __repr__(self): class Node: - """ represents a cassandra/dse node """ + """represents a cassandra/dse node""" def __init__(self): self.start = None @@ -57,7 +57,7 @@ def __init__(self): self.dumps_analyzed = 0 def get_busiest_tables(self, by_prop): - """ return busiest tables by_prop (data or ops) """ + """return busiest tables by_prop (data or ops)""" return sorted( self.tables.items(), key=lambda table: getattr(table[1], by_prop), @@ -65,7 +65,7 @@ def get_busiest_tables(self, by_prop): ) def longest_tp_name_length(self): - """ find the length of the thread pool with the longest name """ + """find the length of the thread pool with the longest name""" longest = 0 for stage in self.stages.values(): slen = len(max(stage, key=len)) @@ -74,7 +74,7 @@ def longest_tp_name_length(self): return longest def longest_tp_value_length(self): - """ find the length of the longest value in any threadpool """ + """find the length of the longest value in any threadpool""" longest = 0 for stage in self.stages.values(): for vals in stage.values(): @@ -84,12 +84,12 @@ def longest_tp_value_length(self): return longest def duration(self): - """ duration this node was analyzed """ + """duration this node was analyzed""" return self.end - self.start class Summary: - """ summarizes a group of nodes """ + """summarizes a group of nodes""" def __init__(self, nodes): if env.DEBUG: @@ -109,7 +109,7 @@ def __init__(self, nodes): ] def get_busiest_tables(self, by_op): - """ get busiest tables by_op """ + """get busiest tables by_op""" busiest = [] for name, node in self.nodes.items(): table = next(iter(node.get_busiest_tables(by_op)), None) @@ -120,7 +120,7 @@ def get_busiest_tables(self, by_op): ) def get_busiest_stages(self): - """ get all stages sorted by highest value """ + """get all stages sorted by highest value""" allstages = [] for name, node in self.nodes.items(): for status, stage in node.stages.items(): @@ -131,7 +131,7 @@ def get_busiest_stages(self): return sorted(allstages, key=lambda x: x[3], reverse=True) def get_stages_in(self, status): - """ return all stages in a given status """ + """return all stages in a given status""" ret = OrderedDict() for name, node in self.nodes.items(): for stage in node.stages: @@ -140,7 +140,7 @@ def get_stages_in(self, status): return ret def get_pauses(self): - """ get all gc pauses """ + """get all gc pauses""" pauses = [] for node in self.nodes.values(): pauses += node.pauses @@ -148,7 +148,7 @@ def get_pauses(self): class StatusLogger: - """ status logger """ + """status logger""" def __init__( self, @@ -181,7 +181,7 @@ def __init__( self.end = date_parse(end) def analyze(self): - """ analyze log files """ + """analyze log files""" if self.analyzed: return event_filter = UniqEventPerNodeFilter() @@ -282,7 +282,7 @@ def __setdates(self, node, date): node.start = date def print_histogram(self): - """ print histogram report, analyzing if necessary """ + """print histogram report, analyzing if necessary""" self.analyze() print("%s version: %s" % (self.command_name, VERSION)) print("") @@ -391,7 +391,7 @@ def __print_recs(self): print("* %s (%s)" % (rec, reason)) def print_summary(self): - """ prints a summary report """ + """prints a summary report""" self.analyze() summary = Summary(self.nodes) print("%s version: %s" % (self.command_name, VERSION)) diff --git a/pysper/jarcheck.py b/pysper/jarcheck.py index 9638fa7..3b54174 100644 --- a/pysper/jarcheck.py +++ b/pysper/jarcheck.py @@ -19,7 +19,7 @@ class JarCheckParser: - """ class to parse and analyze the jars in the classpath of an outlog """ + """class to parse and analyze the jars in the classpath of an outlog""" def __init__(self, diag_dir=None, files=None): self.diag_dir = diag_dir @@ -57,7 +57,7 @@ def analyze(self): self.analyzed = True def print_report(self, diff_only=False): - """ print the report """ + """print the report""" if not self.analyzed: self.analyze() if not self.files_analyzed: diff --git a/pysper/parser/__init__.py b/pysper/parser/__init__.py index 2dc21f0..dafa718 100644 --- a/pysper/parser/__init__.py +++ b/pysper/parser/__init__.py @@ -39,7 +39,7 @@ def _default_capture(line): def read_log(lines, capture_line_func=_default_capture, **extras): - """ parses an iterable set of lines yielding events """ + """parses an iterable set of lines yielding events""" fields = None for line in lines: next_fields = capture_line_func(line) diff --git a/pysper/parser/cases.py b/pysper/parser/cases.py index 2f78248..3b0001c 100644 --- a/pysper/parser/cases.py +++ b/pysper/parser/cases.py @@ -25,7 +25,7 @@ def gc_rules(): - """ rules to capture gc """ + """rules to capture gc""" return ( case("GCInspector"), rule( @@ -106,7 +106,7 @@ def gc_rules(): def solr_rules(): - """ rules to capture solr """ + """rules to capture solr""" return ( case("SolrFilterCache"), rule( @@ -166,7 +166,7 @@ def solr_rules(): def daemon_rules(): - """ rules to capture from daemon """ + """rules to capture from daemon""" return ( case("StorageService"), rule( @@ -249,7 +249,7 @@ def daemon_rules(): def cfs_rules(): - """ rules to capture from ColumnFamilyStore """ + """rules to capture from ColumnFamilyStore""" return ( case("ColumnFamilyStore"), rule( @@ -310,7 +310,7 @@ def cfs_rules(): def memtable_rules(): - """ rules to capture from memtable/cfs """ + """rules to capture from memtable/cfs""" return ( case("Memtable", "ColumnFamilyStore"), rule( @@ -374,7 +374,7 @@ def memtable_rules(): def status_rules(): - """ rules to capture from statuslogger """ + """rules to capture from statuslogger""" return ( case("StatusLogger"), rule( @@ -528,7 +528,7 @@ def status_rules(): def config_rules(): - """ rules to capture configs """ + """rules to capture configs""" return ( case("Config"), # "Node configuration:[aggregated_request_timeout_in_ms=120000; allocate_tokens_for_keyspace=null; allocate_tokens_for_local_replication_factor=3; write_request_timeout_in_ms=2000] @@ -571,7 +571,7 @@ def config_rules(): def dd_rules(): - """ rules to capture from database descriptor """ + """rules to capture from database descriptor""" return ( case("DatabaseDescriptor"), # 6.x disk access mode diff --git a/pysper/parser/outputlog.py b/pysper/parser/outputlog.py index 52ae51b..f688620 100644 --- a/pysper/parser/outputlog.py +++ b/pysper/parser/outputlog.py @@ -26,7 +26,7 @@ def update_message(fields): - """ updates message fields """ + """updates message fields""" subfields = None if "source_file" in fields: subfields = capture_message(fields["source_file"][:-5], fields["message"]) diff --git a/pysper/parser/rules.py b/pysper/parser/rules.py index ec71d62..393d51d 100644 --- a/pysper/parser/rules.py +++ b/pysper/parser/rules.py @@ -203,7 +203,7 @@ def __init__(self, capture_message): self.capture_message = capture_message def __call__(self, fields): - """ updates message fields """ + """updates message fields""" if "source_file" not in fields or "message" not in fields: return subfields = self.capture_message(fields["source_file"][:-5], fields["message"]) @@ -212,7 +212,7 @@ def __call__(self, fields): def mkcapture(cap_rule, update_func, with_date=True): - """ build a top-level capture function """ + """build a top-level capture function""" if with_date: return rule( cap_rule, @@ -244,7 +244,7 @@ def int_with_commas(value): def nodeconfig(nodeconfig_line): - """ converts to nodeconfig """ + """converts to nodeconfig""" config = OrderedDict() tokens = nodeconfig_line.split("; ") for token in tokens: @@ -255,7 +255,7 @@ def nodeconfig(nodeconfig_line): def jvm_args(args_line): - """ converts jvm args to k/v pairs """ + """converts jvm args to k/v pairs""" args = OrderedDict() tokens = args_line.split(",") for token in tokens: diff --git a/pysper/sperf_default.py b/pysper/sperf_default.py index 7d4d178..2ec79bb 100644 --- a/pysper/sperf_default.py +++ b/pysper/sperf_default.py @@ -134,6 +134,10 @@ def _recs_on_configs(recommendations, configs): + "performance on version %s of Cassandra. (SUPPORT-753)" % cassandra_version ) + issue = ( + issue + + " https://support.datastax.com/hc/en-us/articles/360027838911" + ) if issue not in recs_by_issue: recs_by_issue[issue] = { "issue": issue, @@ -151,9 +155,13 @@ def _recs_on_configs(recommendations, configs): if dse_major_version > 5: issue = ( "Disk acess mode %s causes problems " % disk_access - + "with performance on version %s of DSE. (SUPPORT-754)" + + "with performance on version %s of DSE. (SUPPORT-752)" % dse_version ) + issue = ( + issue + + " https://support.datastax.com/hc/en-us/articles/360028294671" + ) if issue not in recs_by_issue: recs_by_issue[issue] = { "issue": issue, diff --git a/pysper/ttop.py b/pysper/ttop.py index 9d0f383..fd65be6 100644 --- a/pysper/ttop.py +++ b/pysper/ttop.py @@ -23,7 +23,7 @@ class TTopParser: - """ parses ttop output files """ + """parses ttop output files""" BEGIN = "begin" PROCESS = "process" @@ -64,7 +64,7 @@ def __init__(self, start=None, end=None): self.end = date_parse(end) def parse(self, log): - """ parse ttop output """ + """parse ttop output""" total = OrderedDict() threads = OrderedDefaultDict(dict) for line in log: @@ -143,7 +143,7 @@ def parse(self, log): continue def convert(self, rate, unit): - """ converts rate to bytes """ + """converts rate to bytes""" if unit == "gb": return int(rate) * 1024 * 1024 * 1024 if unit == "mb": @@ -154,13 +154,13 @@ def convert(self, rate, unit): class TTopAnalyzer: - """ analyzes ttop info """ + """analyzes ttop info""" def __init__(self, files): self.files = files def collate_threads(self, threads): - """ combines similar threads """ + """combines similar threads""" ret = OrderedDefaultDict(lambda: OrderedDefaultDict(float)) exprs = [] exprs.append(r":.*") @@ -176,7 +176,7 @@ def collate_threads(self, threads): return ret def print_report(self, top=None, alloc=False, collate=True, start=None, end=None): - """ analyze and report on ttop files """ + """analyze and report on ttop files""" parser = TTopParser(start=start, end=end) print("ttop version %s" % VERSION) print() diff --git a/pysper/util.py b/pysper/util.py index 0b65134..8fbac78 100644 --- a/pysper/util.py +++ b/pysper/util.py @@ -32,7 +32,7 @@ def print_percentiles( reverse=False, percentiles=(99, 75, 50, 25), ): - """ prints formatted percentiles using numpy from data in a list """ + """prints formatted percentiles using numpy from data in a list""" np_array = perc.Stats(data_list) printables = [label.ljust(indent)] if pmax: @@ -49,7 +49,7 @@ def print_percentiles( def print_percentile_headers( label="", names=("max", "p99", "p75", "p50", "p25", "min"), indent=11, width=8 ): - """ prints evenly spaced headers, appropriate for percentiles """ + """prints evenly spaced headers, appropriate for percentiles""" printables = [label.ljust(indent)] for name in names: printables.append(name.ljust(width)) @@ -65,7 +65,7 @@ def get_percentiles( reverse=False, percentiles=(99, 75, 50, 25), ): - """ gets formatted percentiles using numpy from data in a list """ + """gets formatted percentiles using numpy from data in a list""" np_array = perc.Stats(data_list) printables = [label] if pmax: @@ -80,7 +80,7 @@ def get_percentiles( def get_percentile_headers(label="", names=("max", "p99", "p75", "p50", "p25", "min")): - """ gets evenly spaced headers, appropriate for percentiles """ + """gets evenly spaced headers, appropriate for percentiles""" printables = [label] for name in names: printables.append(name) @@ -88,7 +88,7 @@ def get_percentile_headers(label="", names=("max", "p99", "p75", "p50", "p25", " def node_name(filepath): - """ guess the node name from a filepath """ + """guess the node name from a filepath""" parts = filepath.split(os.path.sep) try: return parts[parts.index("nodes") + 1] diff --git a/requirements_dev.txt b/requirements_dev.txt index 9ecba3b..17113a7 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,6 +1,6 @@ altgraph==0.17 appdirs==1.4.4 -black==20.8b1 +black==21.4b0 click==7.1.2 coverage==5.4 flake8==3.8.4 @@ -12,6 +12,8 @@ pycodestyle==2.6.0 pyflakes==2.2.0 pyinstaller==4.2 pyinstaller-hooks-contrib==2020.11 +pytest==6.2.3 +pytest-cov==2.11.1 regex==2020.11.13 toml==0.10.2 typed-ast==1.4.2 diff --git a/setup.py b/setup.py index 37b4d1d..3e30f26 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="sperf", - version="0.6.7", + version="0.6.8", description="Diagnostic utility for DSE and Cassandra", url="https://www.github.com/DataStax-Toolkit/sperf", scripts=["scripts/sperf"], diff --git a/tests/core/test_gc.py b/tests/core/test_gc.py index 9d317bf..0306ee0 100644 --- a/tests/core/test_gc.py +++ b/tests/core/test_gc.py @@ -22,7 +22,7 @@ class TestCoreGC(unittest.TestCase): """testing the core gc submodule""" def test_gcinspector(self): - """ test gcinspector analysis """ + """test gcinspector analysis""" g = GCInspector(test_dse_tarball()) g.analyze() self.assertEqual(len(g.pauses), 3) diff --git a/tests/test_recs.py b/tests/test_recs.py index 756b990..11a8b97 100644 --- a/tests/test_recs.py +++ b/tests/test_recs.py @@ -107,7 +107,7 @@ def test_tpc_backpressure(self): self.assertIsNone(reason) def test_full_memtable(self): - """verify StageAnalyzer """ + """verify StageAnalyzer""" analyzer = recs.Engine() stage = recs.Stage( name="TPC/all/WRITE_MEMTABLE_FULL", diff --git a/tests/test_sysbottle.py b/tests/test_sysbottle.py index 2b3e0c4..1d7caa6 100644 --- a/tests/test_sysbottle.py +++ b/tests/test_sysbottle.py @@ -23,7 +23,7 @@ class TestSysBottle(unittest.TestCase): """sysbottle tests""" def test_sysbottle_analyze(self): - """ test sysbottle report """ + """test sysbottle report""" iostat = os.path.join( os.path.dirname(os.path.abspath(__file__)), "testdata", "iostat" ) diff --git a/tests/test_util.py b/tests/test_util.py index 7ea5299..03466f9 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -35,7 +35,7 @@ def test_extract_node_name(self): self.assertEqual(my_node, "my_node") def test_bucketize(self): - """ test bucketize """ + """test bucketize""" junk = defaultdict(list) data = [1, 1, 1, 1, 1, 1, 1, 1] junk[datetime.datetime(2019, 5, 11, 0, 0, 0, 0)] = data