From 06e4a085e13d3b6ac7d2ec1f95469c2f985bef22 Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Thu, 3 Oct 2024 10:49:22 -0400 Subject: [PATCH 01/55] added predominant language gauge graph to repo pages Signed-off-by: sachin-panayil --- scripts/gen_graphs.py | 26 +++++++++++++++++++++++++- templates/repo_report_template.md | 2 ++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 0709baf86b..30fe392007 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -216,4 +216,28 @@ def generate_top_committer_bar_graph(oss_entity): bar_chart.add(committer, commits) contributor_count += 1 - write_repo_chart_to_file(oss_entity, bar_chart, "top_committers") \ No newline at end of file + write_repo_chart_to_file(oss_entity, bar_chart, "top_committers") + +def generate_predominant_languages_graph(oss_entity): + """ + This function generates a pygal predominant programming languages guage graph. + + Arguments: + oss_entity: the OSSEntity to create a graph for. + """ + + guage_graph = pygal.Gauge(dynamic_print_values=True, human_readable = True) + graph.title = f"Predominant Languages in {oss_entity.metric_data['name']}" + + predominant_lang = oss_entity.metric_data['predominant_langs'] + max_amount = 0 + + for lang, lines in predominant_lang: + if lines > max_amount: + max_amount = lines + + graph.add(lang, lines) + + guage_graph.range = [round(max_amount + 2000), 0] + + write_repo_chart_to_file(oss_entity, graph, "predominant_langs") diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 13e928227d..5ebd2119a7 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -111,4 +111,6 @@ date_stampLastWeek: {date_stamp} {{% assign optionsArray = '1 Month, 6 Month' | split: ',' %}} {{% assign graphsArray = '/{repo_owner}/{repo_name}/new_commit_contributors_by_day_over_last_month_{repo_name}_data.svg, /{repo_owner}/{repo_name}/new_commit_contributors_by_day_over_last_six_months_{repo_name}_data.svg' | split: ',' %}} {{% render "graph-toggle", baseurl: site.baseurl, name: "new-contributors" options: optionsArray, graphs: graphsArray, title: "Number of Contributors Joining per Interval" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/predominant_langs_{repo_name}_data.svg", title: "Predominant Languages" %}} \ No newline at end of file From bd3cf406d82e50f8f34ac257c42eefa5c0edb5eb Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Thu, 3 Oct 2024 10:14:12 -0500 Subject: [PATCH 02/55] add org libyears Signed-off-by: Isaac Milarsky --- scripts/metricsLib/metrics_data_structures.py | 3 +++ scripts/metricsLib/metrics_definitions.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/scripts/metricsLib/metrics_data_structures.py b/scripts/metricsLib/metrics_data_structures.py index a607034da7..1e71fa592b 100644 --- a/scripts/metricsLib/metrics_data_structures.py +++ b/scripts/metricsLib/metrics_data_structures.py @@ -336,6 +336,9 @@ def get_values(self, params=None): # EX: storing the date and count of each time the amount of followers # increased. try: + #Only continue if the api_label is a list + if type(api_label) != list: + raise TypeError list(api_label) # initialize each label as an empty list diff --git a/scripts/metricsLib/metrics_definitions.py b/scripts/metricsLib/metrics_definitions.py index 2fac3ea936..298b1a33f3 100644 --- a/scripts/metricsLib/metrics_definitions.py +++ b/scripts/metricsLib/metrics_definitions.py @@ -141,6 +141,14 @@ "/repo-groups/{repo_group_id}/top-committers", {"top_committers": ["email", "commits"]})) +ORG_METRICS.append(ListMetric("orgLibyears", ["repo_group_id"], + AUGUR_HOST + + "/repo-groups/{repo_group_id}/libyear", + {"dependency_libyear_list": [ + "repo_name", "name","libyear","most_recent_collection" + ] + })) + CONTRIBS_LABEL_LAST_MONTH = "new_commit_contributors_by_day_over_last_month" PERIODIC_METRICS.append(ListMetric("newContributorsofCommitsWeekly", From 3657ae10d0f22ea1e02b0d81c32a98ab7bde22f9 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Thu, 3 Oct 2024 10:23:02 -0500 Subject: [PATCH 03/55] add libyear api call for repos Signed-off-by: Isaac Milarsky --- scripts/metricsLib/metrics_definitions.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/metricsLib/metrics_definitions.py b/scripts/metricsLib/metrics_definitions.py index 298b1a33f3..b6151ecc0f 100644 --- a/scripts/metricsLib/metrics_definitions.py +++ b/scripts/metricsLib/metrics_definitions.py @@ -239,3 +239,16 @@ NADIA_ENDPOINT = AUGUR_HOST + "/repos/{repo_id}/nadia-project-labeling-badge/" ADVANCED_METRICS.append(CustomMetric("getNadiaBadgeURL",[ "repo_id"],NADIA_ENDPOINT, parse_nadia_label_into_badge)) + +REPO_LIBYEAR_ENDPOINT = AUGUR_HOST + "/repo-groups/{repo_group_id}/repos/{repo_id}/libyear" +ADVANCED_METRICS.append(ListMetric( + "repoLibyears", + ["repo_group_id","repo_id"], + REPO_LIBYEAR_ENDPOINT, + { + "repo_dependency_libyear_list" : [ + "name","libyear","most_recent_collection" + ] + } + ) +) From 44cf849ab9aee1865e927e136447e526afa521cf Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Thu, 3 Oct 2024 15:31:48 -0400 Subject: [PATCH 04/55] changes variable names and added to main generation func Signed-off-by: sachin-panayil --- scripts/gen_graphs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 30fe392007..19a9aa4545 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -16,6 +16,7 @@ def generate_all_graphs_for_repos(all_repos): print(f"Generating graphs for repo {repo.name}") generate_solid_gauge_issue_graph(repo) generate_repo_sparklines(repo) + generate_predominant_languages_graph(repo) try: generate_donut_graph_line_complexity_graph(repo) generate_time_xy_issue_graph(repo, "new_commit_contributors_by_day_over_last_month", "New Contributors") @@ -39,7 +40,6 @@ def generate_all_graphs_for_orgs(all_orgs): generate_time_xy_issue_graph(org, "new_issues_by_day_over_last_month", "New Issues") generate_top_committer_bar_graph(org) - def write_repo_chart_to_file(repo, chart, chart_name, custom_func=None, custom_func_params={}): """ This function's purpose is to save a pygals chart to a path derived from the @@ -227,7 +227,7 @@ def generate_predominant_languages_graph(oss_entity): """ guage_graph = pygal.Gauge(dynamic_print_values=True, human_readable = True) - graph.title = f"Predominant Languages in {oss_entity.metric_data['name']}" + guage_graph.title = f"Predominant Languages in {oss_entity.metric_data['name']}" predominant_lang = oss_entity.metric_data['predominant_langs'] max_amount = 0 @@ -236,8 +236,8 @@ def generate_predominant_languages_graph(oss_entity): if lines > max_amount: max_amount = lines - graph.add(lang, lines) + guage_graph.add(lang, lines) guage_graph.range = [round(max_amount + 2000), 0] - write_repo_chart_to_file(oss_entity, graph, "predominant_langs") + write_repo_chart_to_file(oss_entity, guage_graph, "predominant_langs") From ebc4e9b945ab52d16d428b942f910516ea30de93 Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Fri, 4 Oct 2024 15:31:11 -0400 Subject: [PATCH 05/55] changed graph to bar graph Signed-off-by: sachin-panayil --- scripts/gen_graphs.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 19a9aa4545..a4e15a24d9 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -226,18 +226,12 @@ def generate_predominant_languages_graph(oss_entity): oss_entity: the OSSEntity to create a graph for. """ - guage_graph = pygal.Gauge(dynamic_print_values=True, human_readable = True) - guage_graph.title = f"Predominant Languages in {oss_entity.metric_data['name']}" + bar_chart = pygal.Bar() + bar_chart.title = f"Predominant Languages in {oss_entity.metric_data['name']}" predominant_lang = oss_entity.metric_data['predominant_langs'] - max_amount = 0 - - for lang, lines in predominant_lang: - if lines > max_amount: - max_amount = lines - - guage_graph.add(lang, lines) - - guage_graph.range = [round(max_amount + 2000), 0] + + for lang, lines in predominant_lang.items(): + bar_chart.add(lang, lines) - write_repo_chart_to_file(oss_entity, guage_graph, "predominant_langs") + write_repo_chart_to_file(oss_entity, bar_chart, "predominant_langs") From 83149bf727bfefa8b85691929981237e567f1207 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Fri, 4 Oct 2024 14:43:30 -0500 Subject: [PATCH 06/55] increase timeout Signed-off-by: Isaac Milarsky --- scripts/metricsLib/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/metricsLib/constants.py b/scripts/metricsLib/constants.py index 752f7a6016..32430d8012 100644 --- a/scripts/metricsLib/constants.py +++ b/scripts/metricsLib/constants.py @@ -6,7 +6,7 @@ from pathlib import Path from enum import Enum -TIMEOUT_IN_SECONDS = 20 +TIMEOUT_IN_SECONDS = 120 BASE_PATH = os.path.dirname(os.path.abspath(__file__)) # Folder Names to send over our projects tracked data PATH_TO_METRICS_DATA = (Path(__file__).parent / From 6352e49d704c216f82f8726ccffb311ab507c0ef Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Oct 2024 10:29:09 -0500 Subject: [PATCH 07/55] add dryness parsing function Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index a4e15a24d9..0b545854a0 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -3,6 +3,7 @@ """ import datetime import pygal +import re def generate_all_graphs_for_repos(all_repos): """ @@ -235,3 +236,43 @@ def generate_predominant_languages_graph(oss_entity): bar_chart.add(lang, lines) write_repo_chart_to_file(oss_entity, bar_chart, "predominant_langs") + +def parse_cocomo_dryness_metrics(dryness_string): + """ + This function parses the output of the scc dryness metrics. + + For some reason, ULOC, SLOC, and DRYness don't show up in the json and + only show up in the stdout text. + + Arguments: + dryness_string: the string containing the dryness table to parse + + Returns: + A dictionary with the unique lines of code and DRYness percentage + """ + + dryness_metrics = {} + + for line in dryness_string.split('\n'): + if 'Unique Lines of Code' in line: + dryness_metrics['total_uloc'] = re.sub('[^0-9.]','',line) + if 'DRYness' in line: + dryness_metrics['DRYness_percentage'] = re.sub('[^0-9.]','',line) + + #sloc = uloc / DRYness + return dryness_metrics + + + + +def generate_dryness_percentage_graph(oss_entity): + """ + This function generates a pygal DRYness pie graph. + + DRYness = ULOC / SLOC + + WETness = 1 - DRYness + + DRY = Don't repeat yourself + WET = Waste Everybody's time or Write Everything Twice + """ \ No newline at end of file From e0ff56c65b6d2e0b4e6817a54b11f712e5b709f5 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Oct 2024 10:40:15 -0500 Subject: [PATCH 08/55] add dryness percentage graph and add it to repo_report_template file. also fixed a typo Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 22 +++++++++++++++++++++- templates/repo_report_template.md | 4 +++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 0b545854a0..d824f67cf5 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -25,6 +25,7 @@ def generate_all_graphs_for_repos(all_repos): except KeyError as e: print(f"Could not find metrics to build graphs for repo {repo.name}") print(e) + generate_dryness_percentage_graph(repo) def generate_all_graphs_for_orgs(all_orgs): @@ -275,4 +276,23 @@ def generate_dryness_percentage_graph(oss_entity): DRY = Don't repeat yourself WET = Waste Everybody's time or Write Everything Twice - """ \ No newline at end of file + """ + + dryness_values = parse_cocomo_dryness_metrics( + oss_entity.metric_data["cocomo"]['dryness_table'] + ) + + + pie_chart = pygal.Pie(half_pie=True) + pie_chart.title = 'DRYness Percentage Graph' + + pie_chart.add( + 'Total Unique Lines of Code (ULOC)', dryness_values['total_uloc'] + ) + + pie_chart.add( + 'Total Source Lines of Code (SLOC)', + dryness_values['total_uloc'] / dryness_values['DRYness_percentage'] + ) + + write_repo_chart_to_file(oss_entity, pie_chart, "DRYness") \ No newline at end of file diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 5ebd2119a7..d93d2c7eaa 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -111,6 +111,8 @@ date_stampLastWeek: {date_stamp} {{% assign optionsArray = '1 Month, 6 Month' | split: ',' %}} {{% assign graphsArray = '/{repo_owner}/{repo_name}/new_commit_contributors_by_day_over_last_month_{repo_name}_data.svg, /{repo_owner}/{repo_name}/new_commit_contributors_by_day_over_last_six_months_{repo_name}_data.svg' | split: ',' %}} {{% render "graph-toggle", baseurl: site.baseurl, name: "new-contributors" options: optionsArray, graphs: graphsArray, title: "Number of Contributors Joining per Interval" %}} - + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/predominant_langs_{repo_name}_data.svg", title: "Predominant Languages" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/DRYness_{repo_name}_data.svg", title: "DRYness Percentage Graph" %}} \ No newline at end of file From 879c02cdb24629867b23b8509d9b7ade2b0647ee Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Oct 2024 11:34:42 -0500 Subject: [PATCH 09/55] fix dryness graph Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index d824f67cf5..c8128cae65 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -25,7 +25,15 @@ def generate_all_graphs_for_repos(all_repos): except KeyError as e: print(f"Could not find metrics to build graphs for repo {repo.name}") print(e) - generate_dryness_percentage_graph(repo) + + try: + generate_dryness_percentage_graph(repo) + except ValueError as e: + print("Could not parse DRYness due to percentage values being invalid!") + print(e) + except KeyError as e: + print(f"Could not find metrics to build dryness graphs for repo {repo.name}") + print(e) def generate_all_graphs_for_orgs(all_orgs): @@ -254,13 +262,16 @@ def parse_cocomo_dryness_metrics(dryness_string): dryness_metrics = {} + #Parse output line by line for line in dryness_string.split('\n'): + #Parse the parts that we want into fields if 'Unique Lines of Code' in line: + #Use regex to remove all non-numerals from the string dryness_metrics['total_uloc'] = re.sub('[^0-9.]','',line) if 'DRYness' in line: + #Use regex to remove all non-numerals from the string dryness_metrics['DRYness_percentage'] = re.sub('[^0-9.]','',line) - #sloc = uloc / DRYness return dryness_metrics @@ -286,13 +297,17 @@ def generate_dryness_percentage_graph(oss_entity): pie_chart = pygal.Pie(half_pie=True) pie_chart.title = 'DRYness Percentage Graph' + #print(dryness_values) + pie_chart.add( - 'Total Unique Lines of Code (ULOC)', dryness_values['total_uloc'] + 'Total Unique Lines of Code (ULOC)', float(dryness_values['total_uloc']) ) + #Will cause a value error if the dryness value is NaN which can happen. pie_chart.add( 'Total Source Lines of Code (SLOC)', - dryness_values['total_uloc'] / dryness_values['DRYness_percentage'] + #sloc = uloc / DRYness + float(dryness_values['total_uloc']) / float(dryness_values['DRYness_percentage']) ) write_repo_chart_to_file(oss_entity, pie_chart, "DRYness") \ No newline at end of file From 89643f2aaf29610c3855d9c004bc4771753fb780 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Oct 2024 12:12:28 -0500 Subject: [PATCH 10/55] fix percentage values and make graph use percentages instead of raw values Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index c8128cae65..2725f8e477 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -293,21 +293,26 @@ def generate_dryness_percentage_graph(oss_entity): oss_entity.metric_data["cocomo"]['dryness_table'] ) + sloc = (float(dryness_values['total_uloc']) / float(dryness_values['DRYness_percentage'])) + sloc_diff = sloc - float(dryness_values['total_uloc']) + sloc_percent = (sloc_diff / sloc) * 100 - pie_chart = pygal.Pie(half_pie=True) + uloc_percent = (float(dryness_values['total_uloc']) / sloc) * 100 + + pie_chart = pygal.Pie(half_pie=True, legend_at_bottom=True) pie_chart.title = 'DRYness Percentage Graph' #print(dryness_values) pie_chart.add( - 'Total Unique Lines of Code (ULOC)', float(dryness_values['total_uloc']) + 'Unique Lines of Code (ULOC) %', uloc_percent ) #Will cause a value error if the dryness value is NaN which can happen. pie_chart.add( - 'Total Source Lines of Code (SLOC)', + 'Source Lines of Code (SLOC) %', #sloc = uloc / DRYness - float(dryness_values['total_uloc']) / float(dryness_values['DRYness_percentage']) + sloc_percent ) write_repo_chart_to_file(oss_entity, pie_chart, "DRYness") \ No newline at end of file From c7c5ff94149fa58469ae6a241fa9e4fb31e43228 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Oct 2024 16:17:13 -0500 Subject: [PATCH 11/55] linting Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 2725f8e477..86fcfcb926 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -2,8 +2,8 @@ Module to define methods to create pygals graphs """ import datetime -import pygal import re +import pygal def generate_all_graphs_for_repos(all_repos): """ @@ -20,12 +20,16 @@ def generate_all_graphs_for_repos(all_repos): generate_predominant_languages_graph(repo) try: generate_donut_graph_line_complexity_graph(repo) - generate_time_xy_issue_graph(repo, "new_commit_contributors_by_day_over_last_month", "New Contributors") - generate_time_xy_issue_graph(repo, "new_commit_contributors_by_day_over_last_six_months", "New Contributors") + generate_time_xy_issue_graph( + repo, "new_commit_contributors_by_day_over_last_month", "New Contributors" + ) + generate_time_xy_issue_graph( + repo, "new_commit_contributors_by_day_over_last_six_months", "New Contributors" + ) except KeyError as e: print(f"Could not find metrics to build graphs for repo {repo.name}") print(e) - + try: generate_dryness_percentage_graph(repo) except ValueError as e: @@ -210,7 +214,7 @@ def generate_top_committer_bar_graph(oss_entity): Arguments: oss_entity: the OSSEntity to create a graph for. """ - + # Create a bar chart object bar_chart = pygal.Bar() bar_chart.title = f"Top Committers in {oss_entity.metric_data['name']}" @@ -240,7 +244,7 @@ def generate_predominant_languages_graph(oss_entity): bar_chart.title = f"Predominant Languages in {oss_entity.metric_data['name']}" predominant_lang = oss_entity.metric_data['predominant_langs'] - + for lang, lines in predominant_lang.items(): bar_chart.add(lang, lines) @@ -271,7 +275,7 @@ def parse_cocomo_dryness_metrics(dryness_string): if 'DRYness' in line: #Use regex to remove all non-numerals from the string dryness_metrics['DRYness_percentage'] = re.sub('[^0-9.]','',line) - + return dryness_metrics @@ -308,11 +312,11 @@ def generate_dryness_percentage_graph(oss_entity): 'Unique Lines of Code (ULOC) %', uloc_percent ) - #Will cause a value error if the dryness value is NaN which can happen. + #Will cause a value error if the dryness value is NaN which can happen. pie_chart.add( 'Source Lines of Code (SLOC) %', #sloc = uloc / DRYness sloc_percent ) - - write_repo_chart_to_file(oss_entity, pie_chart, "DRYness") \ No newline at end of file + + write_repo_chart_to_file(oss_entity, pie_chart, "DRYness") From a0863ed0b41d8b93848993755ada07ab792870df Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 9 Oct 2024 10:48:11 -0500 Subject: [PATCH 12/55] add parsing function for libyear dependency list Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 48 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index a4e15a24d9..51cf4fe5c2 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -235,3 +235,51 @@ def generate_predominant_languages_graph(oss_entity): bar_chart.add(lang, lines) write_repo_chart_to_file(oss_entity, bar_chart, "predominant_langs") + +def parse_libyear_dates(dependency_list): + """ + Parses the dependency list returned from the libyear metric into a list of python dictionaries + that have correctly parsed dates. + + Arguments: + dependency_list: the list of lists that has the deps data + + Returns: + A list of dictionaries describing deps + """ + + to_return = [] + for dep in dependency_list: + + + date = datetime.datetime.strptime(dep[3], '%Y-%m-%dT%H:%M:%S.%f') + to_return.append( + { + "repository": dep[0], + "dep_name": dep[1], + "libyear_value": dep[2], + "libyear_date_last_updated": date + } + ) + + return to_return + + +def generate_libyears_graph(oss_entity): + """ + Generates a pygal graph to describe libyear metrics for the requested oss_entity + + Arguments: + oss_entity: the OSSEntity to create a libyears graph for. + """ + + dep_list = oss_entity.metric_data['repo_dependency_libyear_list'] + if not dep_list: + return + + line_chart = pygal.Line() + + line_chart.title = 'Dependency Libyears' + + + line_chart.x_labels = map(str, range(2002, 2013)) \ No newline at end of file From 0877027bddea36d7ef121a50be74d51fe2ae24a9 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 9 Oct 2024 10:54:04 -0500 Subject: [PATCH 13/55] set up the X axis for libyears Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 51cf4fe5c2..9d16000413 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -236,7 +236,7 @@ def generate_predominant_languages_graph(oss_entity): write_repo_chart_to_file(oss_entity, bar_chart, "predominant_langs") -def parse_libyear_dates(dependency_list): +def parse_libyear_list(dependency_list): """ Parses the dependency list returned from the libyear metric into a list of python dictionaries that have correctly parsed dates. @@ -262,7 +262,8 @@ def parse_libyear_dates(dependency_list): } ) - return to_return + #return list sorted by date + return sorted(to_return, key=lambda d : d["libyear_date_last_updated"]) def generate_libyears_graph(oss_entity): @@ -273,13 +274,16 @@ def generate_libyears_graph(oss_entity): oss_entity: the OSSEntity to create a libyears graph for. """ - dep_list = oss_entity.metric_data['repo_dependency_libyear_list'] - if not dep_list: + raw_dep_list = oss_entity.metric_data['repo_dependency_libyear_list'] + if not raw_dep_list: return line_chart = pygal.Line() line_chart.title = 'Dependency Libyears' + dep_list = parse_libyear_list(raw_dep_list) + earliest_year = dep_list[0]["libyear_date_last_updated"].year - 5 + latest_year = dep_list[-1]["libyear_date_last_updated"].year + 1 - line_chart.x_labels = map(str, range(2002, 2013)) \ No newline at end of file + line_chart.x_labels = map(str, range(earliest_year, latest_year)) \ No newline at end of file From 07f806ab2c267fa9b56e9343b35c18dfa71ee877 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 9 Oct 2024 11:17:36 -0500 Subject: [PATCH 14/55] create a timeline viz using TimeDeltaLine in pygal Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 9d16000413..586a5ef641 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -2,6 +2,7 @@ Module to define methods to create pygals graphs """ import datetime +from datetime import timedelta import pygal def generate_all_graphs_for_repos(all_repos): @@ -278,12 +279,25 @@ def generate_libyears_graph(oss_entity): if not raw_dep_list: return - line_chart = pygal.Line() + #This is going to be kind of hacky since pygals doesn't have a + #timeline object + #TODO: Contribute upstream to add a timeline object to pygal + dateline = pygal.TimeDeltaLine(x_label_rotation=25) - line_chart.title = 'Dependency Libyears' + dateline.title = 'Dependency Libyears' dep_list = parse_libyear_list(raw_dep_list) - earliest_year = dep_list[0]["libyear_date_last_updated"].year - 5 - latest_year = dep_list[-1]["libyear_date_last_updated"].year + 1 - line_chart.x_labels = map(str, range(earliest_year, latest_year)) \ No newline at end of file + #We are going to treat the y-axis as having one dep per level in the graph + elevation = 0 + for dep in dep_list: + dateline.add(dep["dep_name"], [ + (timedelta(), elevation), + (datetime.datetime.now() - dep["libyear_date_last_updated"], elevation), + ]) + + #move one line up so that we have no overlap in the timedeltas + elevation += 1 + + write_repo_chart_to_file(oss_entity, dateline, "libyear_timeline") + From 55375ecc6b7fe496da1f788a57192bcf3de8ce2d Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 9 Oct 2024 11:22:13 -0500 Subject: [PATCH 15/55] remove extra lines Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 86fcfcb926..b7c4ff04ed 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -39,7 +39,6 @@ def generate_all_graphs_for_repos(all_repos): print(f"Could not find metrics to build dryness graphs for repo {repo.name}") print(e) - def generate_all_graphs_for_orgs(all_orgs): """ Function to iterate through all orgs and generate graphs for each of them @@ -77,7 +76,6 @@ def write_repo_chart_to_file(repo, chart, chart_name, custom_func=None, custom_f f"Repo {repo.name} has a division by zero error when trying to make graph") # issues_gauge.render_to_file(repo.get_path_to_graph_data("issue_gauge")) - def generate_repo_sparklines(repo): """ This function generates pygals sparklines graphs for a set of Repository objects. @@ -128,7 +126,6 @@ def generate_time_xy_issue_graph(oss_entity,data_key,legend_key): write_repo_chart_to_file(oss_entity, xy_time_issue_chart, data_key) - def generate_donut_graph_line_complexity_graph(oss_entity): """ This function generates pygals line complexity donut graph @@ -156,7 +153,6 @@ def generate_donut_graph_line_complexity_graph(oss_entity): write_repo_chart_to_file(oss_entity, donut_lines_graph, "total_line_makeup") - def generate_solid_gauge_issue_graph(oss_entity): """ This function generates pygals solid gauge issue/pr graphs for a set of Repository objects. @@ -278,9 +274,6 @@ def parse_cocomo_dryness_metrics(dryness_string): return dryness_metrics - - - def generate_dryness_percentage_graph(oss_entity): """ This function generates a pygal DRYness pie graph. From f61ba74c2dd05b09ccaaef116fc27dc05abe444e Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Wed, 9 Oct 2024 12:03:07 -0500 Subject: [PATCH 16/55] added age of project to frontend Signed-off-by: sachin-panayil --- app/site/_layouts/repo-report.liquid | 28 ++++++++++++++++++++++++++++ app/src/js/graphs.js | 14 ++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/app/site/_layouts/repo-report.liquid b/app/site/_layouts/repo-report.liquid index 3fd984cd24..83c058fe9c 100644 --- a/app/site/_layouts/repo-report.liquid +++ b/app/site/_layouts/repo-report.liquid @@ -155,6 +155,34 @@ layout: base

{{ project.pull_requests_count }}

+
+
+

+ + + {% lucide "calendar" %} + + + Age Of Project +

+
+

+ + {{ project.created_at | date: "%B %d, %Y" }} + + +

+
{% else %}

Error Occurred: Object Not Found

diff --git a/app/src/js/graphs.js b/app/src/js/graphs.js index 94a85282c5..5c2237eed7 100644 --- a/app/src/js/graphs.js +++ b/app/src/js/graphs.js @@ -17,3 +17,17 @@ window.showGraph = function (selectedGraphId, className, buttonName) { } }) } + +// goes inside includes/js for the time being +function toggleDateAge(element) { + const dateSpan = element.querySelector('.date'); + const ageSpan = element.querySelector('.age'); + + if (ageSpan.style.display === 'none') { + dateSpan.style.display = 'none'; + ageSpan.style.display = 'inline'; + } else { + dateSpan.style.display = 'inline'; + ageSpan.style.display = 'none'; + } +} \ No newline at end of file From cf520f8b9d892ad9eb24ff599b720809e0043a88 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 9 Oct 2024 12:41:25 -0500 Subject: [PATCH 17/55] generate libyears graph for orgs and sort values properly Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 586a5ef641..bbd7932381 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -25,6 +25,11 @@ def generate_all_graphs_for_repos(all_repos): except KeyError as e: print(f"Could not find metrics to build graphs for repo {repo.name}") print(e) + + try: + generate_libyears_graph(repo) + except KeyError: + print(f"Repository {repo.name} has no deps data associated with it!") def generate_all_graphs_for_orgs(all_orgs): @@ -41,6 +46,11 @@ def generate_all_graphs_for_orgs(all_orgs): generate_time_xy_issue_graph(org, "new_issues_by_day_over_last_month", "New Issues") generate_top_committer_bar_graph(org) + try: + generate_libyears_graph(org) + except KeyError: + print(f"Org {org.name} has no deps data associated with it!") + def write_repo_chart_to_file(repo, chart, chart_name, custom_func=None, custom_func_params={}): """ This function's purpose is to save a pygals chart to a path derived from the @@ -252,19 +262,18 @@ def parse_libyear_list(dependency_list): to_return = [] for dep in dependency_list: - - date = datetime.datetime.strptime(dep[3], '%Y-%m-%dT%H:%M:%S.%f') + #print(dep) + date = datetime.datetime.strptime(dep[-1], '%Y-%m-%dT%H:%M:%S.%f') to_return.append( { - "repository": dep[0], - "dep_name": dep[1], - "libyear_value": dep[2], + "dep_name": dep[-3], + "libyear_value": dep[-2], "libyear_date_last_updated": date } ) - + #return list sorted by date - return sorted(to_return, key=lambda d : d["libyear_date_last_updated"]) + return sorted(to_return, key=lambda d : d["libyear_value"]) def generate_libyears_graph(oss_entity): @@ -275,16 +284,20 @@ def generate_libyears_graph(oss_entity): oss_entity: the OSSEntity to create a libyears graph for. """ - raw_dep_list = oss_entity.metric_data['repo_dependency_libyear_list'] + try: + raw_dep_list = oss_entity.metric_data['repo_dependency_libyear_list'] + except KeyError: + raw_dep_list = oss_entity.metric_data['dependency_libyear_list'] + if not raw_dep_list: return - + #This is going to be kind of hacky since pygals doesn't have a #timeline object #TODO: Contribute upstream to add a timeline object to pygal dateline = pygal.TimeDeltaLine(x_label_rotation=25) - dateline.title = 'Dependency Libyears' + dateline.title = 'Dependency Libyears: Age of Dependency Version in Days' dep_list = parse_libyear_list(raw_dep_list) @@ -293,7 +306,7 @@ def generate_libyears_graph(oss_entity): for dep in dep_list: dateline.add(dep["dep_name"], [ (timedelta(), elevation), - (datetime.datetime.now() - dep["libyear_date_last_updated"], elevation), + (timedelta(days=(dep["libyear_value"] * 365)), elevation), ]) #move one line up so that we have no overlap in the timedeltas From 07461aa565316fe5de22bbcb2ef240398bb7c9da Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 9 Oct 2024 14:30:10 -0500 Subject: [PATCH 18/55] Hide y labels and make legend at bottom. Add graphs to report templates Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 22 +++++++++++++--------- templates/org_report_template.md | 2 ++ templates/repo_report_template.md | 2 ++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index bbd7932381..ac89741ffa 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -25,7 +25,7 @@ def generate_all_graphs_for_repos(all_repos): except KeyError as e: print(f"Could not find metrics to build graphs for repo {repo.name}") print(e) - + try: generate_libyears_graph(repo) except KeyError: @@ -53,7 +53,7 @@ def generate_all_graphs_for_orgs(all_orgs): def write_repo_chart_to_file(repo, chart, chart_name, custom_func=None, custom_func_params={}): """ - This function's purpose is to save a pygals chart to a path derived from the + This function's purpose is to save a pygals chart to a path derived from the repository object passed in. Arguments: @@ -132,7 +132,7 @@ def generate_donut_graph_line_complexity_graph(oss_entity): for a set of Repository objects. Arguments: - oss_entity: The OSSEntity to create a graph for. an + oss_entity: The OSSEntity to create a graph for. an OSSEntity is a data structure that is typically a repository or an organization. """ @@ -211,7 +211,7 @@ def generate_top_committer_bar_graph(oss_entity): Arguments: oss_entity: the OSSEntity to create a graph for. """ - + # Create a bar chart object bar_chart = pygal.Bar() bar_chart.title = f"Top Committers in {oss_entity.metric_data['name']}" @@ -241,7 +241,7 @@ def generate_predominant_languages_graph(oss_entity): bar_chart.title = f"Predominant Languages in {oss_entity.metric_data['name']}" predominant_lang = oss_entity.metric_data['predominant_langs'] - + for lang, lines in predominant_lang.items(): bar_chart.add(lang, lines) @@ -254,7 +254,7 @@ def parse_libyear_list(dependency_list): Arguments: dependency_list: the list of lists that has the deps data - + Returns: A list of dictionaries describing deps """ @@ -292,10 +292,10 @@ def generate_libyears_graph(oss_entity): if not raw_dep_list: return - #This is going to be kind of hacky since pygals doesn't have a + #This is going to be kind of hacky since pygals doesn't have a #timeline object #TODO: Contribute upstream to add a timeline object to pygal - dateline = pygal.TimeDeltaLine(x_label_rotation=25) + dateline = pygal.TimeDeltaLine(x_label_rotation=25,legend_at_bottom=True) dateline.title = 'Dependency Libyears: Age of Dependency Version in Days' @@ -311,6 +311,10 @@ def generate_libyears_graph(oss_entity): #move one line up so that we have no overlap in the timedeltas elevation += 1 - + + if elevation >= 40: + break + + dateline.show_y_labels = False write_repo_chart_to_file(oss_entity, dateline, "libyear_timeline") diff --git a/templates/org_report_template.md b/templates/org_report_template.md index 9211d31f56..dd8af25d67 100644 --- a/templates/org_report_template.md +++ b/templates/org_report_template.md @@ -111,5 +111,7 @@ date_stampLastWeek: {date_stamp} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_owner}_new_issues_by_day_over_last_six_months.svg", title: "New Issues over Last 6 Months" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_owner}_top_committers.svg", title: "Top Committers" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_owner}_libyear_timeline.svg", title: "Top Committers" %}} \ No newline at end of file diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 5ebd2119a7..b660895796 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -113,4 +113,6 @@ date_stampLastWeek: {date_stamp} {{% render "graph-toggle", baseurl: site.baseurl, name: "new-contributors" options: optionsArray, graphs: graphsArray, title: "Number of Contributors Joining per Interval" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/predominant_langs_{repo_name}_data.svg", title: "Predominant Languages" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/libyear_timeline_{repo_name}_data.svg", title: "Dependency Libyears" %}} \ No newline at end of file From 7179b1fee84176a1152542d3ce2cedf6581b97b8 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 9 Oct 2024 14:34:04 -0500 Subject: [PATCH 19/55] lint Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index ac89741ffa..c7ca5a4704 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -20,8 +20,12 @@ def generate_all_graphs_for_repos(all_repos): generate_predominant_languages_graph(repo) try: generate_donut_graph_line_complexity_graph(repo) - generate_time_xy_issue_graph(repo, "new_commit_contributors_by_day_over_last_month", "New Contributors") - generate_time_xy_issue_graph(repo, "new_commit_contributors_by_day_over_last_six_months", "New Contributors") + generate_time_xy_issue_graph( + repo, "new_commit_contributors_by_day_over_last_month", "New Contributors" + ) + generate_time_xy_issue_graph( + repo, "new_commit_contributors_by_day_over_last_six_months", "New Contributors" + ) except KeyError as e: print(f"Could not find metrics to build graphs for repo {repo.name}") print(e) @@ -306,7 +310,7 @@ def generate_libyears_graph(oss_entity): for dep in dep_list: dateline.add(dep["dep_name"], [ (timedelta(), elevation), - (timedelta(days=(dep["libyear_value"] * 365)), elevation), + (timedelta(days=dep["libyear_value"] * 365), elevation), ]) #move one line up so that we have no overlap in the timedeltas @@ -317,4 +321,3 @@ def generate_libyears_graph(oss_entity): dateline.show_y_labels = False write_repo_chart_to_file(oss_entity, dateline, "libyear_timeline") - From bf3d166c8b6d48fa951578d90c5662b9fff0be39 Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Thu, 10 Oct 2024 12:06:30 -0600 Subject: [PATCH 20/55] Add scripts file for language summary --- scripts/gen_graphs.py | 131 ++++++++++++++++-- scripts/metricsLib/constants.py | 2 +- scripts/metricsLib/metrics_data_structures.py | 3 + scripts/metricsLib/metrics_definitions.py | 21 +++ templates/repo_report_template.md | 4 + 5 files changed, 152 insertions(+), 9 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 0709baf86b..ba16fc0cdb 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -2,6 +2,7 @@ Module to define methods to create pygals graphs """ import datetime +import re import pygal def generate_all_graphs_for_repos(all_repos): @@ -16,14 +17,28 @@ def generate_all_graphs_for_repos(all_repos): print(f"Generating graphs for repo {repo.name}") generate_solid_gauge_issue_graph(repo) generate_repo_sparklines(repo) + generate_predominant_languages_graph(repo) + generate_language_summary_pie_chart(repo) try: generate_donut_graph_line_complexity_graph(repo) - generate_time_xy_issue_graph(repo, "new_commit_contributors_by_day_over_last_month", "New Contributors") - generate_time_xy_issue_graph(repo, "new_commit_contributors_by_day_over_last_six_months", "New Contributors") + generate_time_xy_issue_graph( + repo, "new_commit_contributors_by_day_over_last_month", "New Contributors" + ) + generate_time_xy_issue_graph( + repo, "new_commit_contributors_by_day_over_last_six_months", "New Contributors" + ) except KeyError as e: print(f"Could not find metrics to build graphs for repo {repo.name}") print(e) + try: + generate_dryness_percentage_graph(repo) + except ValueError as e: + print("Could not parse DRYness due to percentage values being invalid!") + print(e) + except KeyError as e: + print(f"Could not find metrics to build dryness graphs for repo {repo.name}") + print(e) def generate_all_graphs_for_orgs(all_orgs): """ @@ -39,7 +54,6 @@ def generate_all_graphs_for_orgs(all_orgs): generate_time_xy_issue_graph(org, "new_issues_by_day_over_last_month", "New Issues") generate_top_committer_bar_graph(org) - def write_repo_chart_to_file(repo, chart, chart_name, custom_func=None, custom_func_params={}): """ This function's purpose is to save a pygals chart to a path derived from the @@ -63,7 +77,6 @@ def write_repo_chart_to_file(repo, chart, chart_name, custom_func=None, custom_f f"Repo {repo.name} has a division by zero error when trying to make graph") # issues_gauge.render_to_file(repo.get_path_to_graph_data("issue_gauge")) - def generate_repo_sparklines(repo): """ This function generates pygals sparklines graphs for a set of Repository objects. @@ -114,7 +127,6 @@ def generate_time_xy_issue_graph(oss_entity,data_key,legend_key): write_repo_chart_to_file(oss_entity, xy_time_issue_chart, data_key) - def generate_donut_graph_line_complexity_graph(oss_entity): """ This function generates pygals line complexity donut graph @@ -142,7 +154,6 @@ def generate_donut_graph_line_complexity_graph(oss_entity): write_repo_chart_to_file(oss_entity, donut_lines_graph, "total_line_makeup") - def generate_solid_gauge_issue_graph(oss_entity): """ This function generates pygals solid gauge issue/pr graphs for a set of Repository objects. @@ -200,7 +211,7 @@ def generate_top_committer_bar_graph(oss_entity): Arguments: oss_entity: the OSSEntity to create a graph for. """ - + # Create a bar chart object bar_chart = pygal.Bar() bar_chart.title = f"Top Committers in {oss_entity.metric_data['name']}" @@ -216,4 +227,108 @@ def generate_top_committer_bar_graph(oss_entity): bar_chart.add(committer, commits) contributor_count += 1 - write_repo_chart_to_file(oss_entity, bar_chart, "top_committers") \ No newline at end of file + write_repo_chart_to_file(oss_entity, bar_chart, "top_committers") + +def generate_predominant_languages_graph(oss_entity): + """ + This function generates a pygal predominant programming languages guage graph. + + Arguments: + oss_entity: the OSSEntity to create a graph for. + """ + + bar_chart = pygal.Bar() + bar_chart.title = f"Predominant Languages in {oss_entity.metric_data['name']}" + + predominant_lang = oss_entity.metric_data['predominant_langs'] + + for lang, lines in predominant_lang.items(): + bar_chart.add(lang, lines) + + write_repo_chart_to_file(oss_entity, bar_chart, "predominant_langs") + +def parse_cocomo_dryness_metrics(dryness_string): + """ + This function parses the output of the scc dryness metrics. + + For some reason, ULOC, SLOC, and DRYness don't show up in the json and + only show up in the stdout text. + + Arguments: + dryness_string: the string containing the dryness table to parse + + Returns: + A dictionary with the unique lines of code and DRYness percentage + """ + + dryness_metrics = {} + + #Parse output line by line + for line in dryness_string.split('\n'): + #Parse the parts that we want into fields + if 'Unique Lines of Code' in line: + #Use regex to remove all non-numerals from the string + dryness_metrics['total_uloc'] = re.sub('[^0-9.]','',line) + if 'DRYness' in line: + #Use regex to remove all non-numerals from the string + dryness_metrics['DRYness_percentage'] = re.sub('[^0-9.]','',line) + + return dryness_metrics + +def generate_dryness_percentage_graph(oss_entity): + """ + This function generates a pygal DRYness pie graph. + + DRYness = ULOC / SLOC + + WETness = 1 - DRYness + + DRY = Don't repeat yourself + WET = Waste Everybody's time or Write Everything Twice + """ + + dryness_values = parse_cocomo_dryness_metrics( + oss_entity.metric_data["cocomo"]['dryness_table'] + ) + + sloc = (float(dryness_values['total_uloc']) / float(dryness_values['DRYness_percentage'])) + sloc_diff = sloc - float(dryness_values['total_uloc']) + sloc_percent = (sloc_diff / sloc) * 100 + + uloc_percent = (float(dryness_values['total_uloc']) / sloc) * 100 + + pie_chart = pygal.Pie(half_pie=True, legend_at_bottom=True) + pie_chart.title = 'DRYness Percentage Graph' + + #print(dryness_values) + + pie_chart.add( + 'Unique Lines of Code (ULOC) %', uloc_percent + ) + + #Will cause a value error if the dryness value is NaN which can happen. + pie_chart.add( + 'Source Lines of Code (SLOC) %', + #sloc = uloc / DRYness + sloc_percent + ) + + write_repo_chart_to_file(oss_entity, pie_chart, "DRYness") + +def generate_language_summary_pie_chart(oss_entity): + """ + This function generates a pygal for programming languages and total lines written in each language. + + Arguments: + oss_entity: the OSSEntity to create a graph for. + """ + + pie_chart = pygal.Pie() + pie_chart.title = 'Language Summary' + + language_summary = oss_entity.metric_data['cocomo']['languageSummary'] + + for entry in language_summary: + pie_chart.add(entry['Name'], entry['Code']) + + write_repo_chart_to_file(oss_entity, pie_chart, "language_summary") diff --git a/scripts/metricsLib/constants.py b/scripts/metricsLib/constants.py index 752f7a6016..32430d8012 100644 --- a/scripts/metricsLib/constants.py +++ b/scripts/metricsLib/constants.py @@ -6,7 +6,7 @@ from pathlib import Path from enum import Enum -TIMEOUT_IN_SECONDS = 20 +TIMEOUT_IN_SECONDS = 120 BASE_PATH = os.path.dirname(os.path.abspath(__file__)) # Folder Names to send over our projects tracked data PATH_TO_METRICS_DATA = (Path(__file__).parent / diff --git a/scripts/metricsLib/metrics_data_structures.py b/scripts/metricsLib/metrics_data_structures.py index a607034da7..1e71fa592b 100644 --- a/scripts/metricsLib/metrics_data_structures.py +++ b/scripts/metricsLib/metrics_data_structures.py @@ -336,6 +336,9 @@ def get_values(self, params=None): # EX: storing the date and count of each time the amount of followers # increased. try: + #Only continue if the api_label is a list + if type(api_label) != list: + raise TypeError list(api_label) # initialize each label as an empty list diff --git a/scripts/metricsLib/metrics_definitions.py b/scripts/metricsLib/metrics_definitions.py index 2fac3ea936..b6151ecc0f 100644 --- a/scripts/metricsLib/metrics_definitions.py +++ b/scripts/metricsLib/metrics_definitions.py @@ -141,6 +141,14 @@ "/repo-groups/{repo_group_id}/top-committers", {"top_committers": ["email", "commits"]})) +ORG_METRICS.append(ListMetric("orgLibyears", ["repo_group_id"], + AUGUR_HOST + + "/repo-groups/{repo_group_id}/libyear", + {"dependency_libyear_list": [ + "repo_name", "name","libyear","most_recent_collection" + ] + })) + CONTRIBS_LABEL_LAST_MONTH = "new_commit_contributors_by_day_over_last_month" PERIODIC_METRICS.append(ListMetric("newContributorsofCommitsWeekly", @@ -231,3 +239,16 @@ NADIA_ENDPOINT = AUGUR_HOST + "/repos/{repo_id}/nadia-project-labeling-badge/" ADVANCED_METRICS.append(CustomMetric("getNadiaBadgeURL",[ "repo_id"],NADIA_ENDPOINT, parse_nadia_label_into_badge)) + +REPO_LIBYEAR_ENDPOINT = AUGUR_HOST + "/repo-groups/{repo_group_id}/repos/{repo_id}/libyear" +ADVANCED_METRICS.append(ListMetric( + "repoLibyears", + ["repo_group_id","repo_id"], + REPO_LIBYEAR_ENDPOINT, + { + "repo_dependency_libyear_list" : [ + "name","libyear","most_recent_collection" + ] + } + ) +) diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 13e928227d..d93d2c7eaa 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -111,4 +111,8 @@ date_stampLastWeek: {date_stamp} {{% assign optionsArray = '1 Month, 6 Month' | split: ',' %}} {{% assign graphsArray = '/{repo_owner}/{repo_name}/new_commit_contributors_by_day_over_last_month_{repo_name}_data.svg, /{repo_owner}/{repo_name}/new_commit_contributors_by_day_over_last_six_months_{repo_name}_data.svg' | split: ',' %}} {{% render "graph-toggle", baseurl: site.baseurl, name: "new-contributors" options: optionsArray, graphs: graphsArray, title: "Number of Contributors Joining per Interval" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/predominant_langs_{repo_name}_data.svg", title: "Predominant Languages" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/DRYness_{repo_name}_data.svg", title: "DRYness Percentage Graph" %}} \ No newline at end of file From 8de321d78a2cd2a486c9c2a8d1685044c50f8e85 Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Thu, 10 Oct 2024 12:08:52 -0600 Subject: [PATCH 21/55] Add template.md for Language Summary --- templates/repo_report_template.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index d93d2c7eaa..28078a15b9 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -115,4 +115,6 @@ date_stampLastWeek: {date_stamp} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/predominant_langs_{repo_name}_data.svg", title: "Predominant Languages" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/DRYness_{repo_name}_data.svg", title: "DRYness Percentage Graph" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/language_summary_{repo_name}_data.svg", title: "Language Summary" %}} \ No newline at end of file From c02dfe4fe5debf63116506e7f9928e1f1ccdf611 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Thu, 10 Oct 2024 15:21:27 -0500 Subject: [PATCH 22/55] add percent formatter Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index e0241c83e2..364f26c3a6 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -6,6 +6,19 @@ import re import pygal + +def percent_formatter(x): + """ + Function to format percentage values. + + Arguments: + x: Value to format into a percent + Returns: + A string containing the formatted version of x + """ + + return '{:0.2f}%'.format(x) + def generate_all_graphs_for_repos(all_repos): """ Function to generate and save all graphs for the input @@ -174,8 +187,6 @@ def generate_solid_gauge_issue_graph(oss_entity): issues_gauge = pygal.SolidGauge(inner_radius=0.70, legend_at_bottom=True) - def percent_formatter(x): - return '{:0.2f}%'.format(x) issues_gauge.value_formatter = percent_formatter # Generate graph to measure percentage of issues that are open @@ -379,6 +390,7 @@ def generate_dryness_percentage_graph(oss_entity): uloc_percent = (float(dryness_values['total_uloc']) / sloc) * 100 pie_chart = pygal.Pie(half_pie=True, legend_at_bottom=True) + pie_chart.value_formatter = percent_formatter pie_chart.title = 'DRYness Percentage Graph' #print(dryness_values) From 73109b471649161b671d2db707aee6fbcaf5beb6 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Thu, 10 Oct 2024 15:55:16 -0500 Subject: [PATCH 23/55] add formatting methods to format the libyear graph Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 364f26c3a6..2a6ea47fe1 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -19,6 +19,30 @@ def percent_formatter(x): return '{:0.2f}%'.format(x) +def timedelta_formatter(x): + """ + Function to format percentage values. + + Arguments: + x: Value to format into days + Returns: + A string containing the formatted version of x + """ + + return '{} days'.format(x.days) + +def ignore_formatter(x): + """ + Function to ignore values in formatting + + Arguments: + x: Value to ignore + Returns: + A string containing the formatted version of x + """ + + return '' + def generate_all_graphs_for_repos(all_repos): """ Function to generate and save all graphs for the input @@ -317,7 +341,8 @@ def generate_libyears_graph(oss_entity): #timeline object #TODO: Contribute upstream to add a timeline object to pygal dateline = pygal.TimeDeltaLine(x_label_rotation=25,legend_at_bottom=True) - + dateline.x_value_formatter = timedelta_formatter + dateline.value_formatter = ignore_formatter dateline.title = 'Dependency Libyears: Age of Dependency Version in Days' dep_list = parse_libyear_list(raw_dep_list) @@ -348,7 +373,7 @@ def parse_cocomo_dryness_metrics(dryness_string): Arguments: dryness_string: the string containing the dryness table to parse - + Returns: A dictionary with the unique lines of code and DRYness percentage """ @@ -376,7 +401,7 @@ def generate_dryness_percentage_graph(oss_entity): WETness = 1 - DRYness DRY = Don't repeat yourself - WET = Waste Everybody's time or Write Everything Twice + WET = Waste Everybody's time or Write Everything Twice """ dryness_values = parse_cocomo_dryness_metrics( @@ -401,7 +426,7 @@ def generate_dryness_percentage_graph(oss_entity): #Will cause a value error if the dryness value is NaN which can happen. pie_chart.add( - 'Source Lines of Code (SLOC) %', + 'Source Lines of Code (SLOC) %', #sloc = uloc / DRYness sloc_percent ) From cb1c5f963da0175e4de266923b57e8cc2b391215 Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Thu, 10 Oct 2024 15:58:42 -0600 Subject: [PATCH 24/55] Refactor scripts file for language summary --- scripts/gen_graphs.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index ba16fc0cdb..be6a5d37dd 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -315,20 +315,30 @@ def generate_dryness_percentage_graph(oss_entity): write_repo_chart_to_file(oss_entity, pie_chart, "DRYness") + def generate_language_summary_pie_chart(oss_entity): """ - This function generates a pygal for programming languages and total lines written in each language. - + This function generates a pygal pie chart for programming languages and total lines written in each language. + The total LoC is displayed in the chart's title. + Arguments: - oss_entity: the OSSEntity to create a graph for. + oss_entity: the OSSEntity to create a graph for. """ pie_chart = pygal.Pie() - pie_chart.title = 'Language Summary' - language_summary = oss_entity.metric_data['cocomo']['languageSummary'] + language_summary = oss_entity.metric_data.get('cocomo', {}).get('languageSummary') + if not language_summary: + raise ValueError("No valid 'languageSummary' found in the data.") + + total_loc = sum(entry.get('Code', 0) for entry in language_summary) + + pie_chart.title = f'Language Summary (Total SLOC: {total_loc:,})' + + pie_chart.value_formatter = lambda x: f'{x} SLOC' for entry in language_summary: - pie_chart.add(entry['Name'], entry['Code']) + code_lines = entry.get('Code', 0) + pie_chart.add(entry['Name'], code_lines) write_repo_chart_to_file(oss_entity, pie_chart, "language_summary") From 8f98fa81c3f915c885d02bc27160f6763314c88d Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Thu, 10 Oct 2024 18:28:59 -0600 Subject: [PATCH 25/55] Add cost estimate graph, gen.graphs.py --- scripts/gen_graphs.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index be6a5d37dd..0d3caff2a0 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -19,6 +19,7 @@ def generate_all_graphs_for_repos(all_repos): generate_repo_sparklines(repo) generate_predominant_languages_graph(repo) generate_language_summary_pie_chart(repo) + generate_cost_estimates_bar_chart(repo) try: generate_donut_graph_line_complexity_graph(repo) generate_time_xy_issue_graph( @@ -342,3 +343,31 @@ def generate_language_summary_pie_chart(oss_entity): pie_chart.add(entry['Name'], code_lines) write_repo_chart_to_file(oss_entity, pie_chart, "language_summary") + + +def generate_cost_estimates_bar_chart(oss_entity): + """ + This function generates a pygal bar chart for estimated costs with rounded values and a dollar sign. + + Arguments: + oss_entity: the OSSEntity to create a graph for. + """ + + bar_chart = pygal.Bar() + + metric_data = oss_entity.metric_data['cocomo'] + + estimatedCost_low = metric_data.get('estimatedCost_low', 0) + estimatedCost_high = metric_data.get('estimatedCost_high', 0) + + bar_chart.value_formatter = lambda x: f'${x:,.2f}' + + average_cost = (estimatedCost_low + estimatedCost_high) / 2 + + bar_chart.title = f'Estimated Project Costs in $ (Average Cost: ${average_cost:,.2f})' + + bar_chart.add(f'Estimated Cost Low (${estimatedCost_low:,.2f})', estimatedCost_low) + bar_chart.add(f'Estimated Cost High (${estimatedCost_high:,.2f})', estimatedCost_high) + + write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_costs") + From b6f86f5a05968586d695f4c374d9a6f26f6a8ed2 Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Thu, 10 Oct 2024 18:29:56 -0600 Subject: [PATCH 26/55] Add cost estimate graph, repo_report_template.md --- templates/repo_report_template.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 28078a15b9..73766e613a 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -117,4 +117,6 @@ date_stampLastWeek: {date_stamp} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/DRYness_{repo_name}_data.svg", title: "DRYness Percentage Graph" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/language_summary_{repo_name}_data.svg", title: "Language Summary" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/estimated_project_costs_{repo_name}_data.svg", title: "Estimated Costs" %}} \ No newline at end of file From aec595d14616f890f9dc2836f2b9cafa86b346bb Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Fri, 11 Oct 2024 10:40:03 -0400 Subject: [PATCH 27/55] fixed linting and moved JS Signed-off-by: sachin-panayil --- app/site/_layouts/repo-report.liquid | 26 +++++++++++++++++++++++--- app/src/js/graphs.js | 14 -------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/app/site/_layouts/repo-report.liquid b/app/site/_layouts/repo-report.liquid index 83c058fe9c..7775cfcf23 100644 --- a/app/site/_layouts/repo-report.liquid +++ b/app/site/_layouts/repo-report.liquid @@ -1,7 +1,22 @@ --- layout: base --- + +
{% assign project = projects | findObject: repo %} @@ -172,13 +187,18 @@ layout: base

- {{ project.created_at | date: "%B %d, %Y" }} + {{ project.created_at | date: '%B %d, %Y' }}

diff --git a/app/src/js/graphs.js b/app/src/js/graphs.js index 5c2237eed7..94a85282c5 100644 --- a/app/src/js/graphs.js +++ b/app/src/js/graphs.js @@ -17,17 +17,3 @@ window.showGraph = function (selectedGraphId, className, buttonName) { } }) } - -// goes inside includes/js for the time being -function toggleDateAge(element) { - const dateSpan = element.querySelector('.date'); - const ageSpan = element.querySelector('.age'); - - if (ageSpan.style.display === 'none') { - dateSpan.style.display = 'none'; - ageSpan.style.display = 'inline'; - } else { - dateSpan.style.display = 'inline'; - ageSpan.style.display = 'none'; - } -} \ No newline at end of file From 338dc6640bb670fa9092efb732909d314f1cc377 Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Fri, 11 Oct 2024 14:15:52 -0400 Subject: [PATCH 28/55] added requested changes Signed-off-by: sachin-panayil --- app/site/_layouts/repo-report.liquid | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/app/site/_layouts/repo-report.liquid b/app/site/_layouts/repo-report.liquid index 7775cfcf23..5af2c66eea 100644 --- a/app/site/_layouts/repo-report.liquid +++ b/app/site/_layouts/repo-report.liquid @@ -3,16 +3,16 @@ layout: base --- @@ -170,7 +170,7 @@ layout: base

{{ project.pull_requests_count }}

-
+

@@ -182,10 +182,11 @@ layout: base {% lucide "calendar" %} - Age Of Project + Project Creation Date +

-

+

{{ project.created_at | date: '%B %d, %Y' }} From b3dce6d60956fa68d549e984baa7675391597ce7 Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Fri, 11 Oct 2024 14:55:56 -0400 Subject: [PATCH 29/55] linting Signed-off-by: sachin-panayil --- app/site/_layouts/repo-report.liquid | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/site/_layouts/repo-report.liquid b/app/site/_layouts/repo-report.liquid index 5af2c66eea..58d38c5acb 100644 --- a/app/site/_layouts/repo-report.liquid +++ b/app/site/_layouts/repo-report.liquid @@ -3,14 +3,14 @@ layout: base --- From 655c58585890450945e5bdc2c7653a03eacd004b Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Fri, 11 Oct 2024 15:05:19 -0400 Subject: [PATCH 31/55] even more linting Signed-off-by: sachin-panayil --- app/site/_layouts/repo-report.liquid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/site/_layouts/repo-report.liquid b/app/site/_layouts/repo-report.liquid index a5ccbddc3e..8470dfbffb 100644 --- a/app/site/_layouts/repo-report.liquid +++ b/app/site/_layouts/repo-report.liquid @@ -5,11 +5,11 @@ layout: base function toggleDateAge(element) { const dateElements = element.querySelectorAll('.date') const ageElements = element.querySelectorAll('.age') - + dateElements.forEach((element) => { element.style.display = element.style.display === 'none' ? 'inline' : 'none' }) - + ageElements.forEach((element) => { element.style.display = element.style.display === 'none' ? 'inline' : 'none' }) From fd37e485033c860d54b603fff2b1021a52754a69 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Fri, 11 Oct 2024 14:16:36 -0500 Subject: [PATCH 32/55] add up libyears and add to title Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index ace469b3ba..23dd678d10 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -344,9 +344,13 @@ def generate_libyears_graph(oss_entity): dateline = pygal.TimeDeltaLine(x_label_rotation=25,legend_at_bottom=True) dateline.x_value_formatter = timedelta_formatter dateline.value_formatter = ignore_formatter - dateline.title = 'Dependency Libyears: Age of Dependency Version in Days' + dep_list = parse_libyear_list(raw_dep_list) + total_libyears_ood = sum(n['libyear_value'] for n in dep_list) + + dateline.title = f"""Dependency Libyears: Age of Dependency Version in Days + \nTotal Libyears: {total_libyears_ood}""" #We are going to treat the y-axis as having one dep per level in the graph elevation = 0 From 2844c0920c57c4795e5918d34a7e0a88b61b8190 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Fri, 11 Oct 2024 14:33:10 -0500 Subject: [PATCH 33/55] stop language summary from raising error and then edit libyear chart title Signed-off-by: Isaac Milarsky --- scripts/gen_graphs.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 23dd678d10..f000bf7838 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -349,8 +349,8 @@ def generate_libyears_graph(oss_entity): dep_list = parse_libyear_list(raw_dep_list) total_libyears_ood = sum(n['libyear_value'] for n in dep_list) - dateline.title = f"""Dependency Libyears: Age of Dependency Version in Days - \nTotal Libyears: {total_libyears_ood}""" + dateline.title = f"""Dependency Libyears: Age of Dependency Version + Total Libyears: {round(total_libyears_ood,1)}""" #We are going to treat the y-axis as having one dep per level in the graph elevation = 0 @@ -443,7 +443,7 @@ def generate_language_summary_pie_chart(oss_entity): """ This function generates a pygal pie chart for programming languages and total lines written in each language. The total LoC is displayed in the chart's title. - + Arguments: oss_entity: the OSSEntity to create a graph for. """ @@ -452,10 +452,11 @@ def generate_language_summary_pie_chart(oss_entity): language_summary = oss_entity.metric_data.get('cocomo', {}).get('languageSummary') if not language_summary: - raise ValueError("No valid 'languageSummary' found in the data.") + print("No valid 'languageSummary' found in the data.") + return total_loc = sum(entry.get('Code', 0) for entry in language_summary) - + pie_chart.title = f'Language Summary (Total SLOC: {total_loc:,})' pie_chart.value_formatter = lambda x: f'{x} SLOC' From b4cb3af90aa235ca9d3028de67fae4826b66b511 Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Thu, 10 Oct 2024 18:28:59 -0600 Subject: [PATCH 34/55] Add cost estimate graph, gen.graphs.py Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index be6a5d37dd..0d3caff2a0 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -19,6 +19,7 @@ def generate_all_graphs_for_repos(all_repos): generate_repo_sparklines(repo) generate_predominant_languages_graph(repo) generate_language_summary_pie_chart(repo) + generate_cost_estimates_bar_chart(repo) try: generate_donut_graph_line_complexity_graph(repo) generate_time_xy_issue_graph( @@ -342,3 +343,31 @@ def generate_language_summary_pie_chart(oss_entity): pie_chart.add(entry['Name'], code_lines) write_repo_chart_to_file(oss_entity, pie_chart, "language_summary") + + +def generate_cost_estimates_bar_chart(oss_entity): + """ + This function generates a pygal bar chart for estimated costs with rounded values and a dollar sign. + + Arguments: + oss_entity: the OSSEntity to create a graph for. + """ + + bar_chart = pygal.Bar() + + metric_data = oss_entity.metric_data['cocomo'] + + estimatedCost_low = metric_data.get('estimatedCost_low', 0) + estimatedCost_high = metric_data.get('estimatedCost_high', 0) + + bar_chart.value_formatter = lambda x: f'${x:,.2f}' + + average_cost = (estimatedCost_low + estimatedCost_high) / 2 + + bar_chart.title = f'Estimated Project Costs in $ (Average Cost: ${average_cost:,.2f})' + + bar_chart.add(f'Estimated Cost Low (${estimatedCost_low:,.2f})', estimatedCost_low) + bar_chart.add(f'Estimated Cost High (${estimatedCost_high:,.2f})', estimatedCost_high) + + write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_costs") + From 61ac0d82d6cd373d686a32af74414c7b44d985d2 Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Thu, 10 Oct 2024 18:29:56 -0600 Subject: [PATCH 35/55] Add cost estimate graph, repo_report_template.md Signed-off-by: Dinne Kopelevich --- templates/repo_report_template.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 28078a15b9..73766e613a 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -117,4 +117,6 @@ date_stampLastWeek: {date_stamp} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/DRYness_{repo_name}_data.svg", title: "DRYness Percentage Graph" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/language_summary_{repo_name}_data.svg", title: "Language Summary" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/estimated_project_costs_{repo_name}_data.svg", title: "Estimated Costs" %}}

\ No newline at end of file From 90e6153368827084b48f7024928714820a8030cf Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Tue, 15 Oct 2024 09:24:20 -0600 Subject: [PATCH 36/55] Add Time Estimate Chart Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 29 ++++++++++++++++++++++++++++- templates/repo_report_template.md | 2 ++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 781ee0ff00..02a204e6f0 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -58,6 +58,7 @@ def generate_all_graphs_for_repos(all_repos): generate_predominant_languages_graph(repo) generate_language_summary_pie_chart(repo) generate_cost_estimates_bar_chart(repo) + generate_time_estimates_bar_chart(repo) try: generate_donut_graph_line_complexity_graph(repo) generate_time_xy_issue_graph( @@ -453,7 +454,7 @@ def generate_language_summary_pie_chart(oss_entity): total_loc = sum(entry.get('Code', 0) for entry in language_summary) - pie_chart.title = f'Language Summary (Total SLOC: {total_loc:,})' + pie_chart.title = f'Language Summary \n Total Source Lines of Code (SLOC): {total_loc:,}' pie_chart.value_formatter = lambda x: f'{x} SLOC' @@ -491,3 +492,29 @@ def generate_cost_estimates_bar_chart(oss_entity): write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_costs") + +def generate_time_estimates_bar_chart(oss_entity): + """ + This function generates a pygal bar chart for estimated time of project in months rounded to the nearest tenth. + + Arguments: + oss_entity: the OSSEntity to create a graph for. + """ + + bar_chart = pygal.Bar(legend_at_bottom=True) + + metric_data = oss_entity.metric_data['cocomo'] + + estimatedScheduleMonths_low = metric_data.get('estimatedScheduleMonths_low', 0) + estimatedScheduleMonths_high = metric_data.get('estimatedScheduleMonths_high', 0) + + bar_chart.value_formatter = lambda x: f'{x:,.1f} mos' + + average_time = (estimatedScheduleMonths_low + estimatedScheduleMonths_high) / 2 + + bar_chart.title = f'Estimated Project Time in Months From Constructive Cost Model (COCOMO) \n Average Time: {average_time:,.1f} mos' + + bar_chart.add(f'Estimated Time Low ({estimatedScheduleMonths_low:,.1f} mos)', estimatedScheduleMonths_low) + bar_chart.add(f'Estimated Time High ({estimatedScheduleMonths_high:,.1f} mos)', estimatedScheduleMonths_high) + + write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_time") \ No newline at end of file diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 83b2801707..e3908c8f47 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -121,4 +121,6 @@ date_stampLastWeek: {date_stamp} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/language_summary_{repo_name}_data.svg", title: "Language Summary" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/estimated_project_costs_{repo_name}_data.svg", title: "Estimated Costs" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/estimated_project_time_{repo_name}_data.svg", title: "Estimated Time" %}}
\ No newline at end of file From 28234c6285a61f96f6b8344242a648c5d857189f Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Tue, 15 Oct 2024 10:25:39 -0600 Subject: [PATCH 37/55] Refactor Time Estimate Chart Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 02a204e6f0..30eaffbd05 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -497,6 +497,8 @@ def generate_time_estimates_bar_chart(oss_entity): """ This function generates a pygal bar chart for estimated time of project in months rounded to the nearest tenth. + estimatedScheduleMonths_low is used for time. + Arguments: oss_entity: the OSSEntity to create a graph for. """ @@ -506,15 +508,11 @@ def generate_time_estimates_bar_chart(oss_entity): metric_data = oss_entity.metric_data['cocomo'] estimatedScheduleMonths_low = metric_data.get('estimatedScheduleMonths_low', 0) - estimatedScheduleMonths_high = metric_data.get('estimatedScheduleMonths_high', 0) - + bar_chart.value_formatter = lambda x: f'{x:,.1f} mos' - average_time = (estimatedScheduleMonths_low + estimatedScheduleMonths_high) / 2 - - bar_chart.title = f'Estimated Project Time in Months From Constructive Cost Model (COCOMO) \n Average Time: {average_time:,.1f} mos' + bar_chart.title = f'Estimated Project Time in Months From Constructive Cost Model (COCOMO)' bar_chart.add(f'Estimated Time Low ({estimatedScheduleMonths_low:,.1f} mos)', estimatedScheduleMonths_low) - bar_chart.add(f'Estimated Time High ({estimatedScheduleMonths_high:,.1f} mos)', estimatedScheduleMonths_high) - + write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_time") \ No newline at end of file From 49a0753ccc1139774e67fd6de9df1db69b77c98a Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Tue, 15 Oct 2024 10:27:23 -0600 Subject: [PATCH 38/55] Refactor Time Estimate Chart Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 30eaffbd05..3de593cd5f 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -515,4 +515,5 @@ def generate_time_estimates_bar_chart(oss_entity): bar_chart.add(f'Estimated Time Low ({estimatedScheduleMonths_low:,.1f} mos)', estimatedScheduleMonths_low) - write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_time") \ No newline at end of file + write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_time") + \ No newline at end of file From 3017b5688d9564616654822162e4976c1ff4a02f Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Tue, 15 Oct 2024 10:29:18 -0600 Subject: [PATCH 39/55] Refactor Time Estimate Chart Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 3de593cd5f..e85e6f1f06 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -511,9 +511,8 @@ def generate_time_estimates_bar_chart(oss_entity): bar_chart.value_formatter = lambda x: f'{x:,.1f} mos' - bar_chart.title = f'Estimated Project Time in Months From Constructive Cost Model (COCOMO)' + bar_chart.title = 'Estimated Project Time in Months From Constructive Cost Model (COCOMO)' bar_chart.add(f'Estimated Time Low ({estimatedScheduleMonths_low:,.1f} mos)', estimatedScheduleMonths_low) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_time") - \ No newline at end of file From fdc52a5059087f284a4ed7718afef543035f4336 Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Tue, 15 Oct 2024 13:40:31 -0400 Subject: [PATCH 40/55] consolidated the two language graphs Signed-off-by: sachin-panayil --- app/site/_includes/graph-toggle.liquid | 2 +- templates/repo_report_template.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/site/_includes/graph-toggle.liquid b/app/site/_includes/graph-toggle.liquid index 2ca5907911..be100fb6b3 100644 --- a/app/site/_includes/graph-toggle.liquid +++ b/app/site/_includes/graph-toggle.liquid @@ -34,7 +34,7 @@ {% endif %} {% assign graphPath = graph | strip %} {% assign distPath = baseurl | append: "/assets/img/graphs" | append: graphPath %} - {% assign fileExtension = path | split: '.' | last %} + {% assign fileExtension = graphPathath | split: '.' | last %}
{% if fileExtension == 'svg' %} diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 83b2801707..5e29f676de 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -111,14 +111,14 @@ date_stampLastWeek: {date_stamp} {{% assign optionsArray = '1 Month, 6 Month' | split: ',' %}} {{% assign graphsArray = '/{repo_owner}/{repo_name}/new_commit_contributors_by_day_over_last_month_{repo_name}_data.svg, /{repo_owner}/{repo_name}/new_commit_contributors_by_day_over_last_six_months_{repo_name}_data.svg' | split: ',' %}} {{% render "graph-toggle", baseurl: site.baseurl, name: "new-contributors" options: optionsArray, graphs: graphsArray, title: "Number of Contributors Joining per Interval" %}} - - {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/predominant_langs_{repo_name}_data.svg", title: "Predominant Languages" %}} + + {{% assign optionsArray = 'Summary, Predominant' | split: ',' %}} + {{% assign graphsArray = "/{repo_owner}/{repo_name}/language_summary_{repo_name}_data.svg, /{repo_owner}/{repo_name}/predominant_langs_{repo_name}_data.svg" | split: ',' %}} + {{% render "graph-toggle" baseurl: site.baseurl, name:"language-information" options: optionsArray, graphs: graphsArray, title: "Language Information" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/libyear_timeline_{repo_name}_data.svg", title: "Dependency Libyears" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/DRYness_{repo_name}_data.svg", title: "DRYness Percentage Graph" %}} - - {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/language_summary_{repo_name}_data.svg", title: "Language Summary" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/estimated_project_costs_{repo_name}_data.svg", title: "Estimated Costs" %}} \ No newline at end of file From 1b8ed1bc208423d48be3333465e9f06f441d1c56 Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Tue, 15 Oct 2024 13:46:31 -0400 Subject: [PATCH 41/55] fixed typo Signed-off-by: sachin-panayil --- app/site/_includes/graph-toggle.liquid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/site/_includes/graph-toggle.liquid b/app/site/_includes/graph-toggle.liquid index be100fb6b3..bc693f6ed9 100644 --- a/app/site/_includes/graph-toggle.liquid +++ b/app/site/_includes/graph-toggle.liquid @@ -34,7 +34,7 @@ {% endif %} {% assign graphPath = graph | strip %} {% assign distPath = baseurl | append: "/assets/img/graphs" | append: graphPath %} - {% assign fileExtension = graphPathath | split: '.' | last %} + {% assign fileExtension = graphPath | split: '.' | last %}
{% if fileExtension == 'svg' %} From db8b3f6e030c1f34df211103fdf2aca0439f014d Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Tue, 15 Oct 2024 17:08:07 -0600 Subject: [PATCH 42/55] Refactor time chart to size bar Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index e85e6f1f06..6ad9c94ed4 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -508,11 +508,13 @@ def generate_time_estimates_bar_chart(oss_entity): metric_data = oss_entity.metric_data['cocomo'] estimatedScheduleMonths_low = metric_data.get('estimatedScheduleMonths_low', 0) - + bar_chart.value_formatter = lambda x: f'{x:,.1f} mos' bar_chart.title = 'Estimated Project Time in Months From Constructive Cost Model (COCOMO)' - bar_chart.add(f'Estimated Time Low ({estimatedScheduleMonths_low:,.1f} mos)', estimatedScheduleMonths_low) - + bar_chart.add(None, [0]) + bar_chart.add(f'Estimated Time ({estimatedScheduleMonths_low:,.1f} mos)', estimatedScheduleMonths_low) + bar_chart.add(None, [0]) + write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_time") From 65a92ea37fceee777398d1c9f2911ccd48ca98ed Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Wed, 16 Oct 2024 14:45:50 -0600 Subject: [PATCH 43/55] Add individual contributors graph Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 69 +++++++++++++++++++++++++------ templates/repo_report_template.md | 2 + 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 2d3ae6880a..3d186fc081 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -443,7 +443,9 @@ def generate_dryness_percentage_graph(oss_entity): def generate_language_summary_pie_chart(oss_entity): """ - This function generates a pygal pie chart for programming languages and total lines written in each language. + This function generates a pygal pie chart for programming languages + and total lines written in each language. + The total LoC is displayed in the chart's title. Arguments: @@ -472,7 +474,8 @@ def generate_language_summary_pie_chart(oss_entity): def generate_cost_estimates_bar_chart(oss_entity): """ - This function generates a pygal bar chart for estimated costs with rounded values and a dollar sign. + This function generates a pygal bar chart for estimated costs + with rounded values and a dollar sign. Arguments: oss_entity: the OSSEntity to create a graph for. @@ -480,27 +483,33 @@ def generate_cost_estimates_bar_chart(oss_entity): bar_chart = pygal.Bar(legend_at_bottom=True) + if oss_entity.metric_data is not None: + metric_data = oss_entity.metric_data.get('cocomo', {}) + estimated_cost_low = metric_data.get('estimatedCost_low', 0) + estimated_cost_high = metric_data.get('estimatedCost_high', 0) + else: + estimated_cost_low = 0.0 + estimated_cost_high = 0.0 - metric_data = oss_entity.metric_data['cocomo'] - - estimatedCost_low = metric_data.get('estimatedCost_low', 0) - estimatedCost_high = metric_data.get('estimatedCost_high', 0) + formatted_estimated_cost_low = float(estimated_cost_low or 0.0) + formatted_estimated_cost_high = float(estimated_cost_high or 0.0) bar_chart.value_formatter = lambda x: f'${x:,.2f}' - average_cost = (estimatedCost_low + estimatedCost_high) / 2 + average_cost = (formatted_estimated_cost_low + formatted_estimated_cost_high) / 2 bar_chart.title = f'Estimated Project Costs in $ From Constructive Cost Model (COCOMO) \n Average Cost: ${average_cost:,.2f}' - bar_chart.add(f'Estimated Cost Low (${estimatedCost_low:,.2f})', estimatedCost_low) - bar_chart.add(f'Estimated Cost High (${estimatedCost_high:,.2f})', estimatedCost_high) + bar_chart.add(f'Estimated Cost Low (${formatted_estimated_cost_low:,.2f})', estimated_cost_low) + bar_chart.add(f'Estimated Cost High (${formatted_estimated_cost_high:,.2f})', estimated_cost_high) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_costs") def generate_time_estimates_bar_chart(oss_entity): """ - This function generates a pygal bar chart for estimated time of project in months rounded to the nearest tenth. + This function generates a pygal bar chart for estimated time + of project in months rounded to the nearest tenth. estimatedScheduleMonths_low is used for time. @@ -510,16 +519,50 @@ def generate_time_estimates_bar_chart(oss_entity): bar_chart = pygal.Bar(legend_at_bottom=True) - metric_data = oss_entity.metric_data['cocomo'] + if oss_entity.metric_data is not None: + metric_data = oss_entity.metric_data.get('cocomo', {}) + estimatedScheduleMonths_low = metric_data.get('estimatedScheduleMonths_low', 0) + else: + estimatedScheduleMonths_low = 0 - estimatedScheduleMonths_low = metric_data.get('estimatedScheduleMonths_low', 0) + formatted_estimated_months = float(estimatedScheduleMonths_low or 0.0) bar_chart.value_formatter = lambda x: f'{x:,.1f} mos' bar_chart.title = 'Estimated Project Time in Months From Constructive Cost Model (COCOMO)' bar_chart.add(None, [0]) - bar_chart.add(f'Estimated Time ({estimatedScheduleMonths_low:,.1f} mos)', estimatedScheduleMonths_low) + bar_chart.add(f'Estimated Time ({formatted_estimated_months:,.1f} mos)', estimatedScheduleMonths_low) bar_chart.add(None, [0]) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_time") + + +def generate_people_estimate_bar_chart(oss_entity): + """ + This function generates a pygal bar chart for estimated people + working on the project rounded to the nearest integer. + + estimatedPeople_low is used for contributors. + + Arguments: + oss_entity: the OSSEntity to create a graph for. + """ + + bar_chart = pygal.Bar(legend_at_bottom=True) + + if oss_entity.metric_data is not None: + metric_data = oss_entity.metric_data.get('cocomo', {}) + estimatedPeople_low = metric_data.get('estimatedPeople_low', 0) + else: + estimatedPeople_low = 0 + + bar_chart.value_formatter = lambda x: f'{x:,.0f} ppl' + + bar_chart.title = 'Estimated Individual Project Contributors From Constructive Cost Model (COCOMO)' + + bar_chart.add(None, [0]) + bar_chart.add(f'Estimated Contributors ({estimatedPeople_low:,.0f} ppl)', estimatedPeople_low) + bar_chart.add(None, [0]) + + write_repo_chart_to_file(oss_entity, bar_chart, "estimated_people_contributing") diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 100a05934a..ce506a9f6c 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -123,4 +123,6 @@ date_stampLastWeek: {date_stamp} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/estimated_project_costs_{repo_name}_data.svg", title: "Estimated Costs" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/estimated_project_time_{repo_name}_data.svg", title: "Estimated Time" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/estimated_people_contributing_{repo_name}_data.svg", title: "Estimated Individual Contributors" %}} \ No newline at end of file From 61df101c39f1e8295146ea5cfaa76085a2487954 Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Wed, 16 Oct 2024 15:14:03 -0600 Subject: [PATCH 44/55] Fix camelCase and line length Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 3d186fc081..42bc9eb21b 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -496,12 +496,15 @@ def generate_cost_estimates_bar_chart(oss_entity): bar_chart.value_formatter = lambda x: f'${x:,.2f}' - average_cost = (formatted_estimated_cost_low + formatted_estimated_cost_high) / 2 + average_cost = (formatted_estimated_cost_low + + formatted_estimated_cost_high) / 2 bar_chart.title = f'Estimated Project Costs in $ From Constructive Cost Model (COCOMO) \n Average Cost: ${average_cost:,.2f}' - bar_chart.add(f'Estimated Cost Low (${formatted_estimated_cost_low:,.2f})', estimated_cost_low) - bar_chart.add(f'Estimated Cost High (${formatted_estimated_cost_high:,.2f})', estimated_cost_high) + bar_chart.add(f'Estimated Cost Low (${formatted_estimated_cost_low:,.2f})', + estimated_cost_low) + bar_chart.add(f'Estimated Cost High (${formatted_estimated_cost_high:,.2f})', + estimated_cost_high) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_costs") @@ -521,18 +524,19 @@ def generate_time_estimates_bar_chart(oss_entity): if oss_entity.metric_data is not None: metric_data = oss_entity.metric_data.get('cocomo', {}) - estimatedScheduleMonths_low = metric_data.get('estimatedScheduleMonths_low', 0) + estimated_schedule_months_low = metric_data.get('estimatedScheduleMonths_low', 0) else: - estimatedScheduleMonths_low = 0 + estimated_schedule_months_low = 0 - formatted_estimated_months = float(estimatedScheduleMonths_low or 0.0) + formatted_estimated_months = float(estimated_schedule_months_low or 0.0) bar_chart.value_formatter = lambda x: f'{x:,.1f} mos' bar_chart.title = 'Estimated Project Time in Months From Constructive Cost Model (COCOMO)' bar_chart.add(None, [0]) - bar_chart.add(f'Estimated Time ({formatted_estimated_months:,.1f} mos)', estimatedScheduleMonths_low) + bar_chart.add(f'Estimated Time ({formatted_estimated_months:,.1f} mos)', + estimated_schedule_months_low) bar_chart.add(None, [0]) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_time") @@ -553,16 +557,16 @@ def generate_people_estimate_bar_chart(oss_entity): if oss_entity.metric_data is not None: metric_data = oss_entity.metric_data.get('cocomo', {}) - estimatedPeople_low = metric_data.get('estimatedPeople_low', 0) + estimated_people_low = metric_data.get('estimatedPeople_low', 0) else: - estimatedPeople_low = 0 + estimated_people_low = 0 bar_chart.value_formatter = lambda x: f'{x:,.0f} ppl' bar_chart.title = 'Estimated Individual Project Contributors From Constructive Cost Model (COCOMO)' bar_chart.add(None, [0]) - bar_chart.add(f'Estimated Contributors ({estimatedPeople_low:,.0f} ppl)', estimatedPeople_low) + bar_chart.add(f'Estimated Contributors ({estimated_people_low:,.0f} ppl)', estimated_people_low) bar_chart.add(None, [0]) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_people_contributing") From 4ea02e0274db3f92164a8990f8432a03f4309415 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 16 Oct 2024 16:41:45 -0500 Subject: [PATCH 45/55] patch BaseMetric to handle ratelimits Signed-off-by: Isaac Milarsky --- scripts/metricsLib/metrics_data_structures.py | 61 +++++++++++++------ 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/scripts/metricsLib/metrics_data_structures.py b/scripts/metricsLib/metrics_data_structures.py index 1e71fa592b..e5d2c53922 100644 --- a/scripts/metricsLib/metrics_data_structures.py +++ b/scripts/metricsLib/metrics_data_structures.py @@ -4,6 +4,7 @@ import json from json.decoder import JSONDecodeError import datetime +from time import sleep, mktime, gmtime, time, localtime from functools import reduce import operator import requests @@ -75,25 +76,49 @@ def hit_metric(self, params=None): endpoint_to_hit = self.url.format(**params) request_params = None - if self.headers: - _args_ = (self.method, endpoint_to_hit) - _kwargs_ = { - "params": request_params, - "headers": self.headers, - "timeout": TIMEOUT_IN_SECONDS - } - response = requests.request(*_args_, **_kwargs_) - else: - response = requests.request( - self.method, endpoint_to_hit, params=request_params, timeout=TIMEOUT_IN_SECONDS) - - try: - if response.status_code == 200: - response_json = json.loads(response.text) + attempts = 0 + + while attempts < 10: + if self.headers: + _args_ = (self.method, endpoint_to_hit) + _kwargs_ = { + "params": request_params, + "headers": self.headers, + "timeout": TIMEOUT_IN_SECONDS + } + response = requests.request(*_args_, **_kwargs_) else: - raise ConnectionError(f"Non valid status code {response.status_code}!") - except JSONDecodeError: - response_json = {} + response = requests.request( + self.method, endpoint_to_hit, params=request_params, timeout=TIMEOUT_IN_SECONDS) + + try: + if response.status_code == 200: + response_json = json.loads(response.text) + break + + #check for rate limit response + if response.status_code in (403,429): + #rate limit was triggered. + wait_until = int(response.headers.get("x-ratelimit-reset")) + wait_in_seconds = int( + mktime(gmtime(wait_until)) - + mktime(gmtime(time())) + ) + wait_until_time = localtime(wait_until) + + print(f"Ran into rate limit sleeping for {self.name}!") + print( + f"sleeping until {wait_until_time.tm_hour}:{wait_until_time.tm_min} ({wait_in_seconds} seconds)" + ) + sleep(wait_in_seconds) + + response_json = {} + attempts += 1 + else: + raise ConnectionError(f"Non valid status code {response.status_code}!") + except JSONDecodeError: + response_json = {} + attempts += 1 return response_json From 5e4900606a22b64d85239b753390b3408416cbe5 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 16 Oct 2024 16:44:50 -0500 Subject: [PATCH 46/55] change else to elif Signed-off-by: Isaac Milarsky --- scripts/metricsLib/metrics_data_structures.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/metricsLib/metrics_data_structures.py b/scripts/metricsLib/metrics_data_structures.py index e5d2c53922..01c15163e9 100644 --- a/scripts/metricsLib/metrics_data_structures.py +++ b/scripts/metricsLib/metrics_data_structures.py @@ -95,9 +95,7 @@ def hit_metric(self, params=None): if response.status_code == 200: response_json = json.loads(response.text) break - - #check for rate limit response - if response.status_code in (403,429): + elif response.status_code in (403,429): #rate limit was triggered. wait_until = int(response.headers.get("x-ratelimit-reset")) wait_in_seconds = int( From cd32c41cae52f6e61bcf5e32e641829fa951102a Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Wed, 16 Oct 2024 16:54:57 -0500 Subject: [PATCH 47/55] add retries as a constant in constants.py and raise a connectionError if rate limit can't be slept to fix Signed-off-by: Isaac Milarsky --- scripts/metricsLib/constants.py | 1 + scripts/metricsLib/metrics_data_structures.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/metricsLib/constants.py b/scripts/metricsLib/constants.py index 32430d8012..af1a71ab20 100644 --- a/scripts/metricsLib/constants.py +++ b/scripts/metricsLib/constants.py @@ -7,6 +7,7 @@ from enum import Enum TIMEOUT_IN_SECONDS = 120 +REQUEST_RETRIES = 5 BASE_PATH = os.path.dirname(os.path.abspath(__file__)) # Folder Names to send over our projects tracked data PATH_TO_METRICS_DATA = (Path(__file__).parent / diff --git a/scripts/metricsLib/metrics_data_structures.py b/scripts/metricsLib/metrics_data_structures.py index 01c15163e9..c2d67fcbb9 100644 --- a/scripts/metricsLib/metrics_data_structures.py +++ b/scripts/metricsLib/metrics_data_structures.py @@ -8,7 +8,7 @@ from functools import reduce import operator import requests -from metricsLib.constants import TIMEOUT_IN_SECONDS, GH_GQL_ENDPOINT +from metricsLib.constants import TIMEOUT_IN_SECONDS, GH_GQL_ENDPOINT, REQUEST_RETRIES # Simple metric that can be represented by a count or value. @@ -78,7 +78,7 @@ def hit_metric(self, params=None): attempts = 0 - while attempts < 10: + while attempts < REQUEST_RETRIES: if self.headers: _args_ = (self.method, endpoint_to_hit) _kwargs_ = { @@ -112,6 +112,11 @@ def hit_metric(self, params=None): response_json = {} attempts += 1 + + if attempts >= REQUEST_RETRIES: + raise ConnectionError( + f"Rate limit was reached and couldn't be rectified after {attempts} tries" + ) else: raise ConnectionError(f"Non valid status code {response.status_code}!") except JSONDecodeError: From 62cc52992b769e8de6f9f59c2d74c5ca75b5f100 Mon Sep 17 00:00:00 2001 From: Dinne Kopelevich Date: Thu, 17 Oct 2024 08:19:20 -0600 Subject: [PATCH 48/55] Refactor cost graph Signed-off-by: Dinne Kopelevich --- scripts/gen_graphs.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 42bc9eb21b..1e77182fcb 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -485,25 +485,22 @@ def generate_cost_estimates_bar_chart(oss_entity): if oss_entity.metric_data is not None: metric_data = oss_entity.metric_data.get('cocomo', {}) - estimated_cost_low = metric_data.get('estimatedCost_low', 0) - estimated_cost_high = metric_data.get('estimatedCost_high', 0) + estimated_cost_low = float(metric_data.get('estimatedCost_low', 0) or 0.0) + estimated_cost_high = float(metric_data.get('estimatedCost_high', 0) or 0.0) else: estimated_cost_low = 0.0 estimated_cost_high = 0.0 - formatted_estimated_cost_low = float(estimated_cost_low or 0.0) - formatted_estimated_cost_high = float(estimated_cost_high or 0.0) - bar_chart.value_formatter = lambda x: f'${x:,.2f}' - average_cost = (formatted_estimated_cost_low + - formatted_estimated_cost_high) / 2 + average_cost = (estimated_cost_low + + estimated_cost_high) / 2 bar_chart.title = f'Estimated Project Costs in $ From Constructive Cost Model (COCOMO) \n Average Cost: ${average_cost:,.2f}' - bar_chart.add(f'Estimated Cost Low (${formatted_estimated_cost_low:,.2f})', + bar_chart.add(f'Estimated Cost Low (${estimated_cost_low:,.2f})', estimated_cost_low) - bar_chart.add(f'Estimated Cost High (${formatted_estimated_cost_high:,.2f})', + bar_chart.add(f'Estimated Cost High (${estimated_cost_high:,.2f})', estimated_cost_high) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_costs") From 61839d68664a0f1b06035e8dcfecffa64cd32edd Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Thu, 17 Oct 2024 14:00:37 -0400 Subject: [PATCH 49/55] added frontend and backend for average issue resolution time graph Signed-off-by: sachin-panayil --- scripts/gen_graphs.py | 31 +++++++++++++++++++++++ scripts/metricsLib/metrics_definitions.py | 10 +++++++- templates/repo_report_template.md | 2 ++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index 781ee0ff00..699e4e5be6 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -5,6 +5,7 @@ from datetime import timedelta import re import pygal +import random def percent_formatter(x): @@ -58,6 +59,7 @@ def generate_all_graphs_for_repos(all_repos): generate_predominant_languages_graph(repo) generate_language_summary_pie_chart(repo) generate_cost_estimates_bar_chart(repo) + generate_average_issue_resolution_graph(repo) try: generate_donut_graph_line_complexity_graph(repo) generate_time_xy_issue_graph( @@ -491,3 +493,32 @@ def generate_cost_estimates_bar_chart(oss_entity): write_repo_chart_to_file(oss_entity, bar_chart, "estimated_project_costs") +def generate_average_issue_resolution_graph(oss_entity): + """ + This function generates a pygal gauge chart for average issue resolution time. + + Arguments: + oss_entity: An object containing the metric data. + """ + gauge_graph = pygal.Gauge(legend_at_bottom=True) + + metric_data = oss_entity.metric_data.get('average_issue_resolution_time') + if not metric_data or not metric_data[0]: + print("No data available for average issue resolution time") + return + + data = metric_data[0] + repo_name = data[0] + average_time_str = data[1] + + days_str = average_time_str.split(' days ') + days = int(days_str[0]) + + random_numbers = [10,20,30,40,50,60] + + gauge_graph.range = [0, (days + random.choice(random_numbers))] + + gauge_graph.title = f"Average Issue Resolution Time for {repo_name} \n Average Time: {round(days)} days" + gauge_graph.add("Days", round(days)) + + write_repo_chart_to_file(oss_entity, gauge_graph, "average_issue_resolution_time") diff --git a/scripts/metricsLib/metrics_definitions.py b/scripts/metricsLib/metrics_definitions.py index b6151ecc0f..67331f702d 100644 --- a/scripts/metricsLib/metrics_definitions.py +++ b/scripts/metricsLib/metrics_definitions.py @@ -184,7 +184,6 @@ AUGUR_HOST + "/pull_request_reports/PR_time_to_first_response/" + "?repo_id={repo_id}&start_date={begin_month}&end_date={end_date}")) - ORG_GITHUB_GRAPHQL_QUERY = """ query ($org_login: String!) { organization(login: $org_login) { @@ -252,3 +251,12 @@ } ) ) + +SIMPLE_METRICS.append(ListMetric("averageIssueResolutionTime", sixMonthsParams, AUGUR_HOST + "/repos/" + "{repo_id}" + "/average-issue-resolution-time", {"average_issue_resolution_time": ["repo_name", "avg_issue_resolution_time"]})) + +# Metric for Average Commit Counts per PR +# TODO: - Currently not working because of something wrong on Augur's end. Develop a solution here (hacky) or fix upstream. + +# RESOURCE_METRICS.append(ResourceMetric("averageCommitsPerPR", sixMonthsParams, +# AUGUR_HOST + "/pull_request_reports/average_commits_per_PR/" + +# "?repo_id={repo_id}&start_date={begin_month}&end_date={end_date}")) diff --git a/templates/repo_report_template.md b/templates/repo_report_template.md index 83b2801707..8819640577 100644 --- a/templates/repo_report_template.md +++ b/templates/repo_report_template.md @@ -113,6 +113,8 @@ date_stampLastWeek: {date_stamp} {{% render "graph-toggle", baseurl: site.baseurl, name: "new-contributors" options: optionsArray, graphs: graphsArray, title: "Number of Contributors Joining per Interval" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/predominant_langs_{repo_name}_data.svg", title: "Predominant Languages" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/average_issue_resolution_time_{repo_name}_data.svg", title: "Average Issue Resolution Time" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_name}/libyear_timeline_{repo_name}_data.svg", title: "Dependency Libyears" %}} From 32be9e217b6364e5b477d54f5b96f0502e109f06 Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Thu, 17 Oct 2024 14:13:03 -0400 Subject: [PATCH 50/55] linting Signed-off-by: sachin-panayil --- scripts/gen_graphs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index b0002608de..b8b63ece44 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -4,9 +4,8 @@ import datetime from datetime import timedelta import re -import pygal import random - +import pygal def percent_formatter(x): """ @@ -569,7 +568,7 @@ def generate_people_estimate_bar_chart(oss_entity): bar_chart.add(None, [0]) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_people_contributing") - +git def generate_average_issue_resolution_graph(oss_entity): """ This function generates a pygal gauge chart for average issue resolution time. @@ -578,7 +577,7 @@ def generate_average_issue_resolution_graph(oss_entity): oss_entity: An object containing the metric data. """ gauge_graph = pygal.Gauge(legend_at_bottom=True) - + metric_data = oss_entity.metric_data.get('average_issue_resolution_time') if not metric_data or not metric_data[0]: print("No data available for average issue resolution time") From f527f109e5a6c5aa10e1da0969b87fba0ba31cbe Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Thu, 17 Oct 2024 14:17:17 -0400 Subject: [PATCH 51/55] typo correction Signed-off-by: sachin-panayil --- scripts/gen_graphs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index b8b63ece44..f9a3191557 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -568,7 +568,7 @@ def generate_people_estimate_bar_chart(oss_entity): bar_chart.add(None, [0]) write_repo_chart_to_file(oss_entity, bar_chart, "estimated_people_contributing") -git + def generate_average_issue_resolution_graph(oss_entity): """ This function generates a pygal gauge chart for average issue resolution time. From 54504da2a477291074c8d66d844bc2fe3a0711de Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Thu, 17 Oct 2024 16:51:50 -0400 Subject: [PATCH 52/55] removed randomization Signed-off-by: sachin-panayil --- scripts/gen_graphs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index f9a3191557..e1c039c508 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -590,9 +590,7 @@ def generate_average_issue_resolution_graph(oss_entity): days_str = average_time_str.split(' days ') days = int(days_str[0]) - random_numbers = [10,20,30,40,50,60] - - gauge_graph.range = [0, (days + random.choice(random_numbers))] + gauge_graph.range = [0, round((days + 20))] gauge_graph.title = f"Average Issue Resolution Time for {repo_name} \n Average Time: {round(days)} days" gauge_graph.add("Days", round(days)) From 88e00a16a1f89d279c2f947b3d819b563f51297d Mon Sep 17 00:00:00 2001 From: sachin-panayil Date: Thu, 17 Oct 2024 16:53:00 -0400 Subject: [PATCH 53/55] more linting Signed-off-by: sachin-panayil --- scripts/gen_graphs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index e1c039c508..ec90634463 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -4,7 +4,6 @@ import datetime from datetime import timedelta import re -import random import pygal def percent_formatter(x): From bb8b95f7bf76596d4b1d1544e778d46e11544e9c Mon Sep 17 00:00:00 2001 From: Sachin Panayil Date: Tue, 22 Oct 2024 11:46:57 -0400 Subject: [PATCH 54/55] included bug fixes for errors when generating graphs Signed-off-by: Sachin Panayil --- scripts/gen_graphs.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/gen_graphs.py b/scripts/gen_graphs.py index ec90634463..bcd22a2034 100644 --- a/scripts/gen_graphs.py +++ b/scripts/gen_graphs.py @@ -223,6 +223,10 @@ def generate_solid_gauge_issue_graph(oss_entity): oss_entity.metric_data['issues_count'] except ZeroDivisionError: open_issue_percent = 0 + except TypeError: + print("Repo has no issues") + return + issues_gauge.add( 'Open Issues', [{'value': open_issue_percent * 100, 'max_value': 100}]) @@ -586,8 +590,12 @@ def generate_average_issue_resolution_graph(oss_entity): repo_name = data[0] average_time_str = data[1] - days_str = average_time_str.split(' days ') - days = int(days_str[0]) + if "days" in average_time_str: + days_str = average_time_str.split(' days ') + days = int(days_str[0]) + else: + print("Average issue resolution time is less than a day") + return gauge_graph.range = [0, round((days + 20))] From 4f2330a1632475327ea3008d76f845abf35fca50 Mon Sep 17 00:00:00 2001 From: Sachin Panayil Date: Tue, 22 Oct 2024 11:56:00 -0400 Subject: [PATCH 55/55] fixed typo Signed-off-by: Sachin Panayil --- templates/org_report_template.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/org_report_template.md b/templates/org_report_template.md index dd8af25d67..6d2e04e97e 100644 --- a/templates/org_report_template.md +++ b/templates/org_report_template.md @@ -111,7 +111,7 @@ date_stampLastWeek: {date_stamp} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_owner}_new_issues_by_day_over_last_six_months.svg", title: "New Issues over Last 6 Months" %}} {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_owner}_top_committers.svg", title: "Top Committers" %}} - - {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_owner}_libyear_timeline.svg", title: "Top Committers" %}} + + {{% render "graph-section" baseurl: site.baseurl, path: "/{repo_owner}/{repo_owner}_libyear_timeline.svg", title: "Dependency Libyears" %}} \ No newline at end of file