From 66e3523b2e98f57892843d0f4654d53d8d75e61b Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 27 Dec 2023 14:35:18 -0700 Subject: [PATCH] python: cbl_report: Add yearly report generation Signed-off-by: Nathan Chancellor --- python/scripts/cbl_report.py | 349 ++++++++++++++++++++++++++++++++++- 1 file changed, 344 insertions(+), 5 deletions(-) diff --git a/python/scripts/cbl_report.py b/python/scripts/cbl_report.py index 0fed6528..db00646d 100755 --- a/python/scripts/cbl_report.py +++ b/python/scripts/cbl_report.py @@ -3,6 +3,7 @@ # Copyright (C) 2022-2023 Nathan Chancellor from argparse import ArgumentParser +import calendar import datetime import json import os @@ -85,10 +86,14 @@ def get_report_name(date): return get_month_year(date) + '-cbl-work' -def get_report_path(date): +def get_monthly_report_path(date): return Path(get_report_worktree(), 'content/posts', get_report_file(date)) +def get_yearly_report_path(year): + return Path(get_report_repo(), f"content/posts/{year}-cbl-retrospective.md") + + def get_report_repo(): return Path(os.environ['GITHUB_FOLDER'], 'hugo-files') @@ -187,6 +192,15 @@ def parse_parameters(): help='Target previous month as opposed to current month') update_parser.set_defaults(func=update_report) + yearly_parser = subparsers.add_parser('yearly', + help='Generate yearly ClangBuiltLinux retrospective') + yearly_parser.add_argument('-y', + '--year', + default=datetime.datetime.now().year, + help='Year of report (default: current year)', + type=int) + yearly_parser.set_defaults(func=yearly_report) + return parser.parse_args() @@ -241,7 +255,7 @@ def generate_item(args): raise ValueError(f"Unhandled item type ('{item_type}')") -def create_report_file(report_file, report_date): +def create_monthly_report_file(report_file, report_date): title = f"{report_date.strftime('%B %Y')} ClangBuiltLinux Work" date = report_date.strftime('%Y-%m-%dT%H:%M:%S%z') # yapf: disable @@ -393,6 +407,318 @@ def create_report_file(report_file, report_date): report_file.write_text(template, encoding='utf-8') +def get_yearly_commits(year, source, branch='main', git_log_args=None, update=True): + if update: + subprocess.run(['git', 'remote', 'update', '--prune', 'origin'], + capture_output=True, + check=True, + cwd=source) + git_log_cmd = [ + 'git', + 'log', + '--format=%H %s', + '--no-merges', + f"--since-as-filter=Jan 1, {year}", + f"--until=Jan 1, {year + 1}", + f"origin/{branch}", + ] + if git_log_args: + git_log_cmd += git_log_args + else: + git_log_cmd.append('--author=Nathan Chancellor') + git_log_output = subprocess.run(git_log_cmd, + capture_output=True, + check=True, + cwd=source, + text=True) + + return dict(item.split(' ', 1) for item in git_log_output.stdout.splitlines()) + + +def generate_html_commit_section(commits, repo): + if 'github' in repo: + commits_view = 'commit/' + elif 'gitlab' in repo: + commits_view = '-/commit/' + elif 'kernel.org' in repo: + commits_view = '' + else: + raise RuntimeError(f"Don't know how to handle repo URL: {repo}") + return ''.join([ + f'{sha[1:14]} ("{title}")
\n' + for sha, title in commits.items() + ]) + + +def create_yearly_report_file(report_file, report_date, year): + title = f"{year} ClangBuiltLinux Retrospective" + date = report_date.strftime('%Y-%m-%dT%H:%M:%S%z') + + # yapf: disable + linux_link = 'https://git.kernel.org/linus' + llvm_link = 'https://github.com/llvm/llvm-project' + boot_utils_link = 'https://github.com/ClangBuiltLinux/boot-utils' + ci_link = 'https://github.com/ClangBuiltLinux/continuous-integration2' + tc_build_link = 'https://github.com/ClangBuiltLinux/tc-build' + tuxmake_link = 'https://gitlab.com/Linaro/tuxmake' + links = { + 'boot_utils_log': f'GitHub', + 'ci_log': f'GitHub', + 'gh_org': '[our GitHub organization](https://github.com/ClangBuiltLinux)', + 'google': '[Google](https://www.google.com/)', + 'last_retro': f"[Just like I did last year](/posts/{year - 1}-cbl-retrospective/)", + 'lf': '[the Linux Foundation](https://www.linuxfoundation.org)', + 'linux_log': 'git.kernel.org', + 'llvm_log': f'GitHub', + 'sponsor': '[sponsoring my work](https://www.linuxfoundation.org/press/press-release/google-funds-linux-kernel-developers-to-focus-exclusively-on-security)', + 'tc_build_log': f'GitHub', + 'testimonial': '[which developers do appreciate](https://lore.kernel.org/YtsY5xwmlQ6kFtUz@google.com/)', + 'tuxmake': f"[TuxMake]({tuxmake_link})", + 'tuxmake_log': f'GitLab', + } + + linux_src = Path(os.environ['CBL_SRC'], 'linux-next') + linux_gyc_kwargs = { + 'year': year, + 'source': linux_src, + 'branch': 'master', + } + linux_commits = get_yearly_commits(**linux_gyc_kwargs) + linux_commit_links = generate_html_commit_section(linux_commits, linux_link) + # Updating the Linux repo is no longer necessary after the initial update + linux_gyc_kwargs['update'] = False + # We could generate linux_rep_rev_tst from the combination of the other + # three but then the commits will not be in order as they would be from git + # log, so just generate a fourth dictionary *shrugs* + linux_rep_rev_tst = get_yearly_commits(**linux_gyc_kwargs, git_log_args=['--extended-regexp', '--grep=(Report|Review|Test)ed-by: Nathan Chancellor']) + linux_rep_rev_tst_links = generate_html_commit_section(linux_rep_rev_tst, linux_link) + linux_rep = get_yearly_commits(**linux_gyc_kwargs, git_log_args=['--grep=Reported-by: Nathan Chancellor']) + linux_rev = get_yearly_commits(**linux_gyc_kwargs, git_log_args=['--grep=Reviewed-by: Nathan Chancellor']) + linux_tst = get_yearly_commits(**linux_gyc_kwargs, git_log_args=['--grep=Tested-by: Nathan Chancellor']) + + llvm_src = Path(os.environ['CBL_SRC'], 'llvm-project') + llvm_commits = get_yearly_commits(year, llvm_src) + llvm_links = generate_html_commit_section(llvm_commits, llvm_link) + + boot_utils_src = Path(os.environ['CBL_GIT'], 'boot-utils') + boot_utils_commits = get_yearly_commits(year, boot_utils_src) + boot_utils_links = generate_html_commit_section(boot_utils_commits, boot_utils_link) + + ci_src = Path(os.environ['CBL_GIT'], 'continuous-integration2') + ci_commits = get_yearly_commits(year, ci_src) + ci_links = generate_html_commit_section(ci_commits, ci_link) + + tc_build_src = Path(os.environ['CBL_GIT'], 'tc-build') + tc_build_commits = get_yearly_commits(year, tc_build_src) + tc_build_links = generate_html_commit_section(tc_build_commits, tc_build_link) + + tuxmake_src = Path(os.environ['CBL_SRC'], 'tuxmake') + tuxmake_commits = get_yearly_commits(year, tuxmake_src, branch='master') + tuxmake_links = generate_html_commit_section(tuxmake_commits, tuxmake_link) + + report_links = [f"- [{month} {year}](/posts/{month.lower()}-{year}-cbl-work/)" for month in calendar.month_name if month] + + # Yes, this is Markdown in Python :) + template = ( + '---' '\n' + f"title: {title}" '\n' + f"date: {date}" '\n' + 'toc: false' '\n' + 'images:' '\n' + 'tags:' '\n' + ' - clangbuiltlinux' '\n' + ' - linux' '\n' + ' - linuxfoundation' '\n' + ' - maintainer' '\n' + '---' '\n' + '\n' + f"{links['last_retro']}, I want to do a yearly report/retrospective" + f" for {year} to look at what I (and the whole ClangBuiltLinux team" + ' in some cases) accomplished. I do monthly reports but looking at ' + 'a high level across the year helps put things into perspective and' + ' drive improvements going into the new year.' '\n' + '\n' + '## Linux kernel' '\n' + '\n' + f"This year, I had {len(linux_commits)} commits accepted into " + f"maintainer trees (not all will be merged into mainline in {year} " + f"but they were written and added in maintainer trees in {year}). " + 'They can be viewed on the web or by running the following command ' + 'in an up-to-date Linux repository locally:' '\n' + '\n' + '```' '\n' + '$ git log \\' '\n' + ' --author=\'Nathan Chancellor\' \\' '\n' + ' --oneline \\' '\n' + f" --since-as-filter='Jan 1, {year}' \\" '\n' + f" --until='Jan 1, {year + 1}' \\" '\n' + ' origin/master' '\n' + '```' '\n' + '\n' + 'A similar command will be used to generate all following commit ' + 'logs, which are included for convenience behind some collapsible ' + 'Markdown with links.' '\n' + '\n' + '
' '\n' + '' + f"Kernel contributions in {year} ({links['linux_log']})" + '' '\n' + '

' '\n' + f"{linux_commit_links}" + '

' '\n' + '

' '\n' + '\n' + '' '\n' + '\n' + 'It is important to keep in mind that sending patches is only part' + ' of the development process. The others are reporting problems and' + ' testing and reviewing solutions to those problems. The kernel ' + 'keeps track of these through particular tags: `Reported-by`, ' + f"`Reviewed-by`, and `Tested-by`. In {year}, I provided those tags " + f"on {len(linux_rep_rev_tst)} patches. The break down of patches " + 'that contained:' '\n' + '\n' + f"- `Reported-by`: {len(linux_rep)}" '\n' + f"- `Reviewed-by`: {len(linux_rev)}" '\n' + f"- `Tested-by`: {len(linux_tst)}" '\n' + '\n' + 'A full list of those commits are below, generated with the ' + 'following command in an up-to-date Linux checkout:' '\n' + '\n' + '```' '\n' + '$ git log \\' '\n' + ' --extended-regexp \\' '\n' + ' --grep=\'(Report|Review|Test)ed-by: Nathan Chancellor\' \\' '\n' + ' --oneline \\' '\n' + f" --since-as-filter='Jan 1, {year}' \\" '\n' + f" --until='Jan 1, {year + 1}' \\" '\n' + ' origin/master' '\n' + '```' '\n' + '\n' + '
' '\n' + '' + f"Reported-by, Reviewed-by, and Tested-by in {year}" + '' '\n' + '

' '\n' + f"{linux_rep_rev_tst_links}" + '

' '\n' + '

' '\n' + '\n' + '\n' + '## LLVM' '\n' + '\n' + 'I am far from a large LLVM contributor but I do have occasional ' + 'patches there as part of this work. This year, I had ' + f"{len(llvm_commits)} patches to LLVM. " '\n' + '\n' + '
' '\n' + '' + f"LLVM contributions in {year} ({links['llvm_log']})" + '' '\n' + '

' '\n' + f"{llvm_links}" + '

' '\n' + '

' '\n' + '\n' + '\n' + '## Tooling' '\n' + '\n' + f"We have a few different repositories in {links['gh_org']} that we" + ' use for testing and development, which I call "tooling". Tooling ' + 'is very important for repetitive tasks or tasks where you want to ' + 'take the human out of the equation so that mistakes are less ' + 'likely, such as a bisect. Additionally, there are some other ' + f"repositories that we rely on, like {links['tuxmake']}, that I " + 'consistently contribute to.' '\n' + '\n' + '' '\n' + '\n' + 'Like previously, I have included the `git log` output with direct ' + 'links to commits, along with a web link to browse the history ' + 'there.' '\n' + '\n' + '
' '\n' + f"boot-utils ({links['boot_utils_log']})" '\n' + '

' '\n' + f"{boot_utils_links}" + '

' '\n' + '

' '\n' + '\n' + '
' '\n' + f"continuous-integration2 ({links['ci_log']})" '\n' + '

' '\n' + f"{ci_links}" + '

' '\n' + '

' '\n' + '\n' + '
' '\n' + f"tc-build ({links['tc_build_log']})" '\n' + '

' '\n' + f"{tc_build_links}" + '

' '\n' + '

' '\n' + '\n' + '
' '\n' + f"TuxMake ({links['tuxmake_log']})" '\n' + '

' '\n' + f"{tuxmake_links}" + '

' '\n' + '

' '\n' + '\n' + '## Behind the scenes' '\n' + '\n' + 'There are always things that require time but do not always show ' + 'tangible results. There are three things that I think fall under ' + 'this category:' '\n' + '\n' + '- __Issue tracker management:__ Keeping a clean and accurate issue' + ' tracker is critical for few reason.' '\n' + ' 1. It gives a good high level overview of the "health" of the ' + 'project. We want our issue tracker to be an accurate ' + 'representation of how much help we need (since we always need ' + 'it...)' '\n' + ' 2. It helps assign priority to certain issues. If we have a lot ' + 'of open but resolved issues, it can be hard to decide what needs ' + 'to be worked on next.' '\n' + ' 3. The issue tracker is a wonderful historical reference. We use' + ' the issue tracker to keep track of mailing list posts and such so' + ' it is important that those links are as acccurate as possible and' + ' have as much information as possible in case we have to look back' + ' five years later to wonder why we did something the way that we ' + 'did.' '\n' + "- __Mailing list reading:__ We are not Cc'd on every issue related" + " to `clang`, even though sometimes it is the compiler's problem or" + ' a known difference between the toolchains that we have already ' + 'figured out. By monitoring the mailing list for certain phrases, ' + 'we can provide assistance without being initially notified, ' + f"{links['testimonial']}."'\n' + '- __Hardware testing:__ Every linux-next release, I built and boot' + ' kernels on a variety of hardware to test for issues, as some ' + 'problems only show up on bare metal or with a full distribution. ' + 'I wrote a script to drive a full distribution QEMU on a variety of' + ' architectures to make some of that testing and debugging easier ' + 'but bare metal is always an important testing target, since that ' + 'is where the kernel will run the majority of the time.' '\n' + '\n' + '\n' + '## Special thanks' '\n' + '\n' + f"Special thanks to {links['google']} and {links['lf']} for " + f"{links['sponsor']}. I am in a very fortunate position thanks to " + 'the work of many great and suppportive folks at those ' + 'organizations and I look forward to continuing to contribute under' + f" this umbrella for {year + 1}!" '\n' + '\n' + 'To view individual monthly reports, click on one of the links ' + 'below:' '\n' + '\n' + + '\n'.join(report_links) + '\n' + ) + # yapf: enable + + report_file.write_text(template, encoding='utf-8') + + def finalize_report(args): # Get the source and destination paths and branch based on current time repo = get_report_repo() @@ -475,13 +801,13 @@ def new_report(args): if not worktree.exists(): raise RuntimeError(f"{worktree} does not exist when creating report file?") - report = get_report_path(date) + report = get_monthly_report_path(date) if not report.exists(): report_date = get_initial_report_date() commit_title = f"content/posts: ClangBuiltLinux work in {report_date.strftime('%B')} {report_date.year}" commit_date = report_date.strftime('%a %b %d %H:%M:%S %Y %z') - create_report_file(report, report_date) + create_monthly_report_file(report, report_date) git(worktree, ['add', report]) git(worktree, ['commit', '-m', commit_title, '--date', commit_date]) @@ -493,7 +819,7 @@ def update_report(args): # Get branch based on user's request date = get_prev_datetime() if args.prev_month else get_current_datetime() - if not (report := get_report_path(date)).exists(): + if not (report := get_monthly_report_path(date)).exists(): raise RuntimeError(f"{report} does not exist when updating?") if args.edit or args.all: @@ -509,6 +835,19 @@ def update_report(args): git(worktree, ['push']) +def yearly_report(args): + repo = get_report_repo() + report = get_yearly_report_path(args.year) + if not report.exists(): + report_date = get_initial_report_date() + commit_title = f"content/posts: {args.year} ClangBuiltLinux retrospective" + commit_date = report_date.strftime('%a %b %d %H:%M:%S %Y %z') + + create_yearly_report_file(report, report_date, args.year) + git(repo, ['add', report]) + git(repo, ['commit', '-m', commit_title, '--date', commit_date]) + + if __name__ == '__main__': main_args = parse_parameters() main_args.func(main_args)