diff --git a/bin/daily b/bin/daily index b4ea339958..e6e5304654 100755 --- a/bin/daily +++ b/bin/daily @@ -5,7 +5,6 @@ # This script is expected to be triggered by cron from # /etc/cron.d/datatracker export LANG=en_US.UTF-8 -export PYTHONIOENCODING=utf-8 # Make sure we stop if something goes wrong: program=${0##*/} @@ -17,10 +16,6 @@ cd $DTDIR/ logger -p user.info -t cron "Running $DTDIR/bin/daily" -# Set up the virtual environment -source $DTDIR/env/bin/activate - - # Get IANA-registered yang models #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR') # Hardcode the rsync target to avoid any unwanted deletes: @@ -30,9 +25,3 @@ rsync -avzq --delete /a/www/ietf-ftp/iana/yang-parameters/ /a/www/ietf-ftp/yang/ # Get Yang models from Yangcatalog. #rsync -avzq rsync://rsync.yangcatalog.org:10873/yangdeps /a/www/ietf-ftp/yang/catalogmod/ /a/www/ietf-datatracker/scripts/sync_to_yangcatalog - -# Populate the yang repositories -$DTDIR/ietf/manage.py populate_yang_model_dirs -v0 - -# Re-run yang checks on active documents -$DTDIR/ietf/manage.py run_yang_model_checks -v0 diff --git a/ietf/submit/tasks.py b/ietf/submit/tasks.py index 9a13268bce..9e279fa9f0 100644 --- a/ietf/submit/tasks.py +++ b/ietf/submit/tasks.py @@ -10,7 +10,8 @@ from ietf.submit.models import Submission from ietf.submit.utils import (cancel_submission, create_submission_event, process_uploaded_submission, - process_and_accept_uploaded_submission) + process_and_accept_uploaded_submission, run_all_yang_model_checks, + populate_yang_model_dirs) from ietf.utils import log @@ -66,6 +67,12 @@ def cancel_stale_submissions(): create_submission_event(None, subm, 'Submission canceled: expired without being posted') +@shared_task +def run_yang_model_checks_task(): + populate_yang_model_dirs() + run_all_yang_model_checks() + + @shared_task(bind=True) def poke(self): log.log(f'Poked {self.name}, request id {self.request.id}') diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 618f237e4d..b48168f8a6 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -49,6 +49,7 @@ from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task +from ietf.submit.utils import apply_yang_checker_to_draft, run_all_yang_model_checks from ietf.utils import tool_version from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, get_payload_text @@ -3487,3 +3488,28 @@ def test_submission_checks(self): "Your Internet-Draft failed at least one submission check.", status_code=200, ) + + +class YangCheckerTests(TestCase): + @mock.patch("ietf.submit.utils.apply_yang_checker_to_draft") + def test_run_all_yang_model_checks(self, mock_apply): + active_drafts = WgDraftFactory.create_batch(3) + WgDraftFactory(states=[("draft", "expired")]) + run_all_yang_model_checks() + self.assertEqual(mock_apply.call_count, 3) + self.assertCountEqual( + [args[0][1] for args in mock_apply.call_args_list], + active_drafts, + ) + + def test_apply_yang_checker_to_draft(self): + draft = WgDraftFactory() + submission = SubmissionFactory(name=draft.name, rev=draft.rev) + submission.checks.create(checker="my-checker") + checker = mock.Mock() + checker.name = "my-checker" + checker.symbol = "X" + checker.check_file_txt.return_value = (True, "whee", None, None, {}) + apply_yang_checker_to_draft(checker, draft) + self.assertEqual(checker.check_file_txt.call_args, mock.call(draft.get_file_name())) + diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 770352b4cd..c814b84657 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -4,9 +4,11 @@ import datetime import io +import json import os import pathlib import re +import sys import time import traceback import xml2rfc @@ -15,6 +17,7 @@ from shutil import move from typing import Optional, Union # pyflakes:ignore from unidecode import unidecode +from xym import xym from django.conf import settings from django.core.exceptions import ValidationError @@ -43,6 +46,7 @@ from ietf.community.utils import update_name_contains_indexes_with_new_doc from ietf.submit.mail import ( announce_to_lists, announce_new_version, announce_to_authors, send_approval_request, send_submission_confirmation, announce_new_wg_00, send_manual_post_request ) +from ietf.submit.checkers import DraftYangChecker from ietf.submit.models import ( Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName, SubmissionCheck, SubmissionExtResource ) from ietf.utils import log @@ -1431,3 +1435,133 @@ def process_uploaded_submission(submission): submission.state_id = "uploaded" submission.save() create_submission_event(None, submission, desc="Completed submission validation checks") + + +def apply_yang_checker_to_draft(checker, draft): + submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first() + if submission: + check = submission.checks.filter(checker=checker.name).order_by('-id').first() + if check: + result = checker.check_file_txt(draft.get_file_name()) + passed, message, errors, warnings, items = result + items = json.loads(json.dumps(items)) + new_res = (passed, errors, warnings, message) + old_res = (check.passed, check.errors, check.warnings, check.message) if check else () + if new_res != old_res: + log.log(f"Saving new yang checker results for {draft.name}-{draft.rev}") + qs = submission.checks.filter(checker=checker.name).order_by('time') + submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete() + submission.checks.create(submission=submission, checker=checker.name, passed=passed, + message=message, errors=errors, warnings=warnings, items=items, + symbol=checker.symbol) + else: + log.log(f"Could not run yang checker for {draft.name}-{draft.rev}: missing submission object") + + +def run_all_yang_model_checks(): + checker = DraftYangChecker() + for draft in Document.objects.filter( + type_id="draft", + states=State.objects.get(type="draft", slug="active"), + ): + apply_yang_checker_to_draft(checker, draft) + + +def populate_yang_model_dirs(): + """Update the yang model dirs + + * All yang modules from published RFCs should be extracted and be + available in an rfc-yang repository. + + * All valid yang modules from active, not replaced, Internet-Drafts + should be extracted and be available in a draft-valid-yang repository. + + * All, valid and invalid, yang modules from active, not replaced, + Internet-Drafts should be available in a draft-all-yang repository. + (Actually, given precedence ordering, it would be enough to place + non-validating modules in a draft-invalid-yang repository instead). + + * In all cases, example modules should be excluded. + + * Precedence is established by the search order of the repository as + provided to pyang. + + * As drafts expire, models should be removed in order to catch cases + where a module being worked on depends on one which has slipped out + of the work queue. + + """ + def extract_from(file, dir, strict=True): + saved_stdout = sys.stdout + saved_stderr = sys.stderr + xymerr = io.StringIO() + xymout = io.StringIO() + sys.stderr = xymerr + sys.stdout = xymout + model_list = [] + try: + model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=-2) + for name in model_list: + modfile = moddir / name + mtime = file.stat().st_mtime + os.utime(str(modfile), (mtime, mtime)) + if '"' in name: + name = name.replace('"', '') + modfile.rename(str(moddir / name)) + model_list = [n.replace('"', '') for n in model_list] + except Exception as e: + log.log("Error when extracting from %s: %s" % (file, str(e))) + finally: + sys.stdout = saved_stdout + sys.stderr = saved_stderr + return model_list + + # Extract from new RFCs + + rfcdir = Path(settings.RFC_PATH) + + moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR) + if not moddir.exists(): + moddir.mkdir(parents=True) + + latest = 0 + for item in moddir.iterdir(): + if item.stat().st_mtime > latest: + latest = item.stat().st_mtime + + log.log(f"Extracting RFC Yang models to {moddir} ...") + for item in rfcdir.iterdir(): + if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit(): + if item.stat().st_mtime > latest: + model_list = extract_from(item, moddir) + for name in model_list: + if not (name.startswith('ietf') or name.startswith('iana')): + modfile = moddir / name + modfile.unlink() + + # Extract valid modules from drafts + + six_months_ago = time.time() - 6 * 31 * 24 * 60 * 60 + + def active(dirent): + return dirent.stat().st_mtime > six_months_ago + + draftdir = Path(settings.INTERNET_DRAFT_PATH) + moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR) + if not moddir.exists(): + moddir.mkdir(parents=True) + log.log(f"Emptying {moddir} ...") + for item in moddir.iterdir(): + item.unlink() + + log.log(f"Extracting draft Yang models to {moddir} ...") + for item in draftdir.iterdir(): + try: + if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): + model_list = extract_from(item, moddir, strict=False) + for name in model_list: + if name.startswith('example'): + modfile = moddir / name + modfile.unlink() + except UnicodeDecodeError as e: + log.log(f"Error processing {item.name}: {e}") diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index 76d362bb24..2d34f8361c 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -273,6 +273,16 @@ def create_default_tasks(self): ), ) + PeriodicTask.objects.get_or_create( + name="Run Yang model checks", + task="ietf.submit.tasks.run_yang_model_checks_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["daily"], + description="Re-run Yang model checks on all active drafts", + ), + ) + def show_tasks(self): for label, crontab in self.crontabs.items(): tasks = PeriodicTask.objects.filter(crontab=crontab).order_by( diff --git a/ietf/utils/management/commands/populate_yang_model_dirs.py b/ietf/utils/management/commands/populate_yang_model_dirs.py deleted file mode 100644 index 864dfafb72..0000000000 --- a/ietf/utils/management/commands/populate_yang_model_dirs.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import io -import os -import sys -import time - -from pathlib import Path -from textwrap import dedent -from xym import xym - -from django.conf import settings -from django.core.management.base import BaseCommand - -import debug # pyflakes:ignore - -class Command(BaseCommand): - """ - Populate the yang module repositories from drafts and RFCs. - - Extracts yang models from RFCs (found in settings.RFC_PATH and places - them in settings.SUBMIT_YANG_RFC_MODEL_DIR, and from active drafts, placed in - settings.SUBMIT_YANG_DRAFT_MODEL_DIR. - - """ - - help = dedent(__doc__).strip() - - def add_arguments(self, parser): - parser.add_argument('--clean', - action='store_true', dest='clean', default=False, - help='Remove the current directory content before writing new models.') - - - def handle(self, *filenames, **options): - """ - - * All yang modules from published RFCs should be extracted and be - available in an rfc-yang repository. - - * All valid yang modules from active, not replaced, Internet-Drafts - should be extracted and be available in a draft-valid-yang repository. - - * All, valid and invalid, yang modules from active, not replaced, - Internet-Drafts should be available in a draft-all-yang repository. - (Actually, given precedence ordering, it would be enough to place - non-validating modules in a draft-invalid-yang repository instead). - - * In all cases, example modules should be excluded. - - * Precedence is established by the search order of the repository as - provided to pyang. - - * As drafts expire, models should be removed in order to catch cases - where a module being worked on depends on one which has slipped out - of the work queue. - - """ - - verbosity = int(options.get('verbosity')) - - def extract_from(file, dir, strict=True): - saved_stdout = sys.stdout - saved_stderr = sys.stderr - xymerr = io.StringIO() - xymout = io.StringIO() - sys.stderr = xymerr - sys.stdout = xymout - model_list = [] - try: - model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=verbosity-2) - for name in model_list: - modfile = moddir / name - mtime = file.stat().st_mtime - os.utime(str(modfile), (mtime, mtime)) - if '"' in name: - name = name.replace('"', '') - modfile.rename(str(moddir/name)) - model_list = [ n.replace('"','') for n in model_list ] - except Exception as e: - self.stderr.write("** Error when extracting from %s: %s" % (file, str(e))) - finally: - sys.stdout = saved_stdout - sys.stderr = saved_stderr - # - if verbosity > 1: - outmsg = xymout.getvalue() - if outmsg.strip(): - self.stdout.write(outmsg) - if verbosity>2: - errmsg = xymerr.getvalue() - if errmsg.strip(): - self.stderr.write(errmsg) - return model_list - - # Extract from new RFCs - - rfcdir = Path(settings.RFC_PATH) - - moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR) - if not moddir.exists(): - moddir.mkdir(parents=True) - - latest = 0 - for item in moddir.iterdir(): - if item.stat().st_mtime > latest: - latest = item.stat().st_mtime - - if verbosity > 0: - self.stdout.write("Extracting to %s ..." % moddir) - for item in rfcdir.iterdir(): - if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit(): - if item.stat().st_mtime > latest: - model_list = extract_from(item, moddir) - for name in model_list: - if name.startswith('ietf') or name.startswith('iana'): - if verbosity > 1: - self.stdout.write(" Extracted from %s: %s" % (item, name)) - elif verbosity > 0: - self.stdout.write('.', ending='') - self.stdout.flush() - else: - modfile = moddir / name - modfile.unlink() - if verbosity > 1: - self.stdout.write(" Skipped module from %s: %s" % (item, name)) - if verbosity > 0: - self.stdout.write("") - - # Extract valid modules from drafts - - six_months_ago = time.time() - 6*31*24*60*60 - def active(item): - return item.stat().st_mtime > six_months_ago - - draftdir = Path(settings.INTERNET_DRAFT_PATH) - - moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR) - if not moddir.exists(): - moddir.mkdir(parents=True) - if verbosity > 0: - self.stdout.write("Emptying %s ..." % moddir) - for item in moddir.iterdir(): - item.unlink() - - if verbosity > 0: - self.stdout.write("Extracting to %s ..." % moddir) - for item in draftdir.iterdir(): - try: - if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): - model_list = extract_from(item, moddir, strict=False) - for name in model_list: - if not name.startswith('example'): - if verbosity > 1: - self.stdout.write(" Extracted module from %s: %s" % (item, name)) - elif verbosity > 0: - self.stdout.write('.', ending='') - self.stdout.flush() - else: - modfile = moddir / name - modfile.unlink() - if verbosity > 1: - self.stdout.write(" Skipped module from %s: %s" % (item, name)) - except UnicodeDecodeError as e: - self.stderr.write('\nError: %s' % (e, )) - self.stderr.write(item.name) - self.stderr.write('') - if verbosity > 0: - self.stdout.write('') - diff --git a/ietf/utils/management/commands/run_yang_model_checks.py b/ietf/utils/management/commands/run_yang_model_checks.py deleted file mode 100644 index 7e2f7165b0..0000000000 --- a/ietf/utils/management/commands/run_yang_model_checks.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright The IETF Trust 2017-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import json - -from textwrap import dedent - -from django.core.management.base import BaseCommand - -import debug # pyflakes:ignore - -from ietf.doc.models import Document, State -from ietf.submit.models import Submission -from ietf.submit.checkers import DraftYangChecker - - -class Command(BaseCommand): - """ - Run yang model checks on active drafts. - - Repeats the yang checks in ietf/submit/checkers.py for active drafts, in - order to catch changes in status due to new modules becoming available in - the module directories. - - """ - - help = dedent(__doc__).strip() - - def add_arguments(self, parser): - parser.add_argument('draftnames', nargs="*", help="drafts to check, or none to check all active yang drafts") - parser.add_argument('--clean', - action='store_true', dest='clean', default=False, - help='Remove the current directory content before writing new models.') - - - def check_yang(self, checker, draft, force=False): - if self.verbosity > 1: - self.stdout.write("Checking %s-%s" % (draft.name, draft.rev)) - elif self.verbosity > 0: - self.stderr.write('.', ending='') - submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first() - if submission or force: - check = submission.checks.filter(checker=checker.name).order_by('-id').first() - if check or force: - result = checker.check_file_txt(draft.get_file_name()) - passed, message, errors, warnings, items = result - if self.verbosity > 2: - self.stdout.write(" Errors: %s\n" - " Warnings: %s\n" - " Message:\n%s\n" % (errors, warnings, message)) - items = json.loads(json.dumps(items)) - new_res = (passed, errors, warnings, message) - old_res = (check.passed, check.errors, check.warnings, check.message) if check else () - if new_res != old_res: - if self.verbosity > 1: - self.stdout.write(" Saving new yang checker results for %s-%s" % (draft.name, draft.rev)) - qs = submission.checks.filter(checker=checker.name).order_by('time') - submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete() - submission.checks.create(submission=submission, checker=checker.name, passed=passed, - message=message, errors=errors, warnings=warnings, items=items, - symbol=checker.symbol) - else: - self.stderr.write("Error: did not find any submission object for %s-%s" % (draft.name, draft.rev)) - - def handle(self, *filenames, **options): - """ - """ - - self.verbosity = int(options.get('verbosity')) - drafts = options.get('draftnames') - - active_state = State.objects.get(type="draft", slug="active") - - checker = DraftYangChecker() - if drafts: - for name in drafts: - parts = name.rsplit('-',1) - if len(parts)==2 and len(parts[1])==2 and parts[1].isdigit(): - name = parts[0] - draft = Document.objects.get(name=name) - self.check_yang(checker, draft, force=True) - else: - for draft in Document.objects.filter(states=active_state, type_id='draft'): - self.check_yang(checker, draft)