Merge branch 'master' of https://github.com/djbrown/hbscorez

src/base/management/commands/correct_data.py

@@ -1,6 +1,6 @@
 import datetime
 import logging
-from typing import Any, Dict
+from typing import Any
 
 from django.core.management import BaseCommand
 from django.db import transaction
@@ -11,7 +11,7 @@
 from games.models import Game, Team
 from players.models import Player, Score
 
-LOGGER = logging.getLogger('hbscorez.command')
+LOGGER = logging.getLogger('hbscorez')
 
 
 class Command(BaseCommand):
@@ -49,7 +49,7 @@ def rename_player(team_bhv_id, old_name, new_name):
 
 
 def _score(player_number: int, goals: int = 0, penalty_tries: int = 0,
-           penalty_goals: int = 0, **kwargs) -> Dict[str, Any]:
+           penalty_goals: int = 0, **kwargs) -> dict[str, Any]:
     return {"player_number": player_number, "goals": goals, "penalty_tries": penalty_tries,
             "penalty_goals": penalty_goals, **kwargs}
 
@@ -59,8 +59,8 @@ def time(minutes: int, seconds: int = 0):
 
 
 @transaction.atomic
-def add_scores(league__bhv_id: int, game_number: int, home_score_data: Dict[str, Dict[str, Any]],
-               guest_score_data: Dict[str, Dict[str, Any]]):
+def add_scores(league__bhv_id: int, game_number: int, home_score_data: dict[str, dict[str, Any]],
+               guest_score_data: dict[str, dict[str, Any]]):
     LOGGER.info('add Scores %s %s', league__bhv_id, game_number)
     try:
         game = Game.objects.get(league__bhv_id=league__bhv_id, number=game_number)
@@ -73,7 +73,7 @@ def add_scores(league__bhv_id: int, game_number: int, home_score_data: Dict[str,
         LOGGER.warning('skip Game (not found): %s %s', league__bhv_id, game_number)
 
 
-def _add_scores(game: Game, team: Team, scores_data: Dict[str, Dict[str, Any]]):
+def _add_scores(game: Game, team: Team, scores_data: dict[str, dict[str, Any]]):
     for name, score_data in scores_data.items():
         player = Player(name=name, team=team)
         sco = Score(player=player, game=game, **score_data)

src/base/parsing.py

@@ -1,78 +1,81 @@
 import locale
 import re
 from datetime import datetime, timedelta
-from typing import Match, Optional, Tuple
+from typing import cast
 from urllib.parse import parse_qs, urlsplit
 
 from lxml import html
+from lxml.etree import _Element
 
 from games.models import SportsHall
+from teams.models import Team
 
 
-def html_dom(html_text: str):
+def html_dom(html_text: str) -> _Element:
     return html.fromstring(html_text)
 
 
-def parse_association_urls(dom, root_url):
-    items = dom.xpath('//div[@id="main-content"]//table[@summary]/tbody/tr/td[1]/a/@href')
+def parse_association_urls(dom: _Element, root_url: str) -> list[str]:
+    items = cast(list[str], dom.xpath('//div[@id="main-content"]//table[@summary]/tbody/tr/td[1]/a/@href'))
     return [item if item.startswith('http') else root_url + item for item in items]
 
 
-def parse_association_bhv_id_from_dom(dom):
-    [bhv_id] = dom.xpath('//div[@id="app"]/@data-og-id')
+def parse_association_bhv_id_from_dom(dom: _Element) -> int:
+    [bhv_id] = cast(list[str], dom.xpath('//div[@id="app"]/@data-og-id'))
     return int(bhv_id)
 
 
-def parse_link_query_item(link, query_key):
+def parse_link_query_item(link: _Element, query_key: str) -> str:
     href = link.get('href')
-    query = urlsplit(href).query
+    query = cast(str, urlsplit(href).query)
     return parse_qs(query)[query_key][0]
 
 
-def parse_association_bhv_id(link):
+def parse_association_bhv_id(link: _Element) -> int:
     return int(parse_link_query_item(link, 'orgGrpID'))
 
 
-def parse_association_name(dom):
-    return dom.xpath('//*[@id="results"]/div/h1/text()[2]')[0]
+def parse_association_name(dom: _Element) -> str:
+    return cast(list[str], dom.xpath('//*[@id="results"]/div/h1/text()[2]'))[0]
 
 
-def parse_district_items(dom):
-    return dom.xpath('//select[@name="orgID"]/option[position()>1]')
+def parse_district_items(dom: _Element) -> list[_Element]:
+    return cast(list[_Element], dom.xpath('//select[@name="orgID"]/option[position()>1]'))
 
 
-def parse_district_link_date(link):
+def parse_district_link_date(link: _Element) -> str:
     return parse_link_query_item(link, 'do')
 
 
-def parse_league_links(dom):
-    return dom.xpath('//div[@id="results"]/div/table[2]/tr/td[1]/a')
+def parse_league_links(dom: _Element) -> list[_Element]:
+    return cast(list[_Element], dom.xpath('//div[@id="results"]/div/table[2]/tr/td[1]/a'))
 
 
-def parse_league_bhv_id(link):
+def parse_league_bhv_id(link: _Element) -> int:
     return int(parse_link_query_item(link, 'score'))
 
 
-def parse_district_season_start_year(district_season_heading):
+def parse_district_season_start_year(district_season_heading: str) -> int | None:
     matches = re.match(r"Halle(?:nrunde)? (\d{4})/(\d{4})", district_season_heading)
     return int(matches.group(1)) if matches else None
 
 
-def parse_league_name(dom):
-    heading = dom.xpath('//*[@id="results"]/div/h1/text()[2]')[0]
+def parse_league_name(dom: _Element) -> str:
+    heading = cast(list[str], dom.xpath('//*[@id="results"]/div/h1/text()[2]'))[0]
     return heading.rsplit(' - ', 1)[0]
 
 
-def parse_team_links(dom):
-    return dom.xpath('//table[@class="scoretable"]/tr[position() > 1]/td[3]/a') or \
-        dom.xpath('//table[@class="scoretable"]/tr[position() > 1]/td[2]/a')
+def parse_team_links(dom: _Element) -> list[_Element]:
+    return cast(list[_Element], dom.xpath('//table[@class="scoretable"]/tr[position() > 1]/td[3]/a')) or \
+        cast(list[_Element], dom.xpath('//table[@class="scoretable"]/tr[position() > 1]/td[2]/a'))
 
 
-def parse_retirements(dom):
+def parse_retirements(dom: _Element) -> list[tuple[str, datetime]]:
     retirements = []
-    paragraphs = dom.xpath('//table[@class="scoretable"]/following::div[following::table[@class="gametable"]]')
+    paragraphs = cast(list[html.HtmlMixin], dom.xpath(
+        '//table[@class="scoretable"]/following::div[following::table[@class="gametable"]]'))
     for paragraph in paragraphs:
-        text = paragraph.text_content()
+        text: str = cast(str, paragraph.text_content())
         matches = re.match(r"(?:Der|Die) (.*) hat.* ([0123]\d\.[012]\d\.\d{2,4}).* zurückgezogen.*", text)
         if matches:
             team_name = matches.group(1)
@@ -90,26 +93,26 @@ def date_from_text(text):
     return retirements
 
 
-def parse_team_bhv_id(link):
+def parse_team_bhv_id(link: _Element) -> int:
     return int(parse_link_query_item(link, 'teamID'))
 
 
-def parse_team_names(text: str) -> Tuple[str, str]:
-    match: Optional[Match[str]] = re.match(r"(.+) - (.+)", text)
+def parse_team_names(text: str) -> tuple[str, str]:
+    match: re.Match[str] | None = re.match(r"(.+) - (.+)", text)
     if match:
         return match.group(1), match.group(2)
     raise ValueError(f'invalid team names: {text}')
 
 
-def parse_game_rows(dom):
-    return dom.xpath('//table[@class="gametable"]/tr[position() > 1]')
+def parse_game_rows(dom: _Element) -> list[_Element]:
+    return cast(list[_Element], dom.xpath('//table[@class="gametable"]/tr[position() > 1]'))
 
 
-def parse_team_short_names(game_rows):
-    return [c.text for game_row in game_rows for c in game_row.xpath('td')[4:7:2]]
+def parse_team_short_names(game_rows: list[_Element]) -> list[str | None]:
+    return [c.text for game_row in game_rows for c in cast(list[_Element], game_row.xpath('td'))[4:7:2]]
 
 
-def parse_opening_whistle(text: str) -> Optional[datetime]:
+def parse_opening_whistle(text: str) -> datetime | None:
     if not text.strip():
         return None
     locale.setlocale(locale.LC_ALL, "de_DE.UTF-8")
@@ -120,10 +123,10 @@ def parse_opening_whistle(text: str) -> Optional[datetime]:
     raise ValueError(f'invalid opening whistle: {text}')
 
 
-def parse_sports_hall(dom) -> SportsHall:
-    table = dom.xpath('//table[@class="gym"]')[0]
-    name = table[0][1][0].text
-    city = table[1][1].text
+def parse_sports_hall(dom: _Element) -> SportsHall:
+    table = cast(list[_Element], dom.xpath('//table[@class="gym"]'))[0]
+    name = cast(str, table[0][1][0].text)
+    city = cast(str, table[1][1].text)
     street = table[2][1].text
     address = street + ", " + city if street else city
     phone_number = table[3][1].text
@@ -135,21 +138,21 @@ def parse_sports_hall(dom) -> SportsHall:
                       longitude=longitude)
 
 
-def parse_sports_hall_bhv_id(link):
+def parse_sports_hall_bhv_id(link: _Element) -> int:
     return int(parse_link_query_item(link, 'gymID'))
 
 
-def parse_coordinates(dom) -> Tuple[Optional[str], Optional[str]]:
-    map_scripts = dom.xpath('//script[contains(text(),"new mxn.LatLonPoint")]')
-    if not map_scripts:
+def parse_coordinates(dom: _Element) -> tuple[str | None, str | None]:
+    map_scripts = cast(list[_Element], dom.xpath('//script[contains(text(),"new mxn.LatLonPoint")]'))
+    if not map_scripts or not map_scripts[0].text:
         return (None, None)
     match = re.search(r"new mxn.LatLonPoint\(([.0-9]+),([.0-9]+)\)", map_scripts[0].text)
     if match:
         return match.group(1), match.group(2)
     raise ValueError(f'coordinates not found: {map_scripts}')
 
 
-def parse_goals(game_row) -> Tuple[Optional[int], Optional[int]]:
+def parse_goals(game_row: _Element) -> tuple[int | None, int | None]:
     home_goals = game_row[7].text
     guest_goals = game_row[9].text
     if home_goals and guest_goals:
@@ -167,14 +170,14 @@ def parse_goals(game_row) -> Tuple[Optional[int], Optional[int]]:
     return (None, None)
 
 
-def parse_report_number(cell):
+def parse_report_number(cell: _Element) -> int | None:
     if len(cell) >= 1 and cell[0].text == 'PI':
         return int(parse_link_query_item(cell[0], 'sGID'))
 
     return None
 
 
-def parse_forfeiting_team(cell, home_team, guest_team):
+def parse_forfeiting_team(cell: _Element, home_team: Team, guest_team: Team) -> Team | None:
     text = str(html.tostring(cell))
     if "2:0" in text:
         return guest_team
@@ -183,15 +186,15 @@ def parse_forfeiting_team(cell, home_team, guest_team):
     return None
 
 
-def parse_game_time(text: str) -> Optional[timedelta]:
+def parse_game_time(text: str) -> timedelta | None:
     if not text:
         return None
 
     minutes, seconds = text.split(':')
     return timedelta(minutes=int(minutes), seconds=int(seconds))
 
 
-def parse_penalty_data(text: str) -> Tuple[int, int]:
+def parse_penalty_data(text: str) -> tuple[int, int]:
     match = re.match(r"(\d+)/(\d+)", text)
     if match:
         return int(match.group(1)), int(match.group(2))

src/games/management/commands/import_games.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict
+from typing import Any
 
 from django.core.management import BaseCommand
 
@@ -19,7 +19,7 @@
 
 
 class Command(BaseCommand):
-    options: Dict = {}
+    options: dict[str, Any] = {}
 
     def add_arguments(self, parser):
         add_default_arguments(parser)
@@ -79,7 +79,7 @@ def import_league(self, league: League):
             try:
                 self.import_game(game_row, league)
             except Exception:
-                logging.getLogger('mail').exception("Could not import Game")
+                LOGGER.exception("Could not import Game")
 
     def import_game(self, game_row, league: League):
 

src/games/tests/unit/test_game_leg.py

@@ -1,5 +1,4 @@
 from datetime import datetime, timedelta
-from typing import Tuple
 
 from django.test import TestCase
 
@@ -58,7 +57,7 @@ def test_four_teams(self):
             self.assertEqual(expected, game.leg())
 
 
-def create_test_game(number, league, home_team, guest_team, expected_leg: Leg, unscheduled=False) -> Tuple[Game, Leg]:
+def create_test_game(number, league, home_team, guest_team, expected_leg: Leg, unscheduled=False) -> tuple[Game, Leg]:
     opening_whistle = datetime.now() + timedelta(weeks=number * 2) if not unscheduled else None
     game = Game.objects.create(number=number, league=league, opening_whistle=opening_whistle,
                                home_team=home_team, guest_team=guest_team)

src/games/urls.py

@@ -1,7 +1,5 @@
-from typing import List
-
 app_name = 'games'
 
-urlpatterns: List = [
+urlpatterns: list = [
 
 ]

src/hbscorez/settings.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-from typing import Optional
 
 ROOT_DIR: Path = Path.cwd()
 
@@ -137,20 +136,18 @@
             'formatter': 'simple',
         },
         'mail': {
+            'level': 'ERROR',
             'class': 'django.utils.log.AdminEmailHandler',
             'include_html': True,
             # 'reporter_class': 'base.error_reporter.CustomErrorReporter', # new in django 3
         },
     },
     'loggers': {
         'hbscorez': {
-            'handlers': ['file', 'console'],
+            'handlers': ['file', 'console', 'mail'],
             'level': 'DEBUG',
             'propagate': True,
         },
-        'mail': {
-            'handlers': ['mail'],
-        },
     },
 }
 
@@ -205,7 +202,7 @@
 
 PUBLIC_NAMES = True
 
-MATOMO_URL: Optional[str] = None
+MATOMO_URL: str | None = None
 
 REPORTS_PATH = ROOT_DIR / 'reports'
 

src/leagues/management/commands/setup.py

@@ -62,7 +62,7 @@ def scrape_associations(options):
         try:
             scrape_association(bhv_id, options)
         except Exception:
-            logging.getLogger('mail').exception("Could not create Association")
+            LOGGER.exception("Could not create Association")
 
 
 def scrape_association(bhv_id: int, options):
@@ -98,7 +98,7 @@ def scrape_districs(association: Association, options):
         try:
             scrape_district(item, association, options)
         except Exception:
-            logging.getLogger('mail').exception("Could not create District")
+            LOGGER.exception("Could not create District")
 
 
 def scrape_district(district_item, association: Association, options):
@@ -125,7 +125,7 @@ def scrape_district(district_item, association: Association, options):
         try:
             scrape_season(district, start_year, options)
         except Exception:
-            logging.getLogger('mail').exception("Could not create Season")
+            LOGGER.exception("Could not create Season")
 
 
 def scrape_season(district, start_year, options):
@@ -155,7 +155,7 @@ def scrape_season(district, start_year, options):
         try:
             scrape_league(league_link, district, season, options)
         except Exception:
-            logging.getLogger('mail').exception("Could not create League")
+            LOGGER.exception("Could not create League")
 
 
 @transaction.atomic

src/leagues/tests/integration/test_setup_league.py

@@ -93,7 +93,7 @@ def test_no_youth(self):
 
 
 class LongLeagueNames(IntegrationTestCase):
-    def test_youth(self):
+    def test_long(self):
         self.assert_command('setup', '-a', 83, '-d', 83, '-s', 2019, '-l', 45646, 45651, '--youth')
 
         leagues = self.assert_objects(League, count=2).order_by('name')