Fix news fetch (#302)

* fix news fetch

* fix failing portal tests

backend/penndata/views.py

@@ -36,22 +36,30 @@ class News(APIView):
     def get_article(self):
         article = {"source": "The Daily Pennsylvanian"}
         try:
-            resp = requests.get("https://www.thedp.com/")
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 "
+                "(KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
+            }
+            resp = requests.get("https://www.thedp.com/", headers=headers)
         except ConnectionError:
             return None
 
         html = resp.content.decode("utf8")
 
         soup = BeautifulSoup(html, "html5lib")
 
-        frontpage = soup.find("div", {"class": "col-lg-6 col-md-5 col-sm-12 frontpage-carousel"})
+        if not (
+            frontpage := soup.find(
+                "div", {"class": "col-lg-6 col-md-5 col-sm-12 frontpage-carousel"}
+            )
+        ):
+            return None
 
         # adds all variables for news object
-        if frontpage:
-            title_html = frontpage.find("a", {"class": "frontpage-link large-link"})
-        if title_html:
-            article["link"] = title_html["href"]
-            article["title"] = title_html.get_text()
+        if not (title_html := frontpage.find("a", {"class": "frontpage-link large-link"})):
+            return None
+        article["link"] = title_html["href"]
+        article["title"] = title_html.get_text()
 
         subtitle_html = frontpage.find("p")
         if subtitle_html:

backend/portal/management/commands/load_target_populations.py

@@ -6,6 +6,14 @@
 
 
 class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--years",
+            type=str,
+            help="Comma seperated years to use for get_years."
+            "This is only used for testing currently.",
+        )
+
     def handle(self, *args, **kwargs):
         # loads majors, years, schools, and degrees onto TargetPopulations
         # runs get_or_create to ensure no duplicates
@@ -24,17 +32,20 @@ def handle(self, *args, **kwargs):
             TargetPopulation.objects.get_or_create(
                 kind=TargetPopulation.KIND_DEGREE, population=degree
             )
-        for year in self.get_years():
+        for year in self.get_years(kwargs["years"]):
             TargetPopulation.objects.get_or_create(kind=TargetPopulation.KIND_YEAR, population=year)
         self.stdout.write("Uploaded Target Populations!")
 
     def get_degrees(self):
         return ["BACHELORS", "MASTERS", "PHD", "PROFESSIONAL"]
 
-    def get_years(self):
+    def get_years(self, years):
         # creates new class year in August in preparation for upcoming school year
-        return (
-            [timezone.localtime().year + x for x in range(4)]
-            if timezone.localtime().month < 8
-            else [timezone.localtime().year + x for x in range(1, 5)]
-        )
+        if years is None:
+            return (
+                [+x for x in range(4)]
+                if timezone.localtime().month < 8
+                else [timezone.localtime().year + x for x in range(1, 5)]
+            )
+        else:
+            return [int(x) for x in years.split(",")]

backend/portal/management/commands/polls_populate.py

@@ -73,7 +73,7 @@ def handle(self, *args, **kwargs):
             user_nursing = User.objects.get(username="user_nursing")
 
         # Create target populations
-        call_command("load_target_populations")
+        call_command("load_target_populations", "--years", "2022, 2023, 2024, 2025")
         target_pop_seas = TargetPopulation.objects.get(
             population="School of Engineering and Applied Science"
         ).id

backend/tests/portal/test_permissions.py

@@ -22,7 +22,7 @@ def mock_get_user_clubs(*args, **kwargs):
 
 class PollPermissions(TestCase):
     def setUp(self):
-        call_command("load_target_populations")
+        call_command("load_target_populations", "--years", "2022, 2023, 2024, 2025")
 
         self.client = APIClient()
         self.admin = User.objects.create_superuser("[email protected]", "admin", "admin")

backend/tests/portal/test_polls.py

@@ -61,7 +61,7 @@ class TestPolls(TestCase):
 
     @mock.patch("portal.serializers.get_user_clubs", mock_get_user_clubs)
     def setUp(self):
-        call_command("load_target_populations")
+        call_command("load_target_populations", "--years", "2022, 2023, 2024, 2025")
         self.target_id = TargetPopulation.objects.get(population="2024").id
         year = TargetPopulation.objects.get(population="2024").id
         major = TargetPopulation.objects.get(population="Computer Science, BSE").id
@@ -234,7 +234,7 @@ class TestPollVotes(TestCase):
     """Tests Create/Update Polls and History"""
 
     def setUp(self):
-        call_command("load_target_populations")
+        call_command("load_target_populations", "--years", "2022, 2023, 2024, 2025")
         self.target_id = TargetPopulation.objects.get(population="2024").id
 
         self.client = APIClient()

backend/tests/portal/test_posts.py

@@ -38,7 +38,7 @@ class TestPosts(TestCase):
 
     @mock.patch("portal.serializers.get_user_clubs", mock_get_user_clubs)
     def setUp(self):
-        call_command("load_target_populations")
+        call_command("load_target_populations", "--years", "2022, 2023, 2024, 2025")
         self.target_id = TargetPopulation.objects.get(population="2024").id
         self.client = APIClient()
         self.test_user = User.objects.create_user("user", "[email protected]", "user")