Skip to content

Commit

Permalink
Check
Browse files Browse the repository at this point in the history
  • Loading branch information
nikhil25803 committed May 19, 2024
1 parent 460521a commit c11f4ca
Show file tree
Hide file tree
Showing 6 changed files with 37 additions and 39 deletions.
7 changes: 2 additions & 5 deletions dev-documentation.md
Original file line number Diff line number Diff line change
Expand Up @@ -1649,7 +1649,7 @@ Methods
| Methods | Details |
| -------------------------- | ---------------------------------- |
| `.get_user_data(username)` | Fetches user data from CodeForces. |
| `get_contests()` | Returns information on contests. |
| `get_contests()` | Returns information on contests. |
```

---
Expand Down Expand Up @@ -1745,9 +1745,7 @@ result = steam.ScrapeGames(n0Games=5, tags=["Discounts", "F2P"])
| ----------------------------- | ------------------------------------------- |
| `.ScrapeGames(n0Games, tags)` | Scrapes game data for each specified filter |


-------

---

## Lichess

Expand Down Expand Up @@ -1853,4 +1851,3 @@ trek=Indiantrekking("hidden-lakes-of-kashmir")
| `outline_day_to_day_itinerary` | returns the ouline of the day to day itinerary |

---

3 changes: 1 addition & 2 deletions src/scrape_up/ambitionBox/company.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

# import requests
# from bs4 import BeautifulSoup

Expand Down Expand Up @@ -78,4 +77,4 @@

# if __name__ == "__main__":
# c = Comapiens(10)
# c.scrape_companies()
# c.scrape_companies()
2 changes: 1 addition & 1 deletion src/scrape_up/codeforces/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from .user import Users
from .contests import Contest

__all__ = ["Users", "Contest"]
__all__ = ["Users", "Contest"]
57 changes: 30 additions & 27 deletions src/scrape_up/codeforces/contests.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from bs4 import BeautifulSoup
import json

from scrape_up.config.request_config import RequestConfig, get


class Contest:
"""
First, create an object of class `Contest`
Expand All @@ -15,6 +15,7 @@ class Contest:
| ---------------------------- | ----------------------------------------------------------------------------------------- |
| `get_contests()` | Returns information on active contests like title, start, and duration |
"""

def __init__(self, *, config: RequestConfig = RequestConfig()):
headers = {"User-Agent": "scrapeup"}
self.config = config
Expand All @@ -36,46 +37,48 @@ def get_contests(self):
-------
{
"data": [
{
"name": "Codeforces Round #731 (Div. 3)",
"start": "Aug/08/2021 17:35",
"length": "2 hrs"
},
{
"name": "Codeforces Round 946 (Div. 3)",
"start": "05/20/2024 17:35",
"length": "02:15",
"status": "upcoming"
}
"name": "Codeforces Round #731 (Div. 3)",
"start": "Aug/08/2021 17:35",
"length": "2 hrs"
},
...
],
"message": "Found contest list"
}
"""
codeforces_url = "https://codeforces.com/contests"
response = get(codeforces_url, self.config)

if response.status_code != 200:
json.dumps({"data": None, "message": "Cannot load Contest"})

soup = BeautifulSoup(response.text, "html.parser")
contest_list = []

upcoming_contests = soup.find("div", {"class": "datatable"}).find_all("tr")
for contest in upcoming_contests:
columns = contest.find_all("td")
if len(columns) == 6: # The number of columns in the table row for contests
name = columns[0].text.strip()
start_time_str = columns[2].text.strip()
duration_str = columns[3].text.strip()
try:

name = ' '.join(line.strip() for line in name.splitlines() if line.strip())
name = name.replace('Enter »', '').strip()
upcoming_contests = soup.find("div", {"class": "datatable"}).find_all("tr")
for contest in upcoming_contests:
columns = contest.find_all("td")
if len(columns) == 6:
name = columns[0].text.strip()
start_time_str = columns[2].text.strip()
duration_str = columns[3].text.strip()

contest_list.append({
"name": name,
"start": start_time_str,
"length": duration_str,
})
name = " ".join(
line.strip() for line in name.splitlines() if line.strip()
)
name = name.replace("Enter »", "").strip()

return json.dumps({"data": contest_list, "message": "Found contest list"})
contest_list.append(
{
"name": name,
"start": start_time_str,
"length": duration_str,
}
)

return contest_list
except Exception:
return None
2 changes: 1 addition & 1 deletion src/scrape_up/indiantrekking/trek.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,4 @@ def outline_day_to_day_itinerary(self):
outline = self.soup.find("div", class_="itinerary").text
return outline
except:
return None
return None
5 changes: 2 additions & 3 deletions src/scrape_up/lichess/lichess.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def __fetch_page_games(self, page_num):
game_list.append(game_info)
return game_list

def fetch_games(self,start_page=1,end_page=4):
def fetch_games(self, start_page=1, end_page=4):
"""
Fetch all the games data for the specified username.
Expand All @@ -53,7 +53,7 @@ def fetch_games(self,start_page=1,end_page=4):
```python
# Default usage:
games = scraper.fetch_games()
# Custom usage:
games = scraper.fetch_games(start_page=5, end_page=8)
```
Expand Down Expand Up @@ -130,4 +130,3 @@ def __get_pgn(self, game_data):
pgn_request = requests.get(f"https://lichess.org{gameUrl}")._content
parsed_pgn = BeautifulSoup(pgn_request, "lxml")
return parsed_pgn.find("div", {"class": "pgn"}).text

0 comments on commit c11f4ca

Please sign in to comment.