Final_project_steps_error-free.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#To create the steps and code for the BgGPT News Summarizer Project, we'll break down the requirements into manageable tasks.\n",
    "# This project involves web scraping, text summarization, and data presentation using Streamlit. \n",
    "# We'll also explore options for storing the articles, including SQLite and file storage\n",
    "### Step 1: Retrieve the top 10 News Articles sorted by the website's editors\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "    \n",
    "def get_top_ten_news_articles(url):\n",
    "        response = requests.get(url)\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "        articles = soup.find_all('article', class_='news-item') # Adjust the selector based on the website's structure\\n\",\n",
    "        return articles[:10]\n",
    "    \n",
    "url = 'https://www.mediapool.bg/'\n",
    "articles = get_top_ten_news_articles(url)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Step 2: Summarize Articles with BgGPT\\n\n",
    "#\"Assuming BgGPT is accessible via an API, we'll iterate over the articles and summarize them.\"\n",
    "\n",
    "def summarize_article(article_text):\n",
    "# Placeholder for BgGPT API call\n",
    "   summary = \"This is a placeholder summary for the article\"\n",
    "   return summary\n",
    "   summaries = [summarize_article(article.text) for article in articles]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Step 3: Present Articles in Streamlit GUI\n",
    "#\"We'll use Streamlit to create a simple GUI\n",
    "\n",
    "import streamlit as st\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Define URL and number of articles to scrape\n",
    "url = \"https://www.mediapool.bg/\"\n",
    "get_top_ten_articles = 10\n",
    "\n",
    "# Send request and get response\n",
    "response = requests.get(url)\n",
    "\n",
    "# Check for successful response\n",
    "if response.status_code == 200:\n",
    "    # Parse the HTML content\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "    # Find all article elements\n",
    "    articles = soup.find_all('div', class_='article')[:get_top_ten_articles]\n",
    "\n",
    "    # Extract headlines and article URLs\n",
    "    headlines = []\n",
    "    article_urls = []\n",
    "    for article in articles:\n",
    "        headline = article.find('h2').text.strip()\n",
    "        headlines.append(headline)\n",
    "        article_url = article.find('a')['href']\n",
    "        article_urls.append(article_url)\n",
    "\n",
    "    # Summarize each article\n",
    "    summaries = []\n",
    "    for article_url in article_urls:\n",
    "        article_response = requests.get(article_url)\n",
    "        if article_response.status_code == 200:\n",
    "            article_soup = BeautifulSoup(article_response.content, 'html.parser')\n",
    "            article_text = article_soup.find('div', class_='article-text').text.strip()\n",
    "\n",
    "def get_top_ten_articles(url):\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    articles = soup.find_all('article', class_='news-item') # Adjust the selector based on the website's structure\n",
    "    return articles[:10]\n",
    "\n",
    "def summarize_article(article_text):\n",
    "    # Placeholder for BgGPT API call\n",
    "    summary = \"This is a placeholder summary for the article\"\n",
    "    return summary\n",
    "\n",
    "def get_summaries(articles):\n",
    "    summaries = [summarize_article(article.text) for article in articles]\n",
    "    return summaries\n",
    "\n",
    "def display_articles(articles, summaries):\n",
    "    for i, (article, summary) in enumerate(zip(articles, summaries)):\n",
    "        st.header(f\"Article {i+1}\")\n",
    "        st.write(f\"Title: {article.title}\")\n",
    "        st.write(f\"Link: {article.link}\")\n",
    "        st.write(\"Summary:\")\n",
    "        st.write(summary)\n",
    "\n",
    "url = 'https://www.mediapool.bg/'\n",
    "articles = get_top_ten_articles(url)\n",
    "article_summaries = get_summaries(articles)\n",
    "display_articles(articles, article_summaries)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-17 18:07:43.667 \n",
      "  \u001b[33m\u001b[1mWarning:\u001b[0m to view this Streamlit app on a browser, run it with the following\n",
      "  command:\n",
      "\n",
      "    streamlit run C:\\Users\\Nadia Stoyanova\\AppData\\Roaming\\Python\\Python312\\site-packages\\ipykernel_launcher.py [ARGUMENTS]\n"
     ]
    }
   ],
   "source": [
    "#### Step 4: Retrieve articles periodically\n",
    "#We'll use APScheduler to schedule the data retrieval at specific times.and display process at regular intervals.\n",
    "\n",
    "# Olena: Streamlit and APScheduler are both designed to run continuously. \n",
    "# Streamlit listens for changes and updates its interface accordingly, \n",
    "# while APScheduler is blocking in nature, meaning it stops further code execution until it is stopped or\n",
    "# terminated. This can lead to problems where the Streamlit server does not start or work properly\n",
    "# because APScheduler is blocking the stream.\n",
    "# Instead of using the BlockingScheduler, we can use a non-blocking version - BackgroundScheduler. SEE streamlit_app1.py\n",
    "\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import streamlit as st\n",
    "from apscheduler.schedulers.background import BackgroundScheduler\n",
    "\n",
    "def get_top_ten_news_articles(url):\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    articles = soup.find_all('article')\n",
    "    return articles[:10]\n",
    "\n",
    "def summarize_article(article_text):\n",
    "    return article_text[:100]\n",
    "\n",
    "def display_articles(articles, summaries):\n",
    "    for i, (article, summary) in enumerate(zip(articles, summaries)):\n",
    "        st.header(f\"Article {i+1}\")\n",
    "        title = article.find('h1').text if article.find('h1') else 'No title found'\n",
    "        link = article.find('a')['href'] if article.find('a') else 'No link found'\n",
    "        st.write(f\"Title: {title}\")\n",
    "        st.write(f\"Link: {link}\")\n",
    "        st.write(\"Summary:\")\n",
    "        st.write(summary)\n",
    "\n",
    "url = 'https://www.mediapool.bg/'\n",
    "\n",
    "def retrieve_and_display_articles():\n",
    "    \"\"\"\n",
    "    Retrieve the most recent articles and their summaries, then display them.\n",
    "    \"\"\"\n",
    "    articles = get_top_ten_news_articles(url)\n",
    "    summaries = [summarize_article(article.get_text()) for article in articles]\n",
    "    display_articles(articles, summaries)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    st.title(\"Scheduled Article Summaries\")\n",
    "    scheduler = BackgroundScheduler()\n",
    "    scheduler.add_job(retrieve_and_display_articles, 'interval', hours=12, start_date='2023-04-01 07:00:00')\n",
    "    scheduler.start()\n",
    "    st.button(\"Click to Refresh Articles\")  # Add a button to manually trigger an update\n",
    "\n",
    "    # Use an empty container to display articles initially and update it when button is clicked.\n",
    "    article_container = st.empty()\n",
    "    with article_container:\n",
    "        retrieve_and_display_articles()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Example Articles and Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example article data (usually fetched from a web source or other API)\n",
    "articles = [\n",
    "    {'title': 'Article 1', 'link': 'http://example.com/article1'},\n",
    "    {'title': 'Article 2', 'link': 'http://example.com/article2'}\n",
    "]\n",
    "\n",
    "# Corresponding summaries for the articles\n",
    "summaries = [\n",
    "    \"Summary of Article 1.\",\n",
    "    \"Summary of Article 2.\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Step 5: Store Articles. \n",
    "#For simplicity, we'll store articles in files.\n",
    "\n",
    "import os\n",
    "\"\"\"\n",
    "    Store the articles and their summaries to text files.\n",
    "\n",
    "    Each article and its summary is stored in a separate text file. \n",
    "    The text files are grouped into a folder. The name of the folder indicates the number of\n",
    "    articles in the batch.\n",
    "\n",
    "    Parameters:\n",
    "    articles (list): The articles to be stored.\n",
    "    summaries (list): The summaries of the articles.\n",
    "    \"\"\"\n",
    "def store_articles_to_files(articles, summaries):\n",
    "    batch_size = len(articles)\n",
    "    folder_name = f\"articles_batch_{batch_size}\"\n",
    "    os.makedirs(folder_name, exist_ok=True)  # Ensures that the directory exists\n",
    "\n",
    "    for i, (article, summary) in enumerate(zip(articles, summaries)):\n",
    "        file_name = f\"{folder_name}/article_{i+1}.txt\"\n",
    "        with open(file_name, \"w\") as file:\n",
    "            file.write(f\"Title: {article['title']}\\n\")  # Access title from dictionary\n",
    "            file.write(f\"Link: {article['link']}\\n\")    # Access link from dictionary\n",
    "            file.write(\"Summary:\\n\")\n",
    "            file.write(summary)\n",
    "\n",
    "# Now call the function with the defined articles and summaries\n",
    "store_articles_to_files(articles, summaries)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Step 6: SQLite Database\\n\",\n",
    "#\"For a more robust solution, consider using SQLite to store articles and summaries.\n",
    "\n",
    "import sqlite3\n",
    "import os\n",
    "\n",
    "def create_articles_db():\n",
    "    \"\"\" Create a SQLite database and a table for storing articles. \"\"\"\n",
    "    with sqlite3.connect('articles.db') as conn:\n",
    "        c = conn.cursor()\n",
    "        c.execute('''\n",
    "            CREATE TABLE IF NOT EXISTS articles\n",
    "            (id INTEGER PRIMARY KEY, title TEXT, link TEXT, summary TEXT)\n",
    "        ''')\n",
    "        # Fetch all the rows\n",
    "    rows = c.fetchall()\n",
    "    for row in rows:\n",
    "        print(row)\n",
    "# Close the connection\n",
    "    conn.close()\n",
    "\n",
    "def insert_article_to_db(title, link, summary):\n",
    "    \"\"\" Insert an article into the articles table in the SQLite database. \"\"\"\n",
    "    with sqlite3.connect('articles.db') as conn:\n",
    "        c = conn.cursor()\n",
    "        c.execute(\"INSERT INTO articles (title, link, summary) VALUES (?, ?, ?)\", \n",
    "                  (title, link, summary))\n",
    "        conn.commit()\n",
    "\n",
    "def save_articles_to_files(articles, summaries, time_of_summarization, source):\n",
    "    \"\"\" Save the articles and their summaries to text files. \"\"\"\n",
    "    folder_name = f\"articles_{time_of_summarization}_{source}\"\n",
    "    os.makedirs(folder_name, exist_ok=True)\n",
    "\n",
    "    for i, (article, summary) in enumerate(zip(articles, summaries)):\n",
    "        article_file_path = os.path.join(folder_name, f\"article_{i+1}.txt\")\n",
    "        summary_file_path = os.path.join(folder_name, f\"summary_{i+1}.txt\")\n",
    "        \n",
    "        with open(article_file_path, \"w\") as article_file:\n",
    "            # Correctly accessing 'text' or any relevant content from the dictionary\n",
    "            article_text = article.get('text', 'No content available')  # Safely access the 'text' key with a default\n",
    "            article_file.write(f\"Title: {article['title']}\\n\")  # Accessing title from dictionary\n",
    "            article_file.write(f\"Link: {article['link']}\\n\")    # Accessing link from dictionary\n",
    "            article_file.write(\"Article Content:\\n\")\n",
    "            article_file.write(article_text)\n",
    "        \n",
    "        with open(summary_file_path, \"w\") as summary_file:\n",
    "            summary_file.write(summary)\n",
    "\n",
    "# Example usage\n",
    "if __name__ == '__main__':\n",
    "    create_articles_db()\n",
    "    # Example articles and summaries\n",
    "    articles = [{'title': 'Example Title', 'link': 'http://example.com', 'text': 'Example Article Text'}]\n",
    "    summaries = ['Example Summary']\n",
    "    for article, summary in zip(articles, summaries):\n",
    "        insert_article_to_db(article['title'], article['link'], summary)\n",
    "    save_articles_to_files(articles, summaries, \"2023-04-01\", \"mediapool\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Steamlit_OpenAI_apps",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}