Final_project_steps.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#To create the steps and code for the BgGPT News Summarizer Project, we'll break down the requirements into manageable tasks.\n",
    "# This project involves web scraping, text summarization, and data presentation using Streamlit. \n",
    "# We'll also explore options for storing the articles, including SQLite and file storage\n",
    "### Step 1: Retrieve the top 10 News Articles sorted by the website's editors\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "    \n",
    "def get_most_recent_articles(url):\n",
    "        response = requests.get(url)\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "        articles = soup.find_all('article', class_='news-item') # Adjust the selector based on the website's structure\\n\",\n",
    "        return articles[:10]\n",
    "    \n",
    "url = 'https://www.mediapool.bg/'\n",
    "articles = get_most_recent_articles(url)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "### Step 2: Summarize Articles with BgGPT\\n\n",
    "#\"Assuming BgGPT is accessible via an API, we'll iterate over the articles and summarize them.\"\n",
    "\n",
    "def summarize_article(article_text):\n",
    "# Placeholder for BgGPT API call\n",
    "   summary = \"This is a placeholder summary for the article\"\n",
    "   return summary\n",
    "   summaries = [summarize_article(article.text) for article in articles]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "### Step 3: Present Articles in Streamlit GUI\n",
    "#\"We'll use Streamlit to create a simple GUI\n",
    "\n",
    "import streamlit as st\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Define URL and number of articles to scrape\n",
    "url = \"https://www.mediapool.bg/\"\n",
    "get_top_ten_articles = 10\n",
    "\n",
    "# Send request and get response\n",
    "response = requests.get(url)\n",
    "\n",
    "# Check for successful response\n",
    "if response.status_code == 200:\n",
    "    # Parse the HTML content\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "    # Find all article elements\n",
    "    articles = soup.find_all('div', class_='article')[:get_top_ten_articles]\n",
    "\n",
    "    # Extract headlines and article URLs\n",
    "    headlines = []\n",
    "    article_urls = []\n",
    "    for article in articles:\n",
    "        headline = article.find('h2').text.strip()\n",
    "        headlines.append(headline)\n",
    "        article_url = article.find('a')['href']\n",
    "        article_urls.append(article_url)\n",
    "\n",
    "    # Summarize each article\n",
    "    summaries = []\n",
    "    for article_url in article_urls:\n",
    "        article_response = requests.get(article_url)\n",
    "        if article_response.status_code == 200:\n",
    "            article_soup = BeautifulSoup(article_response.content, 'html.parser')\n",
    "            article_text = article_soup.find('div', class_='article-text').text.strip()\n",
    "\n",
    "def get_top_ten_articles(url):\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    articles = soup.find_all('article', class_='news-item') # Adjust the selector based on the website's structure\n",
    "    return articles[:10]\n",
    "\n",
    "def summarize_article(article_text):\n",
    "    # Placeholder for BgGPT API call\n",
    "    summary = \"This is a placeholder summary for the article\"\n",
    "    return summary\n",
    "\n",
    "def get_summaries(articles):\n",
    "    summaries = [summarize_article(article.text) for article in articles]\n",
    "    return summaries\n",
    "\n",
    "def display_articles(articles, summaries):\n",
    "    for i, (article, summary) in enumerate(zip(articles, summaries)):\n",
    "        st.header(f\"Article {i+1}\")\n",
    "        st.write(f\"Title: {article.title}\")\n",
    "        st.write(f\"Link: {article.link}\")\n",
    "        st.write(\"Summary:\")\n",
    "        st.write(summary)\n",
    "\n",
    "url = 'https://www.mediapool.bg/'\n",
    "articles = get_top_ten_articles(url)\n",
    "article_summaries = get_summaries(articles)\n",
    "display_articles(articles, article_summaries)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#### Step 4: Retrieve articles periodically\n",
    "#We'll use APScheduler to schedule the data retrieval at specific times.and display process at regular intervals.\n",
    "\n",
    "# Olena: Streamlit and APScheduler are both designed to run continuously. \n",
    "# Streamlit listens for changes and updates its interface accordingly, \n",
    "# while APScheduler is blocking in nature, meaning it stops further code execution until it is stopped or\n",
    "# terminated. This can lead to problems where the Streamlit server does not start or work properly\n",
    "# because APScheduler is blocking the stream.\n",
    "# Instead of using the BlockingScheduler, we can use a non-blocking version - BackgroundScheduler. SEE streamlit_app1.py\n",
    "\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import streamlit as st\n",
    "from apscheduler.schedulers.background import BackgroundScheduler\n",
    "\n",
    "def retrieve_and_display_articles():\n",
    "    \"\"\"\n",
    "    Retrieve the most recent articles and their summaries, then display them.\n",
    "    \"\"\"\n",
    "    articles = get_most_recent_articles(url)\n",
    "    summaries = [summarize_article(article.text) for article in articles]\n",
    "    display_articles(articles, summaries)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    st.title(\"Scheduled Article Summaries\")\n",
    "    scheduler = BackgroundScheduler()\n",
    "    scheduler.add_job(retrieve_and_display_articles, 'interval', hours=12, start_date='2023-04-01 07:00:00')\n",
    "    scheduler.start()\n",
    "    st.button(\"Click to Refresh Articles\")  # Add a button to manually trigger an update\n",
    "\n",
    "    # Use an empty container to display articles initially and update it when button is clicked.\n",
    "    article_container = st.empty()\n",
    "    with article_container:\n",
    "        retrieve_and_display_articles()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define Example Articles and Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'articles' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[9], line 27\u001b[0m\n\u001b[0;32m     24\u001b[0m             file\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSummary:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m     25\u001b[0m             file\u001b[38;5;241m.\u001b[39mwrite(summary)\n\u001b[1;32m---> 27\u001b[0m store_articles_to_files(\u001b[43marticles\u001b[49m, summaries)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'articles' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "### Step 5: Store Articles. \n",
    "#For simplicity, we'll store articles in files.\n",
    "import os\n",
    "\n",
    "def store_articles_to_files(articles, summaries):\n",
    "    \"\"\"\n",
    "    Store the articles and their summaries to text files.\n",
    "\n",
    "    Each article and its summary is stored in a separate text file. The text files are grouped into a folder. The name of the folder indicates the number of articles in the batch.\n",
    "\n",
    "    Parameters:\n",
    "    articles (list): The articles to be stored.\n",
    "    summaries (list): The summaries of the articles.\n",
    "    \"\"\"\n",
    "def store_articles_to_files(articles, summaries):\n",
    "    batch_size = len(articles)\n",
    "    folder_name = f\"articles_batch_{batch_size}\"\n",
    "    os.makedirs(folder_name, exist_ok=True)\n",
    "\n",
    "    for i, (article, summary) in enumerate(zip(articles, summaries)):\n",
    "        file_name = f\"{folder_name}/article_{i+1}.txt\"\n",
    "        with open(file_name, \"w\") as file:\n",
    "            file.write(f\"Title: {article.title}\\n\")\n",
    "            file.write(f\"Link: {article.link}\\n\")\n",
    "            file.write(\"Summary:\\n\")\n",
    "            file.write(summary)\n",
    "\n",
    "# Now call the function with the defined articles and summaries\n",
    "store_articles_to_files(articles, summaries)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'articles' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[3], line 41\u001b[0m\n\u001b[0;32m     39\u001b[0m \u001b[38;5;66;03m# Example usage\u001b[39;00m\n\u001b[0;32m     40\u001b[0m create_articles_db()\n\u001b[1;32m---> 41\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m article, summary \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(\u001b[43marticles\u001b[49m, summaries):\n\u001b[0;32m     42\u001b[0m     insert_article_to_db(article\u001b[38;5;241m.\u001b[39mtitle, article\u001b[38;5;241m.\u001b[39mlink, summary)\n\u001b[0;32m     43\u001b[0m save_articles_to_files(articles, summaries, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2023-04-01\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmediapool\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'articles' is not defined"
     ]
    }
   ],
   "source": [
    "### Step 6: SQLite Database\\n\",\n",
    "#\"For a more robust solution, consider using SQLite to store articles and summaries.\n",
    "import sqlite3\n",
    "import os\n",
    "\n",
    "def create_articles_db():\n",
    "    \"\"\"\n",
    "    Create a SQLite database and a table for storing articles.\n",
    "    \"\"\"\n",
    "    conn = sqlite3.connect('articles.db')\n",
    "    c = conn.cursor()\n",
    "    c.execute('''CREATE TABLE IF NOT EXISTS articles\n",
    "                 (id INTEGER PRIMARY KEY, title TEXT, link TEXT, summary TEXT)''')\n",
    "    conn.close()\n",
    "\n",
    "def insert_article_to_db(title, link, summary):\n",
    "    \"\"\"\n",
    "    Insert an article into the articles table in the SQLite database.\n",
    "    \"\"\"\n",
    "    conn = sqlite3.connect('articles.db')\n",
    "    c = conn.cursor()\n",
    "    c.execute(\"INSERT INTO articles (title, link, summary) VALUES (?, ?, ?)\", (title, link, summary))\n",
    "    conn.commit()\n",
    "    conn.close()\n",
    "\n",
    "def save_articles_to_files(articles, summaries, time_of_summarization, source):\n",
    "    \"\"\"\n",
    "    Save the articles and their summaries to text files.\n",
    "    \"\"\"\n",
    "    folder_name = f\"articles_{time_of_summarization}_{source}\"\n",
    "    os.makedirs(folder_name)\n",
    "\n",
    "    for i, (article, summary) in enumerate(zip(articles, summaries)):\n",
    "        with open(os.path.join(folder_name, f\"article_{i+1}.txt\"), \"w\") as article_file:\n",
    "        summary_file_path = os.path.join(folder_name, f\"summary_{i+1}.txt\")\n",
    "        \n",
    "        with open(article_file_path, \"w\") as article_file:\n",
    "            # Correctly accessing 'text' or any relevant content from the dictionary\n",
    "            article_text = article.get('text', 'No content available')  # Safely access the 'text' key with a default\n",
    "            article_file.write(f\"Title: {article['title']}\\n\")  # Accessing title from dictionary\n",
    "            article_file.write(f\"Link: {article['link']}\\n\")    # Accessing link from dictionary\n",
    "            article_file.write(\"Article Content:\\n\")\n",
    "            article_file.write(article_text)\n",
    "        with open(os.path.join(folder_name, f\"summary_{i+1}.txt\"), \"w\") as summary_file:\n",
    "            summary_file.write(summary)\n",
    "\n",
    "# Example usage\n",
    "if __name__ == '__main__':\n",
    "    create_articles_db()\n",
    "    # Example articles and summaries\n",
    "    articles = [{'title': 'Example Title', 'link': 'http://example.com', 'text': 'Example Article Text'}]\n",
    "    summaries = ['Example Summary']\n",
    "for article, summary in zip(articles, summaries):\n",
    "    insert_article_to_db(article.title, article.link, summary)\n",
    "save_articles_to_files(articles, summaries, \"2023-04-01\", \"mediapool\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Steamlit_OpenAI_apps",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}