SAUNE_extrai_Dados_de_Directores_de_Curso_UA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPF5sDRR9aR/96FqspApEuR",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/fmbento/Jupyter-Notebooks_ad-hoc-solutions/blob/main/SAUNE_extrai_Dados_de_Directores_de_Curso_UA.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!touch cursos_url.txt"
      ],
      "metadata": {
        "id": "V5O0sqi34MjX"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "eYFSkp2CuwLl"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install selenium\n",
        "!apt-get update # to update ubuntu to correctly run apt install\n",
        "!apt install -y chromium-chromedriver\n",
        "!cp /usr/lib/chromium-browser/chromedriver /usr/bin\n",
        "\n",
        "import sys\n",
        "sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from selenium import webdriver\n",
        "from selenium.webdriver.chrome.options import Options\n",
        "from selenium.webdriver.common.by import By\n",
        "# from selenium.webdriver.support.ui import WebDriverWait\n",
        "# from selenium.webdriver.support import expected_conditions as EC\n",
        "import csv\n",
        "import time\n",
        "from google.colab import files\n",
        "\n",
        "# Setup chrome options\n",
        "chrome_options = Options()\n",
        "chrome_options.add_argument('--headless')\n",
        "chrome_options.add_argument('--no-sandbox')\n",
        "chrome_options.add_argument('--disable-dev-shm-usage')\n",
        "\n",
        "# Function to extract information from a single page\n",
        "def extract_info(url):\n",
        "    driver = webdriver.Chrome(options=chrome_options)\n",
        "    driver.get(url)\n",
        "\n",
        "    # wait = WebDriverWait(driver, 10)  # wait for up to 10 seconds\n",
        "    time.sleep(10)  # wait for 10 seconds -- safe value, despite 2 seconds did work initially\n",
        "\n",
        "    curso = driver.find_element(By.XPATH, \"/html/body/div[1]/div/div/section[1]/div/div[1]/div/div/h1\").text\n",
        "    diretor = driver.find_element(By.XPATH, \"/html/body/div[1]/div/div/section[1]/div/div[2]/div[2]/div[1]/a\").text\n",
        "    email = driver.find_element(By.XPATH, \"/html/body/div[1]/div/div/section[1]/div/div[2]/div[2]/div[1]/div[2]/a\").text\n",
        "    departamento_escola = driver.find_element(By.XPATH, \"/html/body/div[1]/div/div/section[1]/div/div[2]/div[2]/div[1]/div[3]/a\").text\n",
        "\n",
        "    driver.quit()\n",
        "\n",
        "    print(url)\n",
        "    print(curso)\n",
        "    print(diretor)\n",
        "    print(email)\n",
        "    print(departamento_escola)\n",
        "    print(\"-------------------------\")\n",
        "\n",
        "    return curso, diretor, email, departamento_escola\n",
        "\n",
        "# Read URLs from file and extract information\n",
        "with open('cursos_url.txt', 'r') as url_file, open('SAUNE_DirsCursos.csv', 'w') as output_file:\n",
        "    csv_writer = csv.writer(output_file)\n",
        "    csv_writer.writerow([\"Curso\", \"Diretor(a) de curso\", \"email\", \"DepartamentoEscola\",\"URL\"])  # Write header\n",
        "\n",
        "    for url in url_file:\n",
        "        curso, diretor, email, departamento_escola = extract_info(url.strip())\n",
        "        csv_writer.writerow([curso, diretor, email, departamento_escola, url])\n",
        "\n",
        "# AutoDonwloads 'SAUNE_DirsCursos.csv'\n",
        "files.download('SAUNE_DirsCursos.csv')"
      ],
      "metadata": {
        "id": "OectXZAsxWgM"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}