arpaweather.py

# -*- coding: utf-8 -*-
"""
/***************************************************************************
 ARPAweather
                                 A QGIS plugin
 Simplifies the process of collecting and analyzing meteorological ground sensor data. The data are provided by the Environmental Protection Agency of Lombardia Region (ARPA Lombardia) in Northern Italy and include  comprehensive open datasets of weather observations collected over multiple years.
 Generated by Plugin Builder: http://g-sherman.github.io/Qgis-Plugin-Builder/
                              -------------------
        begin                : 2023-02-14
        git sha              : $Format:%H$
        copyright            : (C) 2023 by Emanuele Capizzi - Politecnico di Milano
        email                : emanuele.capizzi@polimi.it
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""
from qgis.PyQt.QtCore import QSettings, QTranslator, QCoreApplication, QVariant, Qt, QUrl
from qgis.PyQt.QtGui import QIcon, QDesktopServices
from qgis.PyQt.QtWidgets import QAction, QMessageBox, QFileDialog, QProgressBar, QProgressDialog, QApplication
from qgis.core import QgsProject, QgsVectorLayer, QgsField, QgsGeometry, QgsPointXY, QgsFeature, Qgis, QgsVectorFileWriter, QgsApplication
from qgis.utils import iface

# Import libraries
from sodapy import Socrata
import pandas as pd
import dask.dataframe as dd
from datetime import datetime, timedelta
import requests
from zipfile import ZipFile
import os
import time
import os

# Initialize Qt resources from file resources.py
from .resources import *
# Import the code for the dialog
from .arpaweather_dialog import ARPAweatherDialog
import os.path

# Set the directory to this script path
script_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(script_dir)

# Create tmp folder to save csv files
tmp_dir = os.path.join(script_dir, 'tmp')
if not os.path.exists(tmp_dir):
    os.mkdir(tmp_dir)

# Weather sensors types
sensors_types = ["Altezza Neve", "Direzione Vento", "Livello Idrometrico", "Precipitazione", "Radiazione Globale", "Temperatura",
                 "Umidità Relativa", "Velocità Vento"]

# Dictionary that maps the zip files to be downloaded from Open Data Lombardia for each year
switcher = {
            '2024': "",
            '2023': "https://www.dati.lombardia.it/download/48xr-g9b9/application%2Fzip",
            '2022': "https://www.dati.lombardia.it/download/mvvc-nmzv/application%2Fzip",
            '2021': "https://www.dati.lombardia.it/download/49n9-866s/application%2Fzip",
            '2020': "https://www.dati.lombardia.it/download/erjn-istm/application%2Fzip",
            '2019': "https://www.dati.lombardia.it/download/wrhf-6ztd/application%2Fzip",
            '2018': "https://www.dati.lombardia.it/download/sfbe-yqe8/application%2Fzip",
            '2017': "https://www.dati.lombardia.it/download/vx6g-atiu/application%2Fzip",
            '2016': "https://www.dati.lombardia.it/download/kgxu-frcw/application%2Fzip"
        }

class ARPAweather:
    """QGIS Plugin Implementation."""

    def __init__(self, iface):
        """Constructor.

        :param iface: An interface instance that will be passed to this class
            which provides the hook by which you can manipulate the QGIS
            application at run time.
        :type iface: QgsInterface
        """
        # Save reference to the QGIS interface
        self.iface = iface
        # initialize plugin directory
        self.plugin_dir = os.path.dirname(__file__)
        # initialize locale
        locale = QSettings().value('locale/userLocale')[0:2]
        locale_path = os.path.join(
            self.plugin_dir,
            'i18n',
            'ARPAweather_{}.qm'.format(locale))

        if os.path.exists(locale_path):
            self.translator = QTranslator()
            self.translator.load(locale_path)
            QCoreApplication.installTranslator(self.translator)

        # Declare instance attributes
        self.actions = []
        self.menu = self.tr(u'&ARPA Weather')

        # Check if plugin was started the first time in current QGIS session
        # Must be set in initGui() to survive plugin reloads
        self.first_start = None

    # noinspection PyMethodMayBeStatic
    def tr(self, message):
        """Get the translation for a string using Qt translation API.

        We implement this ourselves since we do not inherit QObject.

        :param message: String for translation.
        :type message: str, QString

        :returns: Translated version of message.
        :rtype: QString
        """
        # noinspection PyTypeChecker,PyArgumentList,PyCallByClass
        return QCoreApplication.translate('ARPAweather', message)

    def add_action(
        self,
        icon_path,
        text,
        callback,
        enabled_flag=True,
        add_to_menu=True,
        add_to_toolbar=True,
        status_tip=None,
        whats_this=None,
        parent=None):
        """Add a toolbar icon to the toolbar.

        :param icon_path: Path to the icon for this action. Can be a resource
            path (e.g. ':/plugins/foo/bar.png') or a normal file system path.
        :type icon_path: str

        :param text: Text that should be shown in menu items for this action.
        :type text: str

        :param callback: Function to be called when the action is triggered.
        :type callback: function

        :param enabled_flag: A flag indicating if the action should be enabled
            by default. Defaults to True.
        :type enabled_flag: bool

        :param add_to_menu: Flag indicating whether the action should also
            be added to the menu. Defaults to True.
        :type add_to_menu: bool

        :param add_to_toolbar: Flag indicating whether the action should also
            be added to the toolbar. Defaults to True.
        :type add_to_toolbar: bool

        :param status_tip: Optional text to show in a popup when mouse pointer
            hovers over the action.
        :type status_tip: str

        :param parent: Parent widget for the new action. Defaults None.
        :type parent: QWidget

        :param whats_this: Optional text to show in the status bar when the
            mouse pointer hovers over the action.

        :returns: The action that was created. Note that the action is also
            added to self.actions list.
        :rtype: QAction
        """

        icon = QIcon(icon_path)
        action = QAction(icon, text, parent)
        action.triggered.connect(callback)
        action.setEnabled(enabled_flag)

        if status_tip is not None:
            action.setStatusTip(status_tip)

        if whats_this is not None:
            action.setWhatsThis(whats_this)

        if add_to_toolbar:
            # Adds plugin icon to Plugins toolbar
            self.iface.addToolBarIcon(action)

        if add_to_menu:
            self.iface.addPluginToMenu(
                self.menu,
                action)

        self.actions.append(action)

        return action

    def initGui(self):
        """Create the menu entries and toolbar icons inside the QGIS GUI."""

        icon_path = ':/plugins/arpaweather/icon.png'
        self.add_action(
            icon_path,
            text=self.tr(u'ARPA Weather'),
            callback=self.run,
            parent=self.iface.mainWindow())

        # will be set False in run()
        self.first_start = True


    def unload(self):
        """Removes the plugin menu item and icon from QGIS GUI."""
        for action in self.actions:
            self.iface.removePluginMenu(
                self.tr(u'&ARPA Weather'),
                action)
            self.iface.removeToolBarIcon(action)

    def select_output_file(self):
        """
        Opens a file dialog for the user to select an output file name and format. The selected file name is displayed in the output file name line edit.

        Returns:
        None
        """
        options = QFileDialog.Options()
        options |= QFileDialog.ReadOnly
        filename, _filter = QFileDialog.getSaveFileName(self.dlg, "Save Layer As", "", "CSV Files (*.csv)", options=options)
        self.dlg.leOutputFileName.setText(filename)
    
    def select_output_file_ts(self):
        """
        Opens a file dialog for the user to select an output file name and format (for time series (ts)). The selected file name is displayed in the output file name line edit.

        Returns:
        None
        """
        options = QFileDialog.Options()
        options |= QFileDialog.ReadOnly
        filename, _filter = QFileDialog.getSaveFileName(self.dlg, "Save Layer As", "", "CSV Files (*.csv)", options=options)
        self.dlg.leOutputFileName_ts.setText(filename)

    def select_output_file_si(self):
        """
        Opens a file dialog for the user to select an output file name and format (for sensors information (si)). The selected file name is displayed in the output file name line edit.

        Returns:
        None
        """
        options = QFileDialog.Options()
        options |= QFileDialog.ReadOnly
        filename, _filter = QFileDialog.getSaveFileName(self.dlg, "Save Layer As", "", "CSV Files (*.csv)", options=options)
        self.dlg.leOutputFileName_si.setText(filename)

    def select_output_file_si_merged(self):
        """
        Opens a file dialog for the user to select an output file name and format (for time series merged with sensors information (si_merged)). The selected file name is displayed in the output file name line edit.

        Returns:
        None
        """
        options = QFileDialog.Options()
        options |= QFileDialog.ReadOnly
        filename, _filter = QFileDialog.getSaveFileName(self.dlg, "Save Layer As", "", "CSV Files (*.csv)", options=options)
        self.dlg.leOutputFileName_si_merged.setText(filename)

    def connect_ARPA_api(self, token=""):
        """
        Connect to the ARPA API using the provided authentication token.

        If no token is provided, the client will be unauthenticated and subject to strict throttling limits.
        An authentication token can be obtained from the Open Data Lombardia website.

        Parameters:
            token (str): The authentication token obtained from the Open Data Lombardia website.

        Returns:
            Socrata: A client session object for accessing the ARPA API.
        """
        # Connect to Open Data Lombardia using the token
        if token == "":
            print("No token provided. Requests made without an app_token will be subject to strict throttling limits.")
            client = Socrata("www.dati.lombardia.it", None)
        else:
            print("Using provided token.")
            client = Socrata("www.dati.lombardia.it", app_token=token)

        return client

    def ARPA_sensors_info(self, client, selected_provinces) -> pd.DataFrame:
        """
        Convert the ARPA sensors information obtained from a Socrata client to a Pandas dataframe and fix the data types.

        Parameters:
            client (Socrata): A Socrata client session object for accessing the ARPA API.

        Returns:
            pd.DataFrame: A dataframe containing ARPA sensors information, with fixed data types.
        """

        # Select meteo stations dataset containing positions and information about sensors
        stationsId = "nf78-nj6b"
        sensors_info = client.get_all(stationsId)

        # Convert the sensor information to a Pandas dataframe and fix the data types
        sensors_df = pd.DataFrame(sensors_info)
        sensors_df["idsensore"] = sensors_df["idsensore"].astype("int32")
        sensors_df["tipologia"] = sensors_df["tipologia"].astype(str)
        sensors_df["idstazione"] = sensors_df["idstazione"].astype("int32")
        sensors_df["quota"] = sensors_df["quota"].astype("int16")
        sensors_df["unit_dimisura"] = sensors_df["unit_dimisura"].astype(str)
        sensors_df["provincia"] = sensors_df["provincia"].astype("category")
        sensors_df["storico"] = sensors_df["storico"].astype("category")
        sensors_df["datastart"] = pd.to_datetime(sensors_df["datastart"])
        sensors_df["datastop"] = pd.to_datetime(sensors_df["datastop"])

        # Drop not relevant fields
        sensors_df = sensors_df.drop(columns=["cgb_est", "cgb_nord", "location"]) #, ":@computed_region_6hky_swhk", ":@computed_region_ttgh_9sm5"])
        
        # Filter the sensors data by selected provinces
        if len(selected_provinces) == 0:
            # If no provinces are selected, use default list of provinces
            selected_provinces = ['BG', 'BS', 'CO', 'CR', 'LC', 'LO', 'MB', 'MI', 'MN', 'PV', 'SO', 'VA']
        sensors_df = sensors_df[sensors_df['provincia'].isin(selected_provinces)]

        return sensors_df

    def req_ARPA_start_end_date_API(self, client):
        """
        Requests the start and end date of data available in the ARPA API.

        Parameters:
            client (sodapy.Socrata): Client session for interacting with the ARPA API.

        Returns:
            Tuple[datetime, datetime]: The earliest and latest dates available in the ARPA API.

        Raises:
            Exception: If there is an issue making the API request or parsing the response.

        """
        try:
            with client:
                # Dataset ID for weather sensors on Open Data Lombardia
                weather_sensor_id = "647i-nhxk"

                # Query the API for the minimum and maximum dates available
                query = """ select MAX(data), MIN(data) limit 9999999999999999"""

                # Extract the min and max dates from the API response
                min_max_dates = client.get(weather_sensor_id, query=query)[0]

                # Start and minimum dates from the dict obtained from the API
                start_API_date = min_max_dates['MIN_data']
                end_API_date = min_max_dates['MAX_data']

                # Convert the date strings to datetime objects
                #start_API_date = datetime.strptime(start_API_date, "%Y-%m-%dT%H:%M:%S.%f")
                end_API_date = datetime.strptime(end_API_date, "%Y-%m-%dT%H:%M:%S.%f")

                start_API_date = datetime(end_API_date.year, end_API_date.month, 1, 0, 0, 0)

                return start_API_date, end_API_date

        except Exception as e:
            # If there's an error, print a message and raise an exception
            print(f"Error fetching ARPA API data: {e}")
            raise Exception("Error fetching ARPA API data")

    def req_ARPA_data_API(self, client, start_date, end_date, sensors_list):
        """
        Function to request data from available weather sensors in the ARPA API using a query.

        Parameters:
            client (requests.Session): the client session
            start_date (datetime): the start date in datetime format
            end_date (datetime): the end date in datetime format
            sensors_list (list of int): list of selected sensor ids

        Returns:
            pandas.DataFrame: dataframe with idsensore, data and valore of the weather sensors within the specific time period
        """
        # Select the Open Data Lombardia Meteo sensors dataset
        weather_sensor_id = "i95f-5avh"

        # Convert to string in year-month-day format, accepted by ARPA query
        start_date = start_date.strftime("%Y-%m-%dT%H:%M:%S.%f")
        end_date = end_date.strftime("%Y-%m-%dT%H:%M:%S.%f")

        # Query data
        query = """
        select
            *
        where data >= \'{}\' and data <= \'{}\' limit 9999999999999999
        """.format(start_date, end_date)

        # Get time series and evaluate time spent to request them
        time_series = client.get(weather_sensor_id, query=query)

        # Create dataframe
        df = pd.DataFrame(time_series, columns=['idsensore', 'data', 'valore', 'idoperatore'])

        # Convert types
        df['valore'] = df['valore'].astype('float32')
        df['idsensore'] = df['idsensore'].astype('int32')
        df['data'] = pd.to_datetime(df['data'])
        df['idoperatore'] = df['idoperatore'].astype('float32')

        # Filter with selected sensors list
        df = df[df['valore'] != -9999]
        df = df[df['idsensore'].isin(sensors_list)]

        return df

    def download_extract_csv_from_year(self, year, switcher, bar):
        """
        Downloads a zipped CSV file of meteorological data from ARPA sensors for a given year from the Open Data Lombardia website.
        If the file has already been downloaded, it will be skipped.
        Extracts the downloaded zip file and saves the CSV file to the temporary directory (tmp).

        Parameters:
            year (str): The selected year for downloading the CSV file containing the meteorological sensors time series.

        Returns:
            None
        """
        
        # Create a dictionary with years and corresponding download links on Open Data Lombardia - REQUIRES TO BE UPDATED EVERY YEAR
        switcher = switcher
        
        # Select the URL based on the year and make request
        url = switcher[year]
        filename = 'meteo_'+str(year)+'.zip'
        

        # If year.csv file is already downloaded, skip download
        if not os.path.exists(os.path.join(tmp_dir, f"{year}.csv")):
            print("--- Starting download ---")
            iface.messageBar().pushMessage("Download", "Downloading CSV file. It might take a while... Please wait!", level=Qgis.Info)
            t = time.time()
            print((f'Downloading {filename} -> Started. It might take a while... Please wait!'))
            response = requests.get(url, stream=True)
            
            block_size = 1024
            wrote = 0.0 
            
            # Writing the file to the local file system
            with open(os.path.join(tmp_dir, filename), "wb") as f:
                for data in response.iter_content(block_size):
                    wrote = wrote + len(data)
                    f.write(data)
                    try:
                        bar.setValue(int((wrote / (block_size*block_size))/5))
                        QApplication.processEvents()
                    except Exception as e:
                        print(f"Error: {e}")
                    #percentage = wrote / (block_size*block_size)
                
            elapsed = time.time() - t
            print((f'\nDownloading {filename} -> Completed. Time required for download: {elapsed:0.2f} s.'))
            print((f"Starting unzipping: {filename}"))

            #Loading the .zip and creating a zip object
            with ZipFile(os.path.join(tmp_dir, filename), 'r') as zObject:
                # Extracting all the members of the zip into a specific location
                zObject.extractall(tmp_dir)

            csv_file=str(year)+'.csv'
            print((f"File unzipped: {filename}"))
            print((f"File csv saved: {filename}"))

            #Remove the zip folder
            if os.path.exists(os.path.join(tmp_dir, filename)):
                print(("{filename} removed").format(filename=filename))
                os.remove(os.path.join(tmp_dir, filename))
            else:
                print((f"The file {filename} does not exist in this folder"))
        
        else:
            print(f"{year}.csv already exists. It won't be downloaded.")

    def process_ARPA_csv(self, csv_file, start_date, end_date, sensors_list):
        """
        Reads an ARPA csv file into a Dask dataframe, applies data processing and returns a computed and filtered Dask dataframe. 

        Args:
            csv_file (str): File name of the csv file
            start_date (datetime): Start date for processing
            end_date (datetime): End date for processing
            sensors_list (list of str): List of selected sensors

        Returns:
            df (Dask dataframe): Computed filtered Dask dataframe
        """

        print("--- Starting processing csv data ---")
        print(f"The time range used for the processing is {start_date} to {end_date}")
        
        #Read csv file with Dask dataframe
        csv_file = os.path.join(tmp_dir, csv_file)
        df = dd.read_csv(csv_file, usecols=['IdSensore','Data','Valore', 'Stato']).rename(columns={'IdSensore': 'idsensore', 'Data': 'data', 'Valore': 'valore', 'Stato':'stato'})
        
        # Format data types
        df['valore'] = df['valore'].astype('float32')
        df['idsensore'] = df['idsensore'].astype('int32')
        df['data'] = dd.to_datetime(df.data, format='%d/%m/%Y %H:%M:%S')
        df['stato'] = df['stato'].astype('category')
        
        #Filter using the dates
        df = df[df['valore'] != -9999]
        df = df.loc[(df['data'] >= start_date) & (df['data'] <= end_date)]
        #Filter on temperature sensors list
        sensors_list = list(map(int, sensors_list))
        df = df[df['idsensore'].isin(sensors_list)] #keep only sensors in the list (for example providing a list of temperature sensors, will keep only those)
        df = df[df.stato.isin(["VA", "VV"])] #keep only validated data identified by stato equal to VA and VV
        df = df.drop(['stato'], axis=1)
               
        # Sort the dataframe by date
        # df = df.sort_values(by='data', ascending=True).reset_index(drop=True)
        
        #Compute df
        df = df.compute()
        
        return df 

    def aggregate_group_data(self, df):
        """
        Aggregates ARPA data using statistical aggregation functions (mean, max, min, std, and count), except for wind direction (Direzione Vento).
        The dataframe is grouped by sensor ID (`idsensore`).

        Parameters:
            df (DataFrame): ARPA DataFrame containing the following columns: `idsensore` (int), 
                            `data` (datetime), and `valore` (float)

        Returns:
            DataFrame: aggregated DataFrame containing the following columns: `idsensore` (int), 
                        `mean` (float), `max` (float), `min` (float), `std` (float), and `count` (int)
        """

        # Group the DataFrame by 'idsensore' and compute the statistical metrics
        df = df.set_index('data')
        grouped = df.groupby('idsensore')['valore'].agg(['mean', 'max', 'min', 'std', 'count'])

        # Reset the index to make 'idsensore' a column again
        grouped = grouped.reset_index()

        return grouped
    
    def aggregate_group_data_wind_dir(self, df):
        """
        Aggregates ARPA wind direction data using mode and count functions. The dataframe is grouped by sensor id (idsensore).

        Parameters:
            df(dataframe): ARPA dataframe containing the following columns: "idsensore"(int), "data"(datetime) and "valore"(float)

        Returns:
            df(dataframe): computed filtered and aggregated dask dataframe
        """

        # Group by sensor id and aggregate wind direction values using mode and count functions
        grouped = df.groupby('idsensore')['valore'].agg([lambda x: pd.Series.mode(x)[0], 'count']).rename({'<lambda_0>': 'mode'}, axis=1)
        grouped = grouped.reset_index()

        return grouped

    def cleanup_csv_files():
        """
        Deletes all the CSV files present in the temporary folder (tmp).

        Parameters:
            None

        Returns:
            None
        """
        # Set the path of the temporary folder where CSV files are stored
        folder_path = tmp_dir

        # Loop through all files in the folder and delete CSV files
        for filename in os.listdir(folder_path):
            if filename.endswith(".csv"):
                file_path = os.path.join(folder_path, filename)
                try:
                    if os.path.isfile(file_path):
                        os.unlink(file_path) # delete the file
                except Exception as e:
                    print("Error while deleting file:", e) # print error message if file cannot be deleted

    def update_calendar(self, index):
        """
        Updates the minimum and maximum dates of the date-time edit widgets in the dialog based on the year selected
        in the combo box.

        :param index (int): the index of the selected item in the combo box
        """
        # Get the selected year from the combo box and the current date and last day of previous month
        today = datetime.today()
        first_day_of_month = datetime(today.year, today.month, 1)
        last_day_of_prev_month = first_day_of_month - timedelta(days=1)

        # If the combo box is empty, set the selected year to the current year
        if self.dlg.cb_list_years.count() == 0:
            sel_year = int(today.year)
        else:
            sel_year = int(self.dlg.cb_list_years.currentText())

        # If the selected year is the current year, set the minimum date to the beginning of the year and the maximum date to the end of the previous month
        if sel_year == int(today.year):
            csv_cal_start_date = datetime(sel_year, 1, 1, 0, 0, 0)
            if today.month != 1:
                csv_cal_end_date = datetime(sel_year, today.month-1, last_day_of_prev_month.day, 23, 59, 0)
            else:
                csv_cal_end_date = datetime(sel_year, 1, last_day_of_prev_month.day, 23, 59, 0)
        else: 
            csv_cal_start_date = datetime(sel_year, 1, 1, 0, 0, 0)
            csv_cal_end_date = datetime(sel_year, 12, 31, 23, 59, 0)

        # Set the display format, calendar popup, minimum date, maximum date, start date, and end date of the date-time edit widgets
        self.dlg.dtStartTime.setDisplayFormat("dd-MM-yyyy HH:mm:ss")
        self.dlg.dtEndTime.setDisplayFormat("dd-MM-yyyy HH:mm:ss")

        self.dlg.dtStartTime.setCalendarPopup(True)
        self.dlg.dtEndTime.setCalendarPopup(True)

        self.dlg.dtStartTime.setMinimumDateTime(csv_cal_start_date)
        self.dlg.dtStartTime.setMaximumDateTime(csv_cal_end_date)
        self.dlg.dtEndTime.setMinimumDateTime(csv_cal_start_date)
        self.dlg.dtEndTime.setMaximumDateTime(csv_cal_end_date)

        self.dlg.dtStartTime.setDateTime(csv_cal_start_date)
        self.dlg.dtEndTime.setDateTime(csv_cal_end_date)
        
    def update_CSV(self):
        """
        Updates the contents of the year combo box and the minimum and maximum dates of the date-time edit widgets in the dialog
        based on the year selected in the combo box.
        """
        # Clear the year combo box and add the available years
        self.dlg.cb_list_years.clear()
        years_list = list(switcher.keys())
        self.dlg.cb_list_years.addItems(years_list)

        # Get the selected year from the combo box and the current date and last day of previous month
        sel_year = int(self.dlg.cb_list_years.currentText())
        today = datetime.today()
        first_day_of_month = datetime(today.year, today.month, 1)
        last_day_of_prev_month = first_day_of_month - timedelta(days=1)
            
        # Delimit selectable dates 
        # For current year CSV let select only dates up to the previous month
        if sel_year == int(today.year):
            csv_cal_start_date = datetime(sel_year, 1, 1, 0, 0, 0)
            # minus 1 to get the previous month with respect to current one
            if today.month != 1:
                csv_cal_end_date = datetime(sel_year, today.month-1, last_day_of_prev_month.day, 23, 59, 0)
            else:
                csv_cal_end_date = datetime(sel_year, 1, last_day_of_prev_month.day, 23, 59, 0)
        else: 
            csv_cal_start_date = datetime(sel_year, 1, 1, 0, 0, 0)
            csv_cal_end_date = datetime(sel_year, 12, 31, 23, 59, 0)

        # Set the display format, calendar popup, minimum date, maximum date, start date, and end date of the date-time edit widgets
        self.dlg.dtStartTime.setDisplayFormat("dd-MM-yyyy HH:mm:ss")
        self.dlg.dtEndTime.setDisplayFormat("dd-MM-yyyy HH:mm:ss")
        self.dlg.dtStartTime.setCalendarPopup(True)
        self.dlg.dtEndTime.setCalendarPopup(True)

        self.dlg.dtStartTime.setMinimumDateTime(csv_cal_start_date)
        self.dlg.dtStartTime.setMaximumDateTime(csv_cal_end_date)
        self.dlg.dtEndTime.setMinimumDateTime(csv_cal_start_date)
        self.dlg.dtEndTime.setMaximumDateTime(csv_cal_end_date)

        self.dlg.dtStartTime.setDateTime(csv_cal_start_date)
        self.dlg.dtEndTime.setDateTime(csv_cal_end_date)
    
    def outlier_filter_iqr(self, df):
        """
        Filters outliers from a given dataframe using the interquartile range (IQR) method.
        
        Parameters:
            df (dataframe): ARPA dataframe containing at least the following columns: "idsensore"(int), "data"(datetime) and "valore"(float)
            sensors_list (int list): list of sensors

        Returns:
            df(dataframe): filtered dataframe using IQR
        
        """
        # Compute the first and third quartiles and the IQR
        Q1 = df['valore'].quantile(0.25)
        Q3 = df['valore'].quantile(0.75)
        IQR = Q3 - Q1

        # Filter the DataFrame based on the IQR
        df = df[~((df['valore'] < (Q1 - 1.5 * IQR)) | (df['valore'] > (Q3 + 1.5 * IQR)))]

        return df
    
    def outlier_filter_zscore(self, df, threshold=3):
        """
        Filter dataframe using Z-Score method. The Z-Score method is used to filter out data points that are more than a specified number of standard deviations from the
        mean of the dataset. The default threshold value is 3, which is a common choice in statistics.

        Parameters:
            df (pandas.DataFrame): ARPA dataframe with columns "idsensore" (int), "data" (datetime), and "valore" (float)
            threshold (float, optional): Z-Score threshold to use for filtering, default is 3

        Returns:
            pandas.DataFrame: filtered dataframe using Z-Score method
            
        """
        # Initialize an empty dataframe to store the filtered data
        filtered_df = pd.DataFrame(columns=['idsensore', 'data', 'valore'])
        
        # Loop through each unique sensor in the dataframe
        for sensor in df['idsensore'].unique():
            # Get the rows for the current sensor
            sensor_df = df[df['idsensore'] == sensor]
            # Calculate the mean and standard deviation of the values for the current sensor
            mean = sensor_df['valore'].mean()
            std = sensor_df['valore'].std()
            # Calculate the z-score for each value in the current sensor's dataframe
            z = (sensor_df['valore'] - mean) / std
            # Filter out any rows where the absolute value of the z-score is greater than the threshold
            sensor_df = sensor_df[(z.abs() < threshold)]
            # Concatenate the filtered sensor dataframe to the overall filtered dataframe
            filtered_df = pd.concat([filtered_df, sensor_df], ignore_index=True)
            
        return filtered_df
    
    # To set the progress bar
    def progdialog(self,progress):
        p_dialog = QProgressDialog('ARPA Weather Plugin processing. This may take some time...', 'Cancel', 0, 100)
        p_dialog.setWindowTitle("Progress ARPA Weather Plugin")
        p_dialog.setWindowModality(Qt.WindowModal)
        bar = QProgressBar(p_dialog)
        bar.setTextVisible(True)
        bar.setMaximum(100)
        bar.setValue(0)
        p_dialog.setBar(bar)
        p_dialog.setMinimumWidth(300)
        p_dialog.show()
        return p_dialog, bar

# --- RUN ------------

    def run(self):
        """Run method that performs all the real work"""

        # Create the dialog with elements (after translation) and keep reference
        # Only create GUI ONCE in callback, so that it will only load when the plugin is started

        if self.first_start == True:
            self.first_start = False
            self.dlg = ARPAweatherDialog()
            self.dlg.pbOutputSave.clicked.connect(self.select_output_file)
            self.dlg.pbOutputSave_ts.clicked.connect(self.select_output_file_ts)
            self.dlg.pbOutputSave_si.clicked.connect(self.select_output_file_si)
            self.dlg.pbOutputSave_si_merged.clicked.connect(self.select_output_file_si_merged)
        
        # Group box toggled
        self.dlg.rb1.setChecked(True) # Radio button 1 (API) checked at the beginning

        # Clear widgets, put only those you want to reset each time the Plugin is opened
        self.dlg.cbSensorsType.clear()
        self.dlg.cbSensorsType.addItems([str(sensor) for sensor in sensors_types])
        
        self.dlg.cb_list_years.clear()
        self.dlg.cb_list_years.addItem(list(switcher.keys())[0])
        self.dlg.cb_list_years.currentIndexChanged.connect(self.update_calendar)
        
        self.dlg.cbOutliersRemoval.clear()
        self.dlg.cbOutliersRemoval.addItems(['None', 'IQR', 'Z-Score'])

        self.dlg.leOutputFileName.clear()
        self.dlg.leOutputFileName_ts.clear()
        self.dlg.leOutputFileName_si.clear()
        self.dlg.leOutputFileName_si_merged.clear()

        # Add documentation link
        self.dlg.labelLinkDoc.setText('<a href="https://github.com/capizziemanuele/ARPA_Weather_plugin">GitHub Doc</a>')
        self.dlg.labelLinkDoc.setOpenExternalLinks(True)
        
        # Folder contanining complete CSV
        labelHistCSV = self.dlg.labelHistoricalCSV
        labelHistCSV.setText(f'<a href="{tmp_dir}">Downloaded CSV</a>')
        
        # Create a function to handle the label click event
        def open_folder(event):
            QDesktopServices.openUrl(QUrl.fromLocalFile(tmp_dir))
            
        # Connect the label click event to the open_folder function
        labelHistCSV.mousePressEvent = open_folder
        
        # Modifiy initial widgets
        try:
            # Connect to the ARPA API
            client = self.connect_ARPA_api()

            # Request the start and end dates from the API
            start_date_API, end_date_API = self.req_ARPA_start_end_date_API(client)

            # Convert start and end dates to string format
            label_name_start = start_date_API.strftime("%Y-%m-%d %H:%M:%S")
            label_name_end = end_date_API.strftime("%Y-%m-%d %H:%M:%S")

            # Update date labels in the GUI
            self.dlg.label_startAPIdate.setText(label_name_start)
            self.dlg.label_endAPIdate.setText(label_name_end)

            self.dlg.label_CSVfirstyear.setText(datetime(int(list(switcher.keys())[-1]), 1, 1).strftime("%Y-%m-%d %H:%M:%S"))
            self.dlg.label_CSVendyear.setText((start_date_API - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"))

        except requests.exceptions.RequestException as e:
            # Raise an error message if there is an issue with the request
            QMessageBox.warning(self.dlg, "Error", str(e))
        
        # Initialize calendar to API dates
        self.dlg.dtStartTime.setDisplayFormat("dd-MM-yyyy HH:mm:ss")
        self.dlg.dtEndTime.setDisplayFormat("dd-MM-yyyy HH:mm:ss")
        self.dlg.dtStartTime.setCalendarPopup(True)
        self.dlg.dtEndTime.setCalendarPopup(True)

        self.dlg.dtStartTime.setMinimumDateTime(start_date_API)
        self.dlg.dtStartTime.setMaximumDateTime(end_date_API)
        self.dlg.dtEndTime.setMinimumDateTime(start_date_API)
        self.dlg.dtEndTime.setMaximumDateTime(end_date_API)

        # self.dlg.dtStartTime.setDateTime(start_date_API)
        self.dlg.dtEndTime.setDateTime(end_date_API)
        
        # Functin to update calendar with API dates after creating client connection
        def update_API():
            self.dlg.cb_list_years.clear()
            current_year = list(switcher.keys())[0]  #select last year
            self.dlg.cb_list_years.addItem(current_year)
            
            self.dlg.dtStartTime.setDisplayFormat("dd-MM-yyyy HH:mm:ss")
            self.dlg.dtEndTime.setDisplayFormat("dd-MM-yyyy HH:mm:ss")
            self.dlg.dtStartTime.setCalendarPopup(True)
            self.dlg.dtEndTime.setCalendarPopup(True)

            self.dlg.dtStartTime.setMinimumDateTime(start_date_API)
            self.dlg.dtStartTime.setMaximumDateTime(end_date_API)
            self.dlg.dtEndTime.setMinimumDateTime(start_date_API)
            self.dlg.dtEndTime.setMaximumDateTime(end_date_API)

            self.dlg.dtStartTime.setDateTime(start_date_API)
            self.dlg.dtEndTime.setDateTime(end_date_API)

        # Update widgets according to API dates
        self.dlg.rb1.toggled.connect(update_API)
        
        # Update widgets according to CSV dates
        self.dlg.rb2.toggled.connect(self.update_CSV)
        
        # Show the dialog
        self.dlg.show()

        # Run the dialog event loop
        result = self.dlg.exec_()

        if result:
            
            p_dialog, bar = self.progdialog(0)
            bar.setMaximum(100)
            bar.setValue(0)
            QApplication.processEvents()
            # Select provinces and create list of selected provinces
            selected_provinces = []
            for checkbox in [self.dlg.cb_BG,self.dlg.cb_BS,self.dlg.cb_CO, self.dlg.cb_CR,self.dlg.cb_LC,self.dlg.cb_LO,self.dlg.cb_MB,
                            self.dlg.cb_MI,self.dlg.cb_MN,self.dlg.cb_PV,self.dlg.cb_SO,self.dlg.cb_VA]:
                if checkbox.isChecked():
                    selected_provinces.append(checkbox.text())

            # Get the start and the end date from the gui
            if self.dlg.rb1.isChecked():
                start_date = self.dlg.dtStartTime.dateTime().toPyDateTime()
                end_date = self.dlg.dtEndTime.dateTime().toPyDateTime()
                if start_date.year != end_date.year:
                    QMessageBox.warning(None, "Invalid Date Range", "Dates must be in the same year!")
            else:
                start_date = self.dlg.dtStartTime.dateTime().toPyDateTime()
                end_date = self.dlg.dtEndTime.dateTime().toPyDateTime()

            # Create client
            if self.dlg.rb1.isChecked():
                arpa_token = self.dlg.leToken.text()
            else:
                arpa_token = ""

            client = self.connect_ARPA_api(arpa_token)

            with client:

                # Dataframe containing sensors information
                sensors_df = self.ARPA_sensors_info(client, selected_provinces)

                # Get the selected sensor from the gui
                sensor_sel = self.dlg.cbSensorsType.currentText()
                
                # Get the selected outlier method from the gui
                outlier_method_sel = self.dlg.cbOutliersRemoval.currentText()

                # Filter the sensors depending on the "tipologia" field (sensor type)
                sensors_list = (sensors_df.loc[sensors_df['tipologia'] == sensor_sel]).idsensore.tolist()

                year = start_date.year 
                # Check that the start and end dates are in the same year
                if start_date.year != end_date.year:
                    QMessageBox.warning(None, "Invalid Date Range", "Dates must be in the same year!")
                    return
                elif start_date > end_date:
                    QMessageBox.warning(None, "Invalid Date Range", "Start date must be before end date")
                    return

                # Request time series
                if start_date < start_date_API:
                    print("Requesting CSV. This will take a while.")
                    sensors_values = self.download_extract_csv_from_year(str(year), switcher, bar) #download the csv corresponding to the selected year
                    csv_file = str(year)+'.csv'

                    # Updates the progress bar
                    bar.setValue(70)
                    QApplication.processEvents()

                    sensors_values = self.process_ARPA_csv(csv_file, start_date, end_date, sensors_list) #process csv file with dask
                    if sensor_sel != "Direzione Vento":  # Don't check for outliers if wind direction
                        if outlier_method_sel == 'IQR':
                            sensors_values = sensors_values.groupby('idsensore').apply(self.outlier_filter_iqr)
                        if outlier_method_sel == 'Z-Score':
                            sensors_values = self.outlier_filter_zscore(sensors_values)

                #If the chosen start date is equal or after the start date of API -> request data from API
                elif (start_date >= start_date_API):  # If the end_date is greater than the end_date _API the latter will be used
                    print("Requesting from API")

                    # Updates the progress bar
                    bar.setValue(10)
                    QApplication.processEvents()

                    sensors_values = self.req_ARPA_data_API(client, start_date, end_date, sensors_list) #request data from ARPA API
                    if sensor_sel != "Direzione Vento":  # Don't check for outliers if wind direction
                        if outlier_method_sel == 'IQR':
                            sensors_values = sensors_values.groupby('idsensore').apply(self.outlier_filter_iqr)
                        if outlier_method_sel == 'Z-Score':
                            sensors_values = self.outlier_filter_zscore(sensors_values)

                # Calculate statistics on the whole dataset
                if sensor_sel != "Direzione Vento":
                    sensor_test_agg = self.aggregate_group_data(sensors_values)
                
                if sensor_sel == "Direzione Vento":
                    sensor_test_agg = self.aggregate_group_data_wind_dir(sensors_values)

                # Updates the progress bar
                    bar.setValue(90)
                    QApplication.processEvents()

                # Merge the values with the sensors info
                merged_df = pd.merge(sensor_test_agg, sensors_df, on='idsensore')

                merged_df['lng'] = merged_df['lng'].astype('float64')
                merged_df['lat'] = merged_df['lat'].astype('float64')
                merged_df['idsensore'] = merged_df['idsensore'].astype('int32')
                merged_df['tipologia'] = merged_df['tipologia'].astype(str)
                merged_df['datastart'] = merged_df['datastart'].astype(str)

                # Create vector layer
                layer = QgsVectorLayer("Point?crs=EPSG:4326", sensor_sel+' ({start} / {end})'.format(start=start_date, end=end_date), "memory")

                if sensor_sel != "Direzione Vento":
                    merged_df.round({'media': 1, 'max': 1, 'min': 1, 'std': 1})
                    
                    layer.dataProvider().addAttributes([QgsField("idsensore", QVariant.Int), QgsField("tipologia", QVariant.String), QgsField("unit_dimisura", QVariant.String), QgsField("idstazione", QVariant.Int),
                                                        QgsField("nomestazione", QVariant.String), QgsField("quota", QVariant.Double), QgsField("provincia", QVariant.String), QgsField("datastart", QVariant.String),
                                                        QgsField("storico", QVariant.String), QgsField("lng", QVariant.Double), QgsField("lat", QVariant.Double), QgsField("media", QVariant.Double, 'double', 10, 1), QgsField("max", QVariant.Double, 'double', 10, 1),
                                                        QgsField("min", QVariant.Double, 'double', 10, 1), QgsField("std", QVariant.Double, 'double', 10, 1), QgsField("conteggio", QVariant.Int)])
                    
                
                if sensor_sel == "Direzione Vento":
                    merged_df.round({'moda': 0})
                    
                    layer.dataProvider().addAttributes([QgsField("idsensore", QVariant.Int),
                                                        QgsField("tipologia", QVariant.String), QgsField("unit_dimisura", QVariant.String), QgsField("idstazione", QVariant.Int), QgsField("nomestazione", QVariant.String), QgsField("quota", QVariant.Double),
                                                        QgsField("provincia", QVariant.String), QgsField("datastart", QVariant.String), QgsField("storico", QVariant.String),
                                                        QgsField("lng", QVariant.Double), QgsField("lat", QVariant.Double), QgsField("moda", QVariant.Double,'double', 10, 0), QgsField("conteggio", QVariant.Int)])

                # Update fields and start editing
                layer.updateFields()
                layer.startEditing()

                # Features creation
                features = []
                if sensor_sel != "Direzione Vento":         # If wind direction sensor is NOT selected
                    for index, row in merged_df.iterrows():
                        point = QgsPointXY(row['lng'], row['lat'])
                        feature = QgsFeature()
                        feature.setGeometry(QgsGeometry.fromPointXY(point))
                        feature.setAttributes([QVariant(row['idsensore']),
                                            QVariant(row['tipologia']), QVariant(row['unit_dimisura']),
                                            QVariant(row['idstazione']), QVariant(row['nomestazione']),
                                            QVariant(row['quota']), QVariant(row['provincia']), QVariant(row['datastart']), 
                                            QVariant(row['storico']), QVariant(row['lng']), QVariant(row['lat']), QVariant(row['mean']), QVariant(row['max']),
                                            QVariant(row['min']), QVariant(row['std']), QVariant(row['count'])])
                        features.append(feature)
                
                if sensor_sel == "Direzione Vento":         # If wind direction sensor is selected
                    for index, row in merged_df.iterrows():
                        point = QgsPointXY(row['lng'], row['lat'])
                        feature = QgsFeature()
                        feature.setGeometry(QgsGeometry.fromPointXY(point))
                        feature.setAttributes([QVariant(row['idsensore']),
                                            QVariant(row['tipologia']), QVariant(row['unit_dimisura']),
                                            QVariant(row['idstazione']), QVariant(row['nomestazione']),
                                            QVariant(row['quota']), QVariant(row['provincia']), QVariant(row['datastart']), 
                                            QVariant(row['storico']), QVariant(row['lng']), QVariant(row['lat']),QVariant(round(row['mode'])), QVariant(row['count'])])
                        features.append(feature)

                # Add features and commit changes
                layer.addFeatures(features)
                layer.commitChanges()

                # Add the layer to the QGIS project
                QgsProject.instance().addMapLayer(layer)
                layer.updateExtents()

                # EXPORT - Save file as csv
                filename = self.dlg.leOutputFileName.text()
                context = QgsProject.instance().transformContext()
                beginning_columns = ['idsensore', 'tipologia', 'unit_dimisura',	'idstazione', 'nomestazione', 'quota', 'provincia', 'datastart', 'storico', 'lng', 'lat', 'datastop']
                remaining_columns = [col for col in merged_df.columns if col not in beginning_columns]
                new_order = beginning_columns + remaining_columns
                merged_df = merged_df[new_order]
                
                # Create a dictionary to define the column name mappings
                if sensor_sel != "Direzione Vento": 
                    column_mappings = {'mean': 'media', 'count': 'conteggio'}

                    # Rename the columns using the dictionary
                    merged_df = merged_df.rename(columns=column_mappings)
                
                if sensor_sel == "Direzione Vento": 
                    column_mappings = {'mode': 'moda', 'count': 'conteggio'}

                    # Rename the columns using the dictionary
                    merged_df = merged_df.rename(columns=column_mappings)

                if filename != "":
                    try:
                        merged_df.to_csv(filename, index=False,  encoding="utf-8-sig")
                        
                        # Write message
                        self.iface.messageBar().pushMessage("Success", "Output file written at " + filename, level=Qgis.Success, duration=3)
                    except:
                        raise Exception("Error while exporting the file.")

                # EXPORT - Save time-series as csv
                filename_ts = self.dlg.leOutputFileName_ts.text()

                if filename_ts != "":
                    try: 
                        # Save as csv
                        sensors_values_csv = sensors_values.reset_index(drop=True)
                        sensors_values_csv = sensors_values_csv.sort_values(['idsensore', 'data'], ascending=[True, True])
                        sensors_values_csv.to_csv(filename_ts, index=False, encoding="utf-8-sig")
                        
                        # Write message
                        self.iface.messageBar().pushMessage("Success", "Output file written at " + filename_ts, level=Qgis.Success, duration=3)
                    except:
                        raise Exception("Error while exporting time-series CSV file.")
            
                # EXPORT - Save sensors information as csv
                filename_si = self.dlg.leOutputFileName_si.text()

                if filename_si != "":
                    try:
                        # Save as csv
                        sensors_df_csv = sensors_df.loc[sensors_df['tipologia'] == sensor_sel]
                        sensors_df_csv.to_csv(filename_si, index=False, encoding="utf-8-sig")
                        
                        # Write message
                        self.iface.messageBar().pushMessage("Success", "Output file written at " + filename_si, level=Qgis.Success, duration=3)
                    except:
                        raise Exception("Error while exporting sensors information CSV file.")
                    
                # EXPORT - Save time series and sensors information merged as csv
                filename_si_merged = self.dlg.leOutputFileName_si_merged.text()

                if filename_si_merged != "":
                    try:
                        # Save as csv
                        sensors_values_csv = sensors_values.reset_index(drop=True)
                        sensors_values_csv = sensors_values_csv.sort_values(['idsensore', 'data'], ascending=[True, True])
                        sensors_df_csv = sensors_df.loc[sensors_df['tipologia'] == sensor_sel]

                        sensors_df_merged_csv = pd.merge(sensors_values_csv, sensors_df_csv, how='left', on='idsensore')
                        sensors_df_merged_csv.to_csv(filename_si_merged, index=False, encoding="utf-8-sig")
                        
                        # Write message
                        self.iface.messageBar().pushMessage("Success", "Output file written at " + filename_si_merged, level=Qgis.Success, duration=3)
                    except:
                        raise Exception("Error while exporting time series with sensors information CSV file.")


            # Updates the progress bar
                bar.setValue(10)
                QApplication.processEvents()
            pass

    QgsApplication.instance().aboutToQuit.connect(cleanup_csv_files)