Juliabelloni/weather-forecasting

1from datetime import timedelta2import dateutil 3import requests4import time5import pandas as pd6from .load_abc import LoadABC7 8 9class LoadPollution(LoadABC):10 11    def _get_station_info(self, station_id: str, start_time: str, end_time: str, max_retries: int =10) -> list:12        """13        Retrieves measurements from a given station in the given time period.14 15        Args:16            station_id (str): The ID of the station to retrieve the measurements from.17            start_time (str): The start time of the period to retrieve the measurements from.18            end_time (str): The end time of the period to retrieve the measurements from.19            max_retries (int, optional): The maximum number of retries to perform if the request fails. Defaults to 10.20 21        Returns:22            list: The measurements retrieved from the station in the given time period.23 24        Raises:25            TimeoutError: If the request times out after max_retries attempts.26            ConnectionError: If a network problem occurs while trying to retrieve the data after max_retries attempts.27            HTTPError: If an HTTP error occurs while trying to retrieve the data after max_retries attempts.28            RequestException: If any other request exception occurs while trying to retrieve the data after max_retries attempts.29        """30        gateway_url = "https://api.luchtmeetnet.nl"31        url = f"{gateway_url}/open_api/measurements?station_number={station_id}&formula=&page=&order_by=timestamp_measured&order_direction=desc&end={end_time}&start={start_time}"32 33        payload = {}34        headers = {}35 36        retries = 037        delay = 3  # initial delay in seconds38 39        while retries < max_retries:40            try:41                # Get data from url with timeout42                response = requests.get(url, headers=headers, data=payload)43 44                # Check if the response was successful45                response.raise_for_status()46                # Wait for three seconds so that it does not request more than the set limit (1 request/3sec)47                time.sleep(delay)48 49                # Parse the response JSON50                try:51                    data = response.json()['data']52                except (ValueError, KeyError) as e:53                    raise ValueError(f"Invalid JSON response or missing 'data' field for station {station_id}: {e}")54                return data55 56            except requests.exceptions.Timeout:57                raise TimeoutError(f"Request to station {station_id} timed out.")58            59            except requests.exceptions.ConnectionError:60                raise ConnectionError(f"Network problem while trying to retrieve data for station {station_id}.")61 62            except requests.exceptions.HTTPError as http_err:63 64 65                    raise requests.exceptions.HTTPError(f"HTTP error occurred for station {station_id}: {http_err}")66 67            except requests.exceptions.RequestException as req_err:68                raise requests.exceptions.RequestException(f"Error occurred while requesting data for station {station_id}: {req_err}")69 70        raise requests.exceptions.RequestException(f"Failed to retrieve data for station {station_id} after {max_retries} attempts.")71 72 73    def get_data(self, station_id: str, start_time: str, end_time: str) -> pd.DataFrame:74        """75        Retrieves all data from a given station for a given time period.76 77        This function does so by making multiple requests for 7 day periods, 78        as the API does not allow for larger time ranges.79 80        Parameters81        ----------82        station_id : str83            The id of the station to retrieve data from.84        start_time : str85            The start time of the data to retrieve (inclusive).86        end_time : str87            The end time of the data to retrieve (inclusive).88 89        Returns90        -------91        pd.DataFrame92            A DataFrame containing all the data from the given station for the given time period.93        """94        all_data = []95        current_start = dateutil.parser.parse(start_time)96        end_time = dateutil.parser.parse(end_time)97        while current_start < end_time:98            next_end = min(current_start + timedelta(days=7), end_time)99            data = self._get_station_info(station_id, current_start.isoformat(), next_end.isoformat())100            101            info_str = f"Data successfully retrieved for \102station {station_id} for period {current_start.isoformat()} to {next_end.isoformat()}"    103            self._logs.load_logs(info_str = info_str)104            105            106            all_data.extend(data)107            current_start = next_end + timedelta(days=1)108        return pd.json_normalize(all_data)109