sheethal0703/solar-power-forecasting

1import pandas as pd2import numpy as np3import datetime4 5def load_and_preprocess_data():6    print("Loading data...")7    # Load Generation Data8    gen_df = pd.read_csv('Plant_1_Generation_Data.csv')9    10    # Load Air Quality Data11    # 'Air quality information.xlsx' might have multiple sheets or specific formatting12    try:13        air_df = pd.read_excel('Air quality information.xlsx')14    except Exception as e:15        print(f"Error loading Excel file: {e}")16        return None17 18    print("Data loaded successfully.")19    20    # --- Preprocess Generation Data ---21    # DATE_TIME format in CSV: '15-05-2020 00:00' (dd-mm-yyyy hh:mm)22    gen_df['DATE_TIME'] = pd.to_datetime(gen_df['DATE_TIME'], format='%d-%m-%Y %H:%M')23    24    # Aggregate generation data by timestamp (since there are multiple inverters)25    # We want to predict total generation or average generation per timestamp?26    # Usually forecasting is for the whole plant, so let's normalize. 27    # Or we can keep it at inverter level if we merge weather data to each inverter row.28    # For now, let's keep it at inverter level to have more data points, 29    # as weather is the same for all inverters at a given time.30    31    # --- Preprocess Air Quality Data ---32    # The air quality file usually has 'Date' and 'Time' columns. 33    # 'Time' column in some datasets (like UCI Air Quality) might be in a weird format (e.g., '18.00.00').34    # Let's inspect the first few rows structure based on previous tool output or standard behavior.35    36    # Convert 'Date' and 'Time' to string and combine37    # Assuming 'Date' is datetime object and 'Time' is object or time object38    39    # Helper to clean time if it's a string "HH.MM.SS"40    def clean_time(t):41        if isinstance(t, str):42            return t.replace('.', ':')43        return t44 45    air_df['Time'] = air_df['Time'].apply(clean_time)46    47    # If 'Date' is already datetime, we can extract date component.48    # If 'Time' is datetime.time, we can combine.49    50    try:51        air_df['DATE_TIME'] = pd.to_datetime(air_df['Date'].astype(str) + ' ' + air_df['Time'].astype(str))52    except Exception as e:53        print(f"Error creating DATE_TIME in air quality data: {e}")54        # manual fallback if needed, but let's see.55        pass56 57    # Drop original Date/Time columns to avoid confusion58    if 'Date' in air_df.columns:59        air_df.drop(columns=['Date'], inplace=True)60    if 'Time' in air_df.columns:61        air_df.drop(columns=['Time'], inplace=True)62 63    # --- Handling Missing Values in Air Quality ---64    # In some datasets, -200 is used for null.65    air_df.replace(-200, np.nan, inplace=True)66    67    # Impute missing values (forward fill for time series is often good, or interpolation)68    air_df = air_df.interpolate(method='linear')69    70    # Drop rows that still have NaNs (if any, at the start)71    air_df.dropna(inplace=True)72 73    print("Preprocessing timestamps complete.")74    75    # --- Check Timestamps and Merge ---76    # Solar data is every 15 mins. Air quality might be hourly.77    # We need to resample or merge with nearest tolerance.78    79    # Sort both80    gen_df.sort_values('DATE_TIME', inplace=True)81    air_df.sort_values('DATE_TIME', inplace=True)82    83    # Merging84    # using merge_asof to handle different sampling rates (matching to nearest or backward)85    # direction='nearest' matches the closest time. 86    # tolerance=pd.Timedelta('30min') to ensure we don't match very far away data87    88    merged_df = pd.merge_asof(gen_df, air_df, on='DATE_TIME', direction='nearest', tolerance=pd.Timedelta('30min'))89    90    # Drop rows where merge failed (no weather data found close enough)91    merged_df.dropna(subset=air_df.columns.difference(['DATE_TIME']), inplace=True)92    93    print(f"Merged Data Shape: {merged_df.shape}")94    print("Columns:", merged_df.columns.tolist())95    96    return merged_df97 98if __name__ == "__main__":99    df = load_and_preprocess_data()100    if df is not None:101        df.to_csv('merged_data.csv', index=False)102        print("Merged data saved to 'merged_data.csv'")103