KaiquanMah/DSIP

1import argparse2import pandas as pd3from sklearn.model_selection import train_test_split4from sklearn.preprocessing import MinMaxScaler5from sklearn.preprocessing import LabelEncoder6import os7 8 9 10def parse(csv_path):11    print(f"Location of the file: {csv_path}")12    13    # Step 1: Load the dataset14    # file_path = "dataset.csv"  # Path to the original dataset15    data = pd.read_csv(csv_path)16 17    # Drop NA and duplicates18    data = data.dropna()19    data.to_csv('data/01 dropna.csv', index=False)20    data = data.drop_duplicates()21    data.to_csv('data/02 drop_duplicates.csv', index=False)22    23    # Step 2: Define the feature columns (X) and target column (y)24    # X = data[["name", "attendance percentage", "average sleep time", "average screen time"]]  # Feature columns25    X = data[["DateTime","product","campaign_id","webpage_id","product_category_1","product_category_2","user_group_id","gender","age_level","user_depth","city_development_index","var_1"]]  # Feature columns26    y = data["is_click"]  # Target column27 28    # Extract datetime features29    X.loc[:,'DateTime'] = pd.to_datetime(X['DateTime'], errors='coerce')30    X = X.dropna(subset=['DateTime'])31    # print(X.columns)32    # print(X.iloc[:5,0])33    # print(X.iloc[:5,0].dt.weekday)34    X.loc[:,'weekday'] = pd.to_numeric(pd.to_datetime(X['DateTime'], errors='coerce').dt.weekday, errors='coerce', downcast='integer')35    X.loc[:,'month'] = pd.to_numeric(pd.to_datetime(X['DateTime'], errors='coerce').dt.month, errors='coerce', downcast='integer')36    X.loc[:,"hour"] = pd.to_datetime(X['DateTime'], errors='coerce').dt.hour.values37    X = X.drop('DateTime', axis=1)38 39    # Product label to number40    le = LabelEncoder()41    X.loc[:,"product"] = le.fit_transform(X["product"])42    # Gender label to number43    X['gender'] = X['gender'].map({'Female': 1,44                                   'Male': 0,45                                   'M': 0})46    47    48    # Normalize numerical features49    scaler = MinMaxScaler()50    numerical_features = ['campaign_id','webpage_id','user_depth',"product_category_1","product_category_2","user_group_id", 'age_level',"user_depth", 'city_development_index', 'var_1']51    X[numerical_features] = scaler.fit_transform(X[numerical_features])52    data = pd.concat([X, y.to_frame(name="is_click")], axis=1)53    data.to_csv('data/03 normalize.csv', index=False)54 55    56    57    # Step 3: Split the dataset into training and testing sets58    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)59    60    # Step 4: Combine X and y back into dataframes for train and test61    train_data = pd.concat([X_train, y_train], axis=1)  # Combine features and target for training data62    test_data = pd.concat([X_test, y_test], axis=1)    # Combine features and target for testing data63    64    # Step 5: Create the 'data' folder if it doesn't exist65    output_folder = "data"66    os.makedirs(output_folder, exist_ok=True)67    68    # Step 6: Save the train and test sets as CSV files69    # train_file_path = os.path.join(output_folder, "train.csv")70    train_file_path = "data/train.csv"71    test_file_path = "data/test.csv"72    73    train_data.to_csv(train_file_path, index=False)74    test_data.to_csv(test_file_path, index=False)75    76    print(f"Train and test datasets saved in '{output_folder}' folder.")77    78 79 80 81 82if __name__ == '__main__':83    parser = argparse.ArgumentParser()84    parser.add_argument("--csv-path", type=str)85    86    87    args = parser.parse_args()88    parse(args.csv_path)89