fminaret/Package-algorithm-training

1import argparse2import pandas as pd3import time4import mlflow5from mlflow.models.signature import infer_signature6from sklearn.model_selection import train_test_split 7from sklearn.preprocessing import  StandardScaler, FunctionTransformer, OneHotEncoder8from sklearn.compose import ColumnTransformer9from sklearn.ensemble import RandomForestClassifier10from sklearn.pipeline import Pipeline11 12 13if __name__ == "__main__":14 15    ### MLFLOW Experiment setup16    experiment_name="appointment_cancellation_detector"17    mlflow.set_experiment(experiment_name)18    experiment = mlflow.get_experiment_by_name(experiment_name)19 20    client = mlflow.tracking.MlflowClient()21    run = client.create_run(experiment.experiment_id)22 23    print("training model...")24    25    # Time execution26    start_time = time.time()27 28    # Call mlflow autolog29    mlflow.sklearn.autolog(log_models=False) # We won't log models right away30 31    # Parse arguments given in shell script32    parser = argparse.ArgumentParser()33    parser.add_argument("--n_estimators")34    parser.add_argument("--min_samples_split")35    args = parser.parse_args()36 37    # Import dataset38    df = pd.read_csv("https://full-stack-assets.s3.eu-west-3.amazonaws.com/Deployment/doctolib_simplified_dataset_01.csv")39 40    # X, y split 41    X = df.iloc[:, 3:-1]42    y = df.iloc[:, -1].apply(lambda x: 0 if x=="No" else 1)43 44    # Train / test split 45    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)46 47    # Preprocessing 48    def date_processing(df):49        df = df.copy()50 51        ## Transform datetime into a number52        df["ScheduledDay"] = pd.to_datetime(df["ScheduledDay"], yearfirst=True, infer_datetime_format=True)53        df["AppointmentDay"] = pd.to_datetime(df["AppointmentDay"], yearfirst=True, infer_datetime_format=True)54 55        ## Get the difference between scheduled day and appointment56        df["time_difference_between_scheduled_and_appointment"] = (df["AppointmentDay"] - df["ScheduledDay"]).dt.days57 58        ## Remove redundant info 59        df = df.drop(["ScheduledDay", "AppointmentDay"], axis=1)60 61        return df 62 63    date_preprocessor = FunctionTransformer(date_processing)64 65    # Preprocessing 66    categorical_features = ["Gender", "Neighbourhood"] # Select all the columns containing strings67    categorical_transformer = OneHotEncoder(drop='first', handle_unknown='error', sparse=False)68 69    numerical_feature_mask = ~X_train.columns.isin(["Gender", "Neighbourhood", "ScheduledDay","AppointmentDay"]) # Select all the columns containing anything else than strings70    numerical_features = X_train.columns[numerical_feature_mask]71    numerical_transformer = StandardScaler()72 73    feature_preprocessor = ColumnTransformer(74        transformers=[75            ("categorical_transformer", categorical_transformer, categorical_features),76            ("numerical_transformer", numerical_transformer, numerical_features)77        ]78    )79 80    # Pipeline 81    n_estimators = int(args.n_estimators)82    min_samples_split=int(args.min_samples_split)83 84    model = Pipeline(steps=[85        ("Dates_preprocessing", date_preprocessor),86        ('features_preprocessing', feature_preprocessor),87        ("Regressor",RandomForestClassifier(n_estimators=n_estimators, min_samples_split=min_samples_split))88    ])89 90    # Log experiment to MLFlow91    with mlflow.start_run(run_id = run.info.run_id) as run:92        model.fit(X_train, y_train)93        predictions = model.predict(X_train)94 95        # Log model seperately to have more flexibility on setup 96        mlflow.sklearn.log_model(97            sk_model=model,98            artifact_path="appointment_cancellation_detector",99            registered_model_name="appointment_cancellation_detector_RF",100            signature=infer_signature(X_train, predictions)101        )102        103    print("...Done!")104    print(f"---Total training time: {time.time()-start_time}")