rudrani-rane/ATIS

1import pandas as pd
2from sklearn.preprocessing import StandardScaler
3from pathlib import Path
4from load_data import load_raw_data
5
6PROCESSED_PATH = Path("data/processed/processed_asteroids.csv")
7
8def preprocess():
9
10    df = load_raw_data()
11
12    # Convert Y/N → 1/0
13    df["neo"] = df["neo"].map({"Y":1,"N":0})
14    df["pha"] = df["pha"].map({"Y":1,"N":0})
15
16    # Drop unused text columns
17    df = df.drop(columns=["full_name","pdes","class"])
18
19    # Remove missing important values
20    df = df.dropna(subset=["e","a","i","moid","rms"])
21
22    # Filtering rules
23    df = df[df["condition_code"] <= 5]
24    df = df[df["data_arc"] > 100]
25    df = df[df["moid"] < 0.5]
26
27    features = [
28        "H","e","a","q","i","om","w","ad","n","per_y",
29        "moid","neo","pha","data_arc","condition_code","rms"
30    ]
31
32    scaler = StandardScaler()
33    df[features] = scaler.fit_transform(df[features])
34
35    PROCESSED_PATH.parent.mkdir(parents=True, exist_ok=True)
36    df.to_csv(PROCESSED_PATH, index=False)
37
38    print("Processed dataset saved:", df.shape)
39
40if __name__ == "__main__":
41    preprocess()