it4lia/soil_moisture_dataset

soil_moisture_dataset/
├── field_sensor_data_consortium{0,1,2}.parquet       # IoT sensor measurements
├── irrigation_data_consortium{0,1,2}.parquet         # Irrigation data
├── locations_ids_consortium{0,1,2}.parquet           # Location reference
├── historical_weather_data_consortium{0,1,2}.parquet # Historical weather data
├── forecasted_weather_data_consortium{0,1,2}.parquet # 7-day weather forecast
├── weather_data_consortium{0,1}.parquet              # On-site weather station
├── soil_type_data_consortium{0,1}.parquet            # Soil properties
├── remote_sensing_data_final_consortium{0,1}.parquet # Satellite spectral
└── crop_type_data_consortium{0,1,2}.pickle           # Crop information

import pandas as pd
import glob

# Load and concatenate field sensor data across all consortiums
# Note: cast ground_offset to float to handle int64 vs float64 difference in C2
dfs = []
for f in sorted(glob.glob("field_sensor_data_consortium*.parquet")):
    df = pd.read_parquet(f)
    df["ground_offset"] = df["ground_offset"].astype(float)
    dfs.append(df)
sensors = pd.concat(dfs)

# Filter to Water Content sensors only (not available in C1)
wc = sensors[sensors["sensor_type"] == "Water Content"]

# Load historical weather (note: datastream_name is the last column)
weather = pd.concat([
    pd.read_parquet(f) for f in sorted(glob.glob("historical_weather_data_consortium*.parquet"))
])

# Load remote sensing (columns are lowercase: ndvi, grvi, etc.)
rs = pd.concat([
    pd.read_parquet(f) for f in sorted(glob.glob("remote_sensing_data_final_consortium*.parquet"))
])
# Replace Inf before use
import numpy as np
rs = rs.replace([np.inf, -np.inf], np.nan)

import pickle
with open("crop_type_data_consortium0.pickle", "rb") as f:
    crop = pickle.load(f)