earthroverprogram/lucas-mega

1import json2import os3from pathlib import Path4 5import pandas as pd6from PIL import Image7 8try:9    import rasterio10    import streamlit as st11except ImportError as exc:12    raise SystemExit(13        "viewer_standardization.py requires streamlit and rasterio.\n"14        "Install missing packages, then run: streamlit run viewer_standardization.py"15    ) from exc16 17 18BASE_DIR = Path(__file__).resolve().parent19HOST_NAME = "esdac"20DATASETS_DIR = BASE_DIR / "datasets" / HOST_NAME21STATUS_PATH = BASE_DIR / "src" / HOST_NAME / "status.json"22ICON_PATH = BASE_DIR / "resources" / "erp.jpeg"23DEFAULT_DATASET = "soil-bulk-density-europe"24DEFAULT_FILE = "Public/packing_density.png"25 26VIEWABLE_EXTENSIONS = {27    ".csv",28    ".json",29    ".png",30    ".jpg",31    ".jpeg",32    ".txt",33    ".tif",34    ".tiff",35}36STATUS_OPTIONS = ["UNEXAMINED", "SKIPPED", "REQUESTED", "DOWNLOADED", "PROCESSED"]37 38 39def format_bytes(size):40    size = float(size)41    for unit in ["B", "KB", "MB", "GB"]:42        if size < 1024 or unit == "GB":43            return f"{size:.1f} {unit}" if unit != "B" else f"{int(size)} B"44        size /= 102445    return f"{size:.1f} GB"46 47 48@st.cache_data(show_spinner=False)49def get_datasets():50    datasets = {}51 52    try:53        with open(STATUS_PATH, encoding="utf-8") as f:54            items = json.load(f)55    except Exception as exc:56        return datasets, f"Error reading {STATUS_PATH}: {exc}"57 58    for item in items:59        name = item["name"]60        dataset_path = DATASETS_DIR / name61        processed_path = dataset_path / "processed"62 63        if not processed_path.exists():64            continue65 66        file_list = []67        for root, _, files in os.walk(processed_path):68            root_path = Path(root)69            for file_name in files:70                path = root_path / file_name71                if path.suffix.lower() in VIEWABLE_EXTENSIONS:72                    rel_path = path.relative_to(processed_path)73                    file_list.append(str(rel_path))74 75        datasets[name] = {76            "name": name,77            "title": item.get("title", ""),78            "url": item.get("url"),79            "abstract": item.get("abstract") or "",80            "request_needed": item.get("request_needed", False),81            "status": item.get("status"),82            "notes": item.get("notes"),83            "screened_by": item.get("screened_by"),84            "requested_downloaded_by": item.get("requested_downloaded_by"),85            "processed_by": item.get("processed_by"),86            "files": sorted(file_list),87            "path": str(dataset_path),88            "processed_path": str(processed_path),89        }90 91    return datasets, None92 93 94def file_stats(path):95    stat = path.stat()96    return {97        "Path": str(path),98        "Size": format_bytes(stat.st_size),99        "Modified": pd.Timestamp(stat.st_mtime, unit="s").strftime("%Y-%m-%d %H:%M:%S"),100    }101 102 103def format_value(value):104    if isinstance(value, float):105        return f"{value:.6g}"106    return str(value)107 108 109def render_dataset_info(data):110    st.subheader(data["name"])111    if data.get("title"):112        st.write(data["title"])113 114    cols = st.columns(4)115    cols[0].metric("Status", data.get("status") or "NA")116    cols[1].metric("Files", f"{len(data.get('files', [])):,}")117    cols[2].metric("Request needed", str(data.get("request_needed")))118    cols[3].metric("Processed by", data.get("processed_by") or "NA")119 120    details = {121        "URL": data.get("url"),122        "Screened by": data.get("screened_by"),123        "Requested/downloaded by": data.get("requested_downloaded_by"),124        "Notes": data.get("notes"),125        "Dataset path": data.get("path"),126    }127    visible_details = {k: v for k, v in details.items() if v not in (None, "")}128    if visible_details:129        st.table(pd.DataFrame(visible_details.items(), columns=["Field", "Value"]))130 131    if data.get("abstract"):132        with st.expander("Abstract", expanded=True):133            st.write(data["abstract"])134 135 136def show_csv(path):137    max_rows = st.sidebar.slider("CSV preview rows", 20, 500, 100, step=20)138    df = pd.read_csv(path, low_memory=False, nrows=max_rows)139    st.dataframe(140        df,141        use_container_width=True,142        height=620,143    )144    st.caption(f"Previewing first {len(df):,} rows and {len(df.columns):,} columns.")145 146 147def show_image(path):148    image = Image.open(path)149    st.image(image, use_container_width=True)150    st.caption(f"Shape: {image.height} x {image.width}")151 152 153def show_json(path):154    with open(path, encoding="utf-8") as f:155        content = json.load(f)156    st.json(content, expanded=False)157 158 159def show_raster(path):160    with rasterio.open(path) as src:161        summary = {162            "Shape": f"{src.height} x {src.width}",163            "Bands": src.count,164            "Datatype": ", ".join(src.dtypes),165            "NoData value": src.nodata,166            "CRS": str(src.crs),167            "Bounds": str(src.bounds),168            "Transform": str(src.transform),169        }170    st.table(pd.DataFrame(summary.items(), columns=["Field", "Value"]))171 172 173def show_text(path):174    max_chars = st.sidebar.slider("Text preview characters", 1_000, 100_000, 20_000, step=1_000)175    with open(path, encoding="utf-8", errors="replace") as f:176        content = f.read(max_chars + 1)177    truncated = len(content) > max_chars178    if truncated:179        content = content[:max_chars]180    st.code(content)181    if truncated:182        st.caption(f"Preview truncated at {max_chars:,} characters.")183 184 185def render_file(path):186    suffix = path.suffix.lower()187 188    st.subheader(path.name)189    st.table(pd.DataFrame(file_stats(path).items(), columns=["Field", "Value"]))190 191    try:192        if suffix == ".csv":193            show_csv(path)194        elif suffix in {".png", ".jpg", ".jpeg"}:195            show_image(path)196        elif suffix == ".json":197            show_json(path)198        elif suffix in {".tif", ".tiff"}:199            show_raster(path)200        else:201            show_text(path)202    except Exception as exc:203        st.error(f"Error previewing {path.name}: {exc}")204 205 206def select_dataset(datasets):207    selected_statuses = st.sidebar.multiselect(208        "Status",209        STATUS_OPTIONS,210        default=["PROCESSED"],211    )212 213    search = st.sidebar.text_input("Search dataset", "")214    needle = search.strip().lower()215 216    filtered = [217        item218        for item in datasets.values()219        if item.get("status") in selected_statuses220        and (221            not needle222            or needle in item["name"].lower()223            or needle in (item.get("title") or "").lower()224        )225    ]226    filtered.sort(key=lambda item: item["name"].lower())227 228    if not filtered:229        return None230 231    default_index = 0232    for idx, item in enumerate(filtered):233        if item["name"] == DEFAULT_DATASET:234            default_index = idx235            break236 237    return st.sidebar.selectbox(238        "Dataset",239        filtered,240        index=default_index,241        format_func=lambda item: item["name"],242    )243 244 245def select_file(dataset):246    files = dataset.get("files", [])247    if not files:248        return None249 250    search = st.sidebar.text_input("Search file", "")251    needle = search.strip().lower()252    filtered = [path for path in files if not needle or needle in path.lower()]253 254    if not filtered:255        st.sidebar.warning("No matching files.")256        return None257 258    options = ["Dataset overview"] + filtered259    default_index = options.index(DEFAULT_FILE) if DEFAULT_FILE in options else 0260 261    return st.sidebar.selectbox(262        "Processed file",263        options,264        index=default_index,265    )266 267 268def main():269    st.set_page_config(270        page_title="Standardization Viewer",271        page_icon=str(ICON_PATH) if ICON_PATH.exists() else None,272        layout="wide",273        initial_sidebar_state="expanded",274    )275 276    st.sidebar.title("Standardization Viewer")277    datasets, error = get_datasets()278    if error:279        st.error(error)280        return281 282    dataset = select_dataset(datasets)283    if dataset is None:284        st.warning("No datasets match the selected filters.")285        return286 287    selected_file = select_file(dataset)288 289    st.title("Standardization Viewer")290    render_dataset_info(dataset)291 292    if selected_file and selected_file != "Dataset overview":293        path = Path(dataset["processed_path"]) / selected_file294        st.divider()295        render_file(path)296 297 298if __name__ == "__main__":299    main()300