SafeVixAI/SafeVixAI-Dataset-Hub

1from __future__ import annotations2 3import argparse4import csv5from io import BytesIO6import json7import math8import shutil9import struct10import sys11import zipfile12from pathlib import Path13 14 15PROJECT_ROOT = Path(__file__).resolve().parents[1]16BACKEND_ROOT = PROJECT_ROOT / 'backend'17 18import importlib.util as _ilu19 20 21def _load_backend_module(rel_path: str, module_name: str):22    """Load a module from backend/ by explicit file path and register it in23    sys.modules under *module_name*.  This makes the import fully transparent24    to Pylance/Pyright (no opaque sys.path mutation) while still satisfying25    Python internals that need __module__ to be resolvable (e.g. dataclasses26    with slots=True)."""27    abs_path = BACKEND_ROOT / rel_path28    spec = _ilu.spec_from_file_location(module_name, abs_path)29    mod = _ilu.module_from_spec(spec)  # type: ignore[arg-type]30    sys.modules[module_name] = mod  # register BEFORE exec so __module__ resolves31    spec.loader.exec_module(mod)  # type: ignore[union-attr]32    return mod33 34 35_seed_viol = _load_backend_module("scripts/seed_violations.py", "scripts.seed_violations")36DEFAULT_RULES = _seed_viol.DEFAULT_RULES37OVERRIDE_COLUMNS = _seed_viol.OVERRIDE_COLUMNS38RULE_COLUMNS = _seed_viol.RULE_COLUMNS39_load_override_rows = _seed_viol._load_override_rows40_load_rule_rows = _seed_viol._load_rule_rows41_rule_to_row = _seed_viol._rule_to_row42_write_csv = _seed_viol._write_csv43 44_emerg_catalog = _load_backend_module("services/local_emergency_catalog.py", "services.local_emergency_catalog")45load_local_emergency_catalog = _emerg_catalog.load_local_emergency_catalog46 47 48 49CHATBOT_DATA_DIR = PROJECT_ROOT / 'chatbot_service' / 'data'50FRONTEND_OFFLINE_DIR = PROJECT_ROOT / 'frontend' / 'public' / 'offline-data'51BACKEND_CHALLAN_DIR = PROJECT_ROOT / 'backend' / 'datasets' / 'challan'52ROADS_DIR = CHATBOT_DATA_DIR / 'roads'53PMGSY_MAX_POINTS_PER_SEGMENT = 2454 55OFFLINE_CITY_CENTERS: dict[str, tuple[float, float]] = {56    'chennai': (13.0827, 80.2707),57    'coimbatore': (11.0168, 76.9558),58    'madurai': (9.9252, 78.1198),59    'thiruvananthapuram': (8.5241, 76.9366),60    'kochi': (9.9312, 76.2673),61    'bengaluru': (12.9716, 77.5946),62    'mumbai': (19.0760, 72.8777),63    'pune': (18.5204, 73.8567),64    'nagpur': (21.1458, 79.0882),65    'hyderabad': (17.3850, 78.4867),66    'delhi': (28.6139, 77.2090),67    'jaipur': (26.9124, 75.7873),68    'ahmedabad': (23.0225, 72.5714),69    'surat': (21.1702, 72.8311),70    'vadodara': (22.3072, 73.1812),71    'kolkata': (22.5726, 88.3639),72    'patna': (25.5941, 85.1376),73    'bhopal': (23.2599, 77.4126),74    'indore': (22.7196, 75.8577),75    'lucknow': (26.8467, 80.9462),76    'agra': (27.1767, 78.0081),77    'varanasi': (25.3176, 82.9739),78    'chandigarh': (30.7333, 76.7794),79    'visakhapatnam': (17.6868, 83.2185),80    'bhubaneswar': (20.2961, 85.8245),81}82CITY_RADIUS_METERS = 80_00083 84 85def sync_challan_assets() -> None:86    rules_source = CHATBOT_DATA_DIR / 'violations_seed.csv'87    overrides_source = CHATBOT_DATA_DIR / 'state_overrides.csv'88    rule_map = {rule.violation_code: _rule_to_row(rule) for rule in DEFAULT_RULES}89    if rules_source.exists():90        for row in _load_rule_rows(rules_source):91            rule_map[row['violation_code']] = row92    override_rows = _load_override_rows(overrides_source) if overrides_source.exists() else []93 94    sorted_rules = [rule_map[key] for key in sorted(rule_map)]95    sorted_overrides = sorted(96        override_rows,97        key=lambda row: (row['state_code'], row['violation_code'], row['vehicle_class']),98    )99 100    BACKEND_CHALLAN_DIR.mkdir(parents=True, exist_ok=True)101    FRONTEND_OFFLINE_DIR.mkdir(parents=True, exist_ok=True)102    _write_csv(BACKEND_CHALLAN_DIR / 'violations.csv', RULE_COLUMNS, sorted_rules)103    _write_csv(BACKEND_CHALLAN_DIR / 'state_overrides.csv', OVERRIDE_COLUMNS, sorted_overrides)104    _write_csv(FRONTEND_OFFLINE_DIR / 'violations.csv', RULE_COLUMNS, sorted_rules)105    _write_csv(FRONTEND_OFFLINE_DIR / 'state_overrides.csv', OVERRIDE_COLUMNS, sorted_overrides)106    print(f'Challan assets synced: rules={len(sorted_rules)} overrides={len(sorted_overrides)}')107 108 109def sync_first_aid_bundle() -> None:110    """Always sync first-aid.json from frontend (canonical 20-article source) to chatbot data.111 112    The chatbot_service/data/first_aid.json was historically only 4 entries.113    The frontend/public/offline-data/first-aid.json contains the full 20 WHO-based articles114    and is the ground truth. This function overwrites unconditionally so the chatbot is never115    left with the stale 4-entry version.116    """117    source = FRONTEND_OFFLINE_DIR / 'first-aid.json'118    target = CHATBOT_DATA_DIR / 'first_aid.json'119    if not source.exists():120        print(f'WARNING: first-aid.json source not found at {source} — skipping sync')121        return122    shutil.copyfile(source, target)123    print(f'Synced first aid bundle ({source.stat().st_size:,} bytes) -> {target}')124 125 126def build_emergency_geojson() -> None:127    catalog = load_local_emergency_catalog(PROJECT_ROOT)128    features = []129    for entry in catalog:130        city, distance = _nearest_city(entry.lat, entry.lon)131        if city is None or distance > CITY_RADIUS_METERS:132            continue133        features.append(134            {135                'type': 'Feature',136                'id': entry.id,137                'geometry': {'type': 'Point', 'coordinates': [entry.lon, entry.lat]},138                'properties': {139                    'city': city.title(),140                    'name': entry.name,141                    'category': entry.category,142                    'sub_category': entry.sub_category,143                    'phone': entry.phone,144                    'phone_emergency': entry.phone_emergency,145                    'address': entry.address,146                    'has_trauma': entry.has_trauma,147                    'has_icu': entry.has_icu,148                    'is_24hr': entry.is_24hr,149                    'source': entry.source,150                },151            }152        )153 154    payload = {155        'type': 'FeatureCollection',156        'properties': {157            'generated_from': 'chatbot_service/data local CSV catalog',158            'feature_count': len(features),159            'cities': [city.title() for city in OFFLINE_CITY_CENTERS],160        },161        'features': features,162    }163    output_path = FRONTEND_OFFLINE_DIR / 'india-emergency.geojson'164    output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8')165    print(f'Emergency GeoJSON written: features={len(features)} path={output_path}')166 167 168def export_pmgsy_geojson() -> None:169    source = ROADS_DIR / 'pmgsy-geosadak-master.zip'170    target = ROADS_DIR / 'pmgsy_roads.geojson'171    if not source.exists():172        print('PMGSY archive not found; skipping pmgsy_roads.geojson export')173        return174 175    planned_states: list[str] = []176    skipped_archives: list[str] = []177    feature_count = 0178 179    with zipfile.ZipFile(source) as outer, target.open('w', encoding='utf-8') as handle:180        planned_states = _list_pmgsy_state_members(outer)181        skipped_archives = _list_pmgsy_split_archives(outer)182        properties = {183            'generated_from': source.name,184            'geometry_generalization': f'max {PMGSY_MAX_POINTS_PER_SEGMENT} points per segment',185            'planned_states': planned_states,186            'skipped_archives': skipped_archives,187        }188        handle.write('{"type":"FeatureCollection","properties":')189        json.dump(properties, handle, ensure_ascii=False, separators=(',', ':'))190        handle.write(',"features":[')191 192        is_first_feature = True193        exported_states: list[str] = []194        for state_name, archive_bytes in _iter_pmgsy_state_archives(outer):195            try:196                shp_bytes, dbf_bytes = _read_shapefile_bundle(archive_bytes)197            except ValueError:198                continue199 200            exported_states.append(state_name)201            for row, geometry in zip(_iter_dbf_rows(dbf_bytes), _iter_polyline_geometries(shp_bytes)):202                if geometry is None:203                    continue204                feature = {205                    'type': 'Feature',206                    'id': f'pmgsy-{state_name}-{row.get("ER_ID") or feature_count + 1}',207                    'geometry': geometry,208                    'properties': _build_pmgsy_properties(row, state_name),209                }210                if not is_first_feature:211                    handle.write(',')212                json.dump(feature, handle, ensure_ascii=False, separators=(',', ':'))213                is_first_feature = False214                feature_count += 1215 216        handle.write(']}')217 218    print(219        'PMGSY GeoJSON exported: '220        f'rows={feature_count} states={len(exported_states)} skipped={len(skipped_archives)} path={target}'221    )222 223 224def export_national_highways_csv() -> None:225    target = ROADS_DIR / 'national_highways.csv'226    if target.exists() and target.stat().st_size > 0:227        print(f'National highways CSV already present: {target}')228        return229 230    candidates = sorted(231        path for path in ROADS_DIR.glob('*.csv')232        if path.name != target.name and any(token in path.stem.lower() for token in ('nh', 'highway', 'nhai'))233    )234    if not candidates:235        summary_rows = _build_road_summary_rows()236        if not summary_rows:237            print('No usable local road CSVs found; skipping national_highways.csv export')238            return239 240        target.parent.mkdir(parents=True, exist_ok=True)241        with target.open('w', encoding='utf-8', newline='') as handle:242            writer = csv.DictWriter(243                handle,244                fieldnames=[245                    'source_file',246                    'geography_level',247                    'geography_name',248                    'period',249                    'metric_name',250                    'value',251                    'unit',252                    'notes',253                ],254            )255            writer.writeheader()256            writer.writerows(summary_rows)257        print(258            'National highways CSV synthesized from local road tables: '259            f'rows={len(summary_rows)} path={target}'260        )261        return262 263    shutil.copyfile(candidates[0], target)264    print(f'National highways CSV copied from {candidates[0].name} to {target}')265 266 267def _nearest_city(lat: float, lon: float) -> tuple[str | None, float]:268    best_city = None269    best_distance = float('inf')270    for city, (city_lat, city_lon) in OFFLINE_CITY_CENTERS.items():271        distance = _distance_meters(lat, lon, city_lat, city_lon)272        if distance < best_distance:273            best_city = city274            best_distance = distance275    return best_city, best_distance276 277 278def _distance_meters(lat1: float, lon1: float, lat2: float, lon2: float) -> float:279    radius = 6_371_000280    phi1 = math.radians(lat1)281    phi2 = math.radians(lat2)282    delta_phi = math.radians(lat2 - lat1)283    delta_lambda = math.radians(lon2 - lon1)284    a = (285        math.sin(delta_phi / 2) ** 2286        + math.cos(phi1) * math.cos(phi2) * math.sin(delta_lambda / 2) ** 2287    )288    return 2 * radius * math.atan2(math.sqrt(a), math.sqrt(1 - a))289 290 291def _list_pmgsy_state_members(outer: zipfile.ZipFile) -> list[str]:292    return [293        Path(member).stem294        for member in sorted(name for name in outer.namelist() if '/Road_DRRP/' in name and name.endswith('.zip'))295        if not member.endswith('-split.zip')296    ]297 298 299def _list_pmgsy_split_archives(outer: zipfile.ZipFile) -> list[str]:300    return [301        Path(member).stem302        for member in sorted(name for name in outer.namelist() if '/Road_DRRP/' in name and name.endswith('-split.zip'))303    ]304 305 306def _iter_pmgsy_state_archives(outer: zipfile.ZipFile):307    for member in sorted(name for name in outer.namelist() if '/Road_DRRP/' in name and name.endswith('.zip')):308        if member.endswith('-split.zip'):309            continue310        yield Path(member).stem, outer.read(member)311 312 313def _read_shapefile_bundle(archive_bytes: bytes) -> tuple[bytes, bytes]:314    with zipfile.ZipFile(BytesIO(archive_bytes)) as archive:315        shp_names = [name for name in archive.namelist() if name.lower().endswith('.shp')]316        dbf_names = [name for name in archive.namelist() if name.lower().endswith('.dbf')]317        if not shp_names or not dbf_names:318            raise ValueError('Missing shapefile members')319        return archive.read(shp_names[0]), archive.read(dbf_names[0])320 321 322def _iter_dbf_rows(dbf_bytes: bytes) -> list[dict[str, object]]:323    header_length = struct.unpack('<H', dbf_bytes[8:10])[0]324    record_length = struct.unpack('<H', dbf_bytes[10:12])[0]325    field_specs = []326    pos = 32327    offset = 1328    while pos < header_length - 1:329        field = dbf_bytes[pos:pos + 32]330        if field[0] == 0x0D:331            break332        field_specs.append(333            (334                field[:11].split(b'\x00', 1)[0].decode('ascii', 'ignore'),335                chr(field[11]),336                field[16],337                field[17],338                offset,339            )340        )341        offset += field[16]342        pos += 32343 344    records = struct.unpack('<I', dbf_bytes[4:8])[0]345    row_start = header_length346    for _ in range(records):347        record = dbf_bytes[row_start:row_start + record_length]348        row_start += record_length349        if not record or record[0:1] == b'*':350            continue351        row: dict[str, object] = {}352        for name, field_type, field_len, decimals, value_offset in field_specs:353            raw = record[value_offset:value_offset + field_len]354            text = raw.decode('latin1', 'ignore').strip()355            if not text:356                continue357            if field_type == 'N':358                if decimals:359                    try:360                        row[name] = float(text)361                    except ValueError:362                        row[name] = text363                else:364                    try:365                        row[name] = int(text)366                    except ValueError:367                        row[name] = text368            else:369                row[name] = text370        yield row371 372 373def _iter_polyline_geometries(shp_bytes: bytes) -> list[dict[str, object] | None]:374    pos = 100375    total_size = len(shp_bytes)376    while pos + 8 <= total_size:377        content_length_words = struct.unpack('>i', shp_bytes[pos + 4:pos + 8])[0]378        record_end = pos + 8 + content_length_words * 2379        record = shp_bytes[pos + 8:record_end]380        pos = record_end381        if len(record) < 44:382            yield None383            continue384 385        shape_type = struct.unpack('<i', record[:4])[0]386        if shape_type == 0:387            yield None388            continue389        if shape_type not in {3, 13, 23}:390            yield None391            continue392 393        num_parts = struct.unpack('<i', record[36:40])[0]394        num_points = struct.unpack('<i', record[40:44])[0]395        parts_offset = 44396        points_offset = parts_offset + 4 * num_parts397        parts = [398            struct.unpack('<i', record[parts_offset + index * 4:parts_offset + (index + 1) * 4])[0]399            for index in range(num_parts)400        ]401        points = [402            struct.unpack('<2d', record[points_offset + index * 16:points_offset + (index + 1) * 16])403            for index in range(num_points)404        ]405 406        coordinates = []407        for index, start in enumerate(parts):408            end = parts[index + 1] if index + 1 < len(parts) else len(points)409            line = _downsample_line(points[start:end], max_points=PMGSY_MAX_POINTS_PER_SEGMENT)410            if len(line) < 2:411                continue412            coordinates.append([[round(lon, 6), round(lat, 6)] for lon, lat in line])413 414        if not coordinates:415            yield None416        elif len(coordinates) == 1:417            yield {'type': 'LineString', 'coordinates': coordinates[0]}418        else:419            yield {'type': 'MultiLineString', 'coordinates': coordinates}420 421 422def _downsample_line(points: list[tuple[float, float]], *, max_points: int) -> list[tuple[float, float]]:423    if len(points) <= max_points:424        return points425    last_index = len(points) - 1426    indexes = {427        0,428        last_index,429        *(430            min(last_index, round(step * last_index / (max_points - 1)))431            for step in range(1, max_points - 1)432        ),433    }434    return [points[index] for index in sorted(indexes)]435 436 437def _build_pmgsy_properties(row: dict[str, object], state_name: str) -> dict[str, object]:438    props: dict[str, object] = {'pmgsy_state': state_name}439    field_map = {440        'ER_ID': 'er_id',441        'STATE_ID': 'state_id',442        'BLOCK_ID': 'block_id',443        'DISTRICT_I': 'district_id',444        'DRRP_ROAD_': 'road_code',445        'RoadCatego': 'road_category',446        'RoadName': 'road_name',447        'RoadOwner': 'road_owner',448    }449    for source_key, target_key in field_map.items():450        value = row.get(source_key)451        if value not in (None, ''):452            props[target_key] = value453    return props454 455 456def _build_road_summary_rows() -> list[dict[str, str]]:457    rows: list[dict[str, str]] = []458    for path in sorted(ROADS_DIR.glob('*.csv')):459        if path.name in {'national_highways.csv', 'tolls-with-metadata.csv'}:460            continue461        if path.name.endswith('-metadata-hotosm_ind_roads_lines_geojson-zip.csv'):462            continue463        rows.extend(_normalize_road_summary_table(path))464    return rows465 466 467def _normalize_road_summary_table(path: Path) -> list[dict[str, str]]:468    with path.open('r', encoding='utf-8-sig', newline='') as handle:469        reader = csv.DictReader(handle)470        if reader.fieldnames is None:471            return []472 473        geography_column = _detect_geography_column(reader.fieldnames)474        serial_columns = {'Sr. No.', 'Sl. No.', 'Sl.No.', 'S.No.', 'S. No.'}475        notes = (476            'Generated from local road programme CSVs because no direct NHAI/NH master CSV '477            'was present in chatbot_service/data/roads.'478        )479        rows: list[dict[str, str]] = []480        for raw in reader:481            geography_name = (raw.get(geography_column) or '').strip() if geography_column else ''482            if not geography_name:483                continue484            for column, value in raw.items():485                if column in serial_columns or column == geography_column:486                    continue487                metric_value = _normalize_metric_value(value or '')488                if metric_value is None:489                    continue490                metric_name, period = _split_metric_column(column)491                rows.append(492                    {493                        'source_file': path.name,494                        'geography_level': 'district' if geography_column == 'District Name' else 'state',495                        'geography_name': geography_name,496                        'period': period,497                        'metric_name': metric_name,498                        'value': metric_value,499                        'unit': 'km_or_count',500                        'notes': notes,501                    }502                )503        return rows504 505 506def _detect_geography_column(fieldnames: list[str]) -> str | None:507    candidates = ['District Name', 'State/UT', 'State', 'District', 'State/UT ']508    for candidate in candidates:509        if candidate in fieldnames:510            return candidate511    return None512 513 514def _normalize_metric_value(value: str) -> str | None:515    cleaned = value.strip()516    if not cleaned or cleaned.upper() in {'NA', 'N/A', '-'}:517        return None518    try:519        return str(int(cleaned))520    except ValueError:521        try:522            return str(float(cleaned))523        except ValueError:524            return None525 526 527def _split_metric_column(column: str) -> tuple[str, str]:528    cleaned = column.strip()529    period_match = None530    for token in ('2024-25', '2023-24', '2022-23', '2021-22', '2020-21', '2019-20'):531        if token in cleaned:532            period_match = token533            break534    if period_match is None:535        return cleaned, ''536 537    metric_name = cleaned.replace(period_match, '').replace(' - ', ' ').replace('(as on 14.07.2022)', '').strip()538    metric_name = ' '.join(metric_name.split()) or cleaned539    return metric_name, period_match540 541 542def main() -> None:543    parser = argparse.ArgumentParser(description='Build app-facing assets from chatbot_service/data local datasets.')544    parser.add_argument('--skip-pmgsy', action='store_true', help='Skip extracting PMGSY shapefiles into GeoJSON.')545    args = parser.parse_args()546 547    sync_challan_assets()548    sync_first_aid_bundle()549    build_emergency_geojson()550    export_national_highways_csv()551    if not args.skip_pmgsy:552        export_pmgsy_geojson()553 554 555if __name__ == '__main__':556    main()557