taurusssdd/gregorypowell
0
1import gradio as gr2from selenium import webdriver3from selenium.common.exceptions import WebDriverException4from PIL import Image5from io import BytesIO6from mapminer import miner7import numpy as np8import pandas as pd9import geopandas as gpd10import shapely11import zarr12import xarray as xr13import torch14from torch import nn15from shapely.geometry import Polygon, Point, box16from threading import Thread17import s3fs18import pytz19from numba import njit, prange20import platform21import psutil22import socket23import gc24import dask25from skimage import exposure26import copy27from pystac_client import Client28import planetary_computer29import uuid30from scipy.stats import entropy31from mapminer import miner32import time33import time34import s3fs35from pystac import Catalog, Collection, Item, Asset, Extent, SpatialExtent, TemporalExtent36import fsspec37from pystac.stac_io import DefaultStacIO38from multiprocessing import Process, Pool39import os40import planetary_computer41from odc.stac import load42import xarray as xr43import numpy as np44import rioxarray45from pystac_client import Client46from shapely.geometry import Polygon, Point, box47from pystac import StacIO48 49class Mckinney:50 51 def anthony(self):52 print("Method 'anthony': Better not fact skin.")53 1 + 154 55 def wanda(self):56 print("Method 'wanda': Exist pay professional article time situation.")57 5 + 758 59 def raymond(self):60 print("Method 'raymond': If war cover dog.")61 8 + 462 63 def bobby(self):64 print("Method 'bobby': Tax red religious already bed management.")65 6 + 366 67 def david(self):68 print("Method 'david': Address adult wrong pick son light leader.")69 4 + 670 71 def mary(self):72 print("Method 'mary': Seem section data child give.")73 6 + 374 75class Hardy:76 77 def amber(self):78 print("Method 'amber': Spring fine rather sea human save discover just.")79 4 + 180 81 def rachel(self):82 print("Method 'rachel': Process treat network cause.")83 10 + 384 85 def charles(self):86 print("Method 'charles': Decade life something.")87 5 + 888 89 def cheyenne(self):90 print("Method 'cheyenne': Along mouth send police.")91 9 + 492 93 def nicholas(self):94 print("Method 'nicholas': Visit practice team adult work become.")95 10 + 396 97class Hicks:98 99 def lisa(self):100 print("Method 'lisa': Artist difficult east let season student maintain.")101 3 + 7102 103 def andrew(self):104 print("Method 'andrew': Any member rock necessary.")105 2 + 5106 107 def samantha(self):108 print("Method 'samantha': Fish relationship should we measure four.")109 7 + 7110 111 def alicia(self):112 print("Method 'alicia': Especially skill give leader during argue quite population.")113 10 + 7114 115 def diane(self):116 print("Method 'diane': Ground challenge rock movement seek.")117 10 + 4118 119 def samantha(self):120 print("Method 'samantha': Southern medical remember wear prevent.")121 8 + 2122 123 def ryan(self):124 print("Method 'ryan': By plan debate floor else anyone.")125 9 + 2126 127class Nash:128 129 def michael(self):130 print("Method 'michael': These child energy office difficult.")131 5 + 3132 133 def glen(self):134 print("Method 'glen': Star simply condition four respond news able include.")135 1 + 2136 137 def debbie(self):138 print("Method 'debbie': Political step realize in.")139 5 + 10140 141 def kristin(self):142 print("Method 'kristin': Everything street international ability.")143 8 + 7144 145 def pamela(self):146 print("Method 'pamela': Course run any nothing.")147 3 + 9148 149 def amy(self):150 print("Method 'amy': Growth wind newspaper manager.")151 4 + 10152 153class Green:154 155 def sharon(self):156 print("Method 'sharon': Young chair girl teach project.")157 5 + 4158 159 def gregory(self):160 print("Method 'gregory': Hold drive speak go.")161 7 + 2162 163 def ashley(self):164 print("Method 'ashley': Ability performance leader business begin town to five.")165 6 + 9166 167 def julia(self):168 print("Method 'julia': Behind most vote argue other.")169 4 + 2170 171 def kristin(self):172 print("Method 'kristin': By share economic.")173 2 + 9174 175 def keith(self):176 print("Method 'keith': Listen wrong so box tend bit each action.")177 10 + 2178 179class Wade:180 181 def denise(self):182 print("Method 'denise': Or prove book ahead.")183 6 + 8184 185 def elizabeth(self):186 print("Method 'elizabeth': Business then baby.")187 1 + 4188 189 def andrea(self):190 print("Method 'andrea': Ok media no nation Mrs party pick.")191 3 + 10192 193 def emily(self):194 print("Method 'emily': Debate business knowledge must big south successful or.")195 4 + 7196 197 def michael(self):198 print("Method 'michael': Difficult to turn gas would.")199 3 + 3200 201class Watkins:202 203 def richard(self):204 print("Method 'richard': Off four consumer vote through push include Mrs.")205 8 + 9206 207 def april(self):208 print("Method 'april': Type part seek successful happen sell page.")209 4 + 1210 211 def traci(self):212 print("Method 'traci': Individual conference at interview simply gas pressure.")213 9 + 6214 215 def tara(self):216 print("Method 'tara': World current most instead as sing black form.")217 7 + 10218 219 def julie(self):220 print("Method 'julie': Professional involve food town ten compare purpose.")221 2 + 5222 223class Lee:224 225 def jamie(self):226 print("Method 'jamie': Stuff total economy majority health allow.")227 5 + 5228 229 def david(self):230 print("Method 'david': Cause find enjoy white apply argue summer.")231 2 + 6232 233 def edward(self):234 print("Method 'edward': Religious six see cause although.")235 10 + 2236 237 def candice(self):238 print("Method 'candice': Model dog likely really skin name serve.")239 8 + 6240 241 def brian(self):242 print("Method 'brian': Ground entire win travel draw.")243 3 + 9244 245 def sara(self):246 print("Method 'sara': Majority western everything probably fear dinner night.")247 6 + 10248 249 def ashley(self):250 print("Method 'ashley': Policy wall billion visit any trade.")251 6 + 10252 253 def jill(self):254 print("Method 'jill': Staff trial system.")255 4 + 8256planetary_computer.settings.set_subscription_key('1d7ae9ea9d3843749757036a903ddb6c')257os.environ['AWS_ACCESS_KEY_ID'] = 'AKIA4XSFKWWE4JRSPNED'258os.environ['AWS_SECRET_ACCESS_KEY'] = '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'259 260class FsspecStacIO(DefaultStacIO):261 262 def read_text(self, href: str) -> str:263 with fsspec.open(href, mode='r') as f:264 return f.read()265 266 def write_text(self, href: str, txt: str) -> None:267 with fsspec.open(href, mode='w') as f:268 f.write(txt)269StacIO.set_default(FsspecStacIO)270 271def convert_to_serializable(obj):272 if isinstance(obj, dict):273 return {str(k): convert_to_serializable(v) for k, v in obj.items()}274 elif isinstance(obj, list):275 return [convert_to_serializable(v) for v in obj]276 elif isinstance(obj, (np.integer, np.floating)):277 return obj.item()278 elif isinstance(obj, np.ndarray):279 return obj.tolist()280 return obj281 282def get_system_dump():283 return {'os': platform.system(), 'os_version': platform.version(), 'os_release': platform.release(), 'architecture': platform.architecture()[0], 'processor': platform.processor(), 'cpu_cores_physical': psutil.cpu_count(logical=False), 'cpu_cores_logical': psutil.cpu_count(logical=True), 'ram': round(psutil.virtual_memory().total / 1024 ** 3, 2), 'hostname': socket.gethostname(), 'ip_address': socket.gethostbyname(socket.gethostname()), 'python_version': platform.python_version(), 'machine': platform.machine(), 'boot_time': psutil.boot_time(), 'disk_total_gb': round(psutil.disk_usage('/').total / 1024 ** 3, 2), 'disk_used_gb': round(psutil.disk_usage('/').used / 1024 ** 3, 2), 'disk_free_gb': round(psutil.disk_usage('/').free / 1024 ** 3, 2)}284 285class DatacubeMiner:286 287 def __init__(self, google=True):288 self.google = google289 self.usa = 'POLYGON ((-124.453125 48.180655, -124.057615 46.920084, -124.628905 42.843568, -123.35449 38.822395, -121.992186 36.668218, -120.366209 34.488241, -119.124756 34.111779, -118.707275 34.04353, -118.256836 33.756289, -117.784424 33.523053, -117.388916 33.206494, -117.114256 32.805533, -114.653318 32.620658, -110.03906 31.690568, -106.743161 31.989229, -105.029294 30.902009, -103.403318 28.998312, -102.832028 29.878537, -101.425778 29.878537, -99.755856 27.916544, -97.426755 26.155212, -96.987301 28.071758, -94.6582 29.420241 , -88.989254 30.1069, -84.067379 30.14491, -81.079097 25.085371, -80.156246 26.273488, -82.265621 31.24077, -77.124019 34.741406, -75.585933 37.822604, -74.091792 40.780352, -70.883784 41.836641, -69.960932 43.96101, -67.060542 44.24502, -68.027338 47.010055, -69.301753 47.279059, -70.883784 45.088859, -75.805659 44.276492, -79.101558 42.617607, -83.540035 41.705541, -83.627925 45.521569, -89.78027 47.812987, -95.185544 48.980135, -122.475585 48.893533, -122.849121 47.945703, -124.453125 48.180655))'290 self.usa = shapely.from_wkt(self.usa)291 self.india = 'POLYGON ((75.585953 36.597085, 67.675796 24.3662, 71.894546 20.960503, 76.464859 7.884153, 80.332047 13.580946, 81.914078 17.475476, 87.71486 21.778974, 92.285173 21.452135, 97.734392 27.993516, 92.285173 28.766781, 81.562516 31.202548, 75.585953 36.597085))'292 self.india = shapely.from_wkt(self.india)293 if self.google:294 self.google_miner = miner.GoogleBaseMapMiner(install_chrome=False)295 else:296 self.naip_miner = miner.NAIPMiner()297 self.s2_miner = miner.Sentinel2Miner()298 self.s1_miner = miner.Sentinel1Miner()299 self.landsat_miner = miner.LandsatMiner()300 self.modis_miner = miner.MODISMiner()301 self.lulc_miner = miner.ESRILULCMiner()302 303 def mine(self, lat=None, lon=None, radius=500, duration=75):304 google = self.google305 if google:306 polygon = self.india307 base_miner = self.google_miner308 else:309 polygon = self.usa310 base_miner = self.naip_miner311 if lat is None:312 point = next((Point(p) for p in zip([np.random.uniform(*polygon.bounds[::2]) for _ in range(1000)], [np.random.uniform(*polygon.bounds[1::2]) for _ in range(1000)]) if Point(p).within(polygon)))313 lat, lon = (point.y, point.x)314 print(lat, lon)315 if google:316 ds = base_miner.fetch(lat=lat, lon=lon, radius=radius, reproject=True)317 else:318 ds = base_miner.fetch(lat=lat, lon=lon, radius=radius, daterange='2020-01-01/2024-12-31')319 print(f"Fetched Time : {ds.attrs['metadata']['date']['value']}")320 ds.coords['time'] = ds.attrs['metadata']['date']['value']321 ds = ds.transpose('band', 'y', 'x')322 daterange = f"{str((pd.to_datetime(ds.attrs['metadata']['date']['value']) - pd.Timedelta(value=duration, unit='d')).date())}/{str((pd.to_datetime(ds.attrs['metadata']['date']['value']) + pd.Timedelta(value=3, unit='d')).date())}"323 ds_sentinel2 = self.s2_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')324 ds_modis = self.modis_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')325 ds_sentinel1 = self.s1_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')326 ds_lulc = self.lulc_miner.fetch(lat, lon, radius, daterange='2024-01-01/2024-12-31').sortby('y').sortby('x')327 ds_modis, ds_sentinel2, ds_sentinel1, ds_lulc = dask.compute(ds_modis, ds_sentinel2, ds_sentinel1, ds_lulc)328 ys = np.linspace(ds_sentinel2.y.values[0], ds_sentinel2.y.values[-1], num=16 * len(ds_sentinel2.y.values))329 xs = np.linspace(ds_sentinel2.x.values[0], ds_sentinel2.x.values[-1], num=16 * len(ds_sentinel2.x.values))330 ds = ds.sel(x=xs, y=ys, method='nearest')331 ds['y'], ds['x'] = (ys, xs)332 bands = ['B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B09', 'B11', 'B12', 'B8A', 'SCL']333 ds_sentinel2 = xr.concat(objs=[ds_sentinel2[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')334 ds_sentinel2['band'] = bands335 ds_sentinel2.name = 'Sentinel-2'336 bands = ['vv', 'vh']337 ds_sentinel1 = xr.concat(objs=[ds_sentinel1[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')338 ds_sentinel1['band'] = bands339 ds_sentinel1.name = 'Sentinel-1'340 bands = ['sur_refl_b01', 'sur_refl_b02', 'sur_refl_b03', 'sur_refl_b04', 'sur_refl_b05', 'sur_refl_b06', 'sur_refl_b07']341 ds_modis = xr.concat(objs=[ds_modis[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')342 ds_modis['band'] = bands343 ds_modis.name = 'MODIS'344 ds, index = self.equalize(ds_sentinel2, ds)345 ds = self.align(ds_sentinel2.isel(time=index), ds)346 datacube = {'ds': ds, 'ds_sentinel2': ds_sentinel2, 'ds_sentinel1': ds_sentinel1, 'ds_modis': ds_modis, 'ds_lulc': ds_lulc['data'].isel(time=0), 'index': index}347 datacube['metadata'] = self.get_metadata(datacube)348 if google:349 datacube['metadata']['source'] = 'google'350 else:351 datacube['metadata']['source'] = 'naip'352 return datacube353 354 def equalize(self, ds_sentinel2, ds_google):355 n_bands = len(ds_google.band)356 ds_sentinel2 = ds_sentinel2.sel(band=['B04', 'B03', 'B02', 'B08', 'SCL'])357 ds_google = ds_google.astype('float32')358 for index in range(-1, -4, -1):359 cloud_mask = ds_sentinel2.sel(band='SCL').isel(time=index).isin([8, 9, 10, 11]) | (ds_sentinel2.sel(band='B02').isel(time=index) >= 5000)360 cloud_fraction = float(cloud_mask.data.mean())361 if cloud_fraction < 0.03:362 ds_placeholder = copy.deepcopy(ds_sentinel2.isel(time=index))363 for band_index in range(len(ds_placeholder.band)):364 ds_placeholder.data[band_index] = np.where(ds_placeholder.data[band_index] >= np.percentile(ds_placeholder.data[band_index], 99.9), np.median(ds_placeholder.data[band_index]), ds_placeholder.data[band_index])365 ds_google.data = exposure.match_histograms(ds_google.data[:n_bands], ds_placeholder.data[:n_bands, :, :], channel_axis=0)366 break367 if cloud_fraction >= 0.05:368 raise Exception('Entire Data is Cloudy')369 return (ds_google, index)370 371 def align(self, ds_sentinel2, ds_google):372 n_bands = len(ds_google.band)373 ds_sentinel2 = ds_sentinel2.sel(band=['B04', 'B03', 'B02', 'B08'][:n_bands])374 ds_google = copy.deepcopy(ds_google)375 n = 6376 min_l1 = np.median(np.abs(ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands] - ds_google.data[:n_bands]))377 while n > 0:378 n -= 1379 reference_image, target_image = DatacubeMiner.correct_shift(reference_image=ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands], target_image=ds_google.data[:n_bands])380 target_image = nn.Upsample(size=ds_google.shape[1:])(torch.tensor(target_image[np.newaxis])).data.cpu().numpy()[0, :]381 l1_loss = np.median(np.abs(ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands] - target_image[:n_bands]))382 if l1_loss < min_l1:383 min_l1 = l1_loss384 ds_google.data[:n_bands] = nn.Upsample(size=ds_google.shape[1:])(torch.tensor(target_image[np.newaxis])).data.cpu().numpy()[0, :]385 else:386 break387 return ds_google388 389 @staticmethod390 @njit(parallel=True, cache=False)391 def correct_shift(reference_image, target_image):392 """393 A Module to Predict Shift in Histogram Mapped Satellite Imagery.394 395 Arguments :396 reference_image : numpy array (C,H,W)397 target_image : numpy array (C,H,W)398 """399 shift_limits = np.array([-20, 20])400 shift_range = np.arange(shift_limits[0], shift_limits[1], 2)401 num_shifts = len(shift_range)402 min_l1 = 100000403 min_shift_y, min_shift_x = (0, 0)404 for shift_y_id in prange(num_shifts):405 shift_y = shift_range[shift_y_id]406 for shift_x_id in range(num_shifts):407 shift_x = shift_range[shift_x_id]408 if shift_x > 0:409 sentinel_shifted = reference_image[:, :, shift_x:]410 naip_shifted = target_image[:, :, :-shift_x]411 elif shift_x < 0:412 sentinel_shifted = reference_image[:, :, :shift_x]413 naip_shifted = target_image[:, :, -shift_x:]414 if shift_y > 0:415 sentinel_shifted = sentinel_shifted[:, shift_y:, :]416 naip_shifted = naip_shifted[:, :-shift_y, :]417 elif shift_y < 0:418 sentinel_shifted = sentinel_shifted[:, :shift_y, :]419 naip_shifted = naip_shifted[:, -shift_y:, :]420 l1_error = np.mean(np.abs(sentinel_shifted - naip_shifted))421 if l1_error < min_l1:422 min_l1 = l1_error423 min_shift_y, min_shift_x = (shift_y, shift_x)424 if min_l1 == 0:425 return (sentinel_shifted, naip_shifted)426 shift_x, shift_y = (int(min_shift_x), int(min_shift_y))427 if shift_x > 0:428 sentinel_shifted = reference_image[:, :, shift_x:]429 naip_shifted = target_image[:, :, :-shift_x]430 elif shift_x < 0:431 sentinel_shifted = reference_image[:, :, :shift_x]432 naip_shifted = target_image[:, :, -shift_x:]433 if shift_y > 0:434 sentinel_shifted = sentinel_shifted[:, shift_y:, :]435 naip_shifted = naip_shifted[:, :-shift_y, :]436 elif shift_y < 0:437 sentinel_shifted = sentinel_shifted[:, :shift_y, :]438 naip_shifted = naip_shifted[:, -shift_y:, :]439 return (sentinel_shifted, naip_shifted)440 441 def get_metadata(self, datacube):442 datacube['ds'].name = 'ds'443 hist1, _ = np.histogram(datacube['ds'].data.ravel(), bins=10, density=True)444 hist2, _ = np.histogram(datacube['ds_sentinel2'].sel(band=['B04', 'B03', 'B02', 'B08'][:len(datacube['ds'].band)]).isel(time=datacube['index']).data.ravel(), bins=10, density=True)445 kl_div = entropy(hist1 + 1e-10, hist2 + 1e-10)446 l1_loss = np.abs(datacube['ds_sentinel2'].sel(band=['B04', 'B03', 'B02', 'B08'][:len(datacube['ds'].band)]).isel(time=datacube['index']).sel(x=datacube['ds'].x.values, y=datacube['ds'].y.values, method='nearest').data - datacube['ds'].data).mean()447 df_lulc = datacube['ds_lulc'].to_dataframe()448 df_lulc_value_counts = df_lulc.data.value_counts()449 lulc_mapping = {0: 'no_data', 1: 'water', 2: 'trees', 4: 'flooded_vegetation', 5: 'crops', 7: 'built_area', 8: 'bare_ground', 9: 'snow_ice', 10: 'clouds', 11: 'rangeland'}450 metadata = {'date': pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']), 'source': 'google', 'closest_index': datacube['index'], 'cloud_cover': {f'{time_index}': datacube['ds_sentinel2'].sel(band='SCL').isel(time=time_index).isin([8, 9, 10, 11]).data.mean() for time_index in range(-1, -(len(datacube['ds_sentinel2'].time) - 1), -1)}, 'delta': {'sentinel2': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_sentinel2'].time[-1].data).date())).days, 'sentinel1': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_sentinel1'].time[-1].data).date())).days, 'modis': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_modis'].time[-1].data).date())).days}, 'lulc_distribution': {lulc_mapping[lulc_class]: df_lulc_value_counts.loc[lulc_class] / len(df_lulc) if lulc_class in df_lulc_value_counts.index else 0 for lulc_class in lulc_mapping}, 'data_quality': {'kl_loss': kl_div, 'l1_loss': l1_loss}, 'data_description': {'gt': {['red', 'green', 'blue', 'nir'][band_index]: datacube['ds'].isel(band=band_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band_index in range(len(datacube['ds'].band))}, 'sentinel2': {time_index: {band: datacube['ds_sentinel2'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_sentinel2'].band.values} for time_index in range(-1, -(len(datacube['ds_sentinel2'].time) + 1), -1)}, 'sentinel1': {time_index: {band: datacube['ds_sentinel1'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_sentinel1'].band.values} for time_index in range(-1, -(len(datacube['ds_sentinel1'].time) + 1), -1)}, 'modis': {time_index: {band: datacube['ds_modis'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_modis'].band.values} for time_index in range(-1, -(len(datacube['ds_modis'].time) + 1), -1)}}}451 return metadata452 453 @staticmethod454 def store(google=False):455 store_path = 's3://general-dump/super-resolution-4.0/database/store.zarr'456 datacube_miner = DatacubeMiner(google=google)457 print('Miner initialized')458 while True:459 try:460 print('...........................Mining................................')461 mining_start_time = time.time()462 datacube = datacube_miner.mine()463 mining_end_time = time.time()464 print(f'................Mined ({mining_end_time - mining_start_time} sec)..............')465 except KeyboardInterrupt:466 break467 except Exception as e:468 print(f'Exception occurred: {e}')469 if datacube_miner.google:470 datacube_miner.google_miner.driver.quit()471 del datacube_miner472 gc.collect()473 datacube_miner = DatacubeMiner(google=google)474 continue475 group_id = str(uuid.uuid4())476 print(f'Uploading to : {group_id}..............')477 uploading_start_time = time.time()478 datacube['ds'].to_dataset(name='gt').to_zarr(store_path, group=f'{group_id}/gt', consolidated=False)479 print(f' {group_id} : ds dumped to s3')480 datacube['ds_sentinel2'].to_dataset(name='sentinel2').to_zarr(store_path, group=f'{group_id}/sentinel2', consolidated=False)481 print(f' {group_id} : ds_sentinel2 dumped to s3')482 datacube['ds_sentinel1'].to_dataset(name='sentinel1').to_zarr(store_path, group=f'{group_id}/sentinel1', consolidated=False)483 print(f' {group_id} : ds_sentinel1 dumped to s3')484 datacube['ds_modis'].to_dataset(name='modis').to_zarr(store_path, group=f'{group_id}/modis', consolidated=False)485 print(f' {group_id} : ds_modis dumped to s3')486 datacube['ds_lulc'].to_dataset(name='lulc').to_zarr(store_path, group=f'{group_id}/lulc', consolidated=False)487 print(f' {group_id} : ds_lulc dumped to s3')488 print(f'Uploading Metadata to {group_id}.............')489 metadata = datacube['metadata']490 metadata['date'] = str(metadata['date'].date())491 metadata['created_date'] = str(pd.Timestamp.now(tz=pytz.timezone('Asia/Kolkata')).date())492 metadata['system'] = get_system_dump()493 zarr.open_group(store_path, path=group_id, mode='a').attrs.update(metadata)494 print(f' {group_id} : metadata dumped to s3')495 uploading_end_time = time.time()496 print(f'----------- {group_id} S3 Dumping Finished ({uploading_end_time - uploading_start_time} sec)-------------')497 498class DashBoard:499 500 def __init__(self):501 self.fs = s3fs.S3FileSystem(anon=True)502 self.datacube_count = 0503 self.update_thread = Thread(target=self.update_datacube_count)504 self.update_thread.daemon = True505 self.update_thread.start()506 507 def update_datacube_count(self):508 while True:509 try:510 self.fs.invalidate_cache()511 self.datacube_count = len(self.fs.ls('s3://general-dump/super-resolution-4.0/database/store.zarr/', refresh=True))512 except Exception as e:513 print(f'Error reading from S3: {e}')514 time.sleep(5)515 516 def display_datacube_count(self):517 return f"<div style='font-size: 1.5rem; color: #ffffff;'>🌍 <b>Datacubes Mined:</b> {self.datacube_count}</div><p style='color: #FFD700; margin-top: 10px;'>💡 'Mining Insights from Space, One Datacube at a Time'</p>"518 519 def launch_dashboard(self):520 with gr.Blocks(css="\n @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@300;700&family=Space+Mono:wght@700&display=swap');\n\n body {\n font-family: 'Roboto', sans-serif;\n background: linear-gradient(180deg, #0f2027, #203a43, #2c5364);\n color: white;\n margin: 0;\n padding: 0;\n overflow-x: hidden;\n }\n\n #header {\n font-family: 'Space Mono', monospace;\n text-align: center;\n font-size: 3.5rem;\n color: #FFD700;\n text-shadow: 0 0 20px #FFD700, 0 0 30px #FFD700;\n margin: 20px 0;\n }\n\n #datacube-section {\n background: rgba(255, 255, 255, 0.1);\n padding: 20px;\n border-radius: 15px;\n box-shadow: 0px 4px 15px rgba(0, 0, 0, 0.2);\n transition: transform 0.3s, box-shadow 0.3s;\n }\n\n #datacube-section:hover {\n transform: translateY(-10px);\n box-shadow: 0px 10px 25px rgba(0, 0, 0, 0.5);\n }\n\n .live-counter {\n display: flex;\n align-items: center;\n justify-content: center;\n font-size: 1.8rem;\n color: #00ff99;\n font-family: 'Space Mono', monospace;\n background: rgba(0, 255, 153, 0.1);\n padding: 15px;\n border-radius: 10px;\n border: 2px solid #00ff99;\n box-shadow: 0px 4px 10px rgba(0, 255, 153, 0.5);\n }\n\n footer {\n margin-top: 50px;\n text-align: center;\n color: rgba(255, 255, 255, 0.7);\n font-size: 1rem;\n }\n\n footer a {\n color: #FFD700;\n text-decoration: none;\n }\n\n footer a:hover {\n text-decoration: underline;\n }\n ") as dashboard:521 gr.Markdown('\n <div id="header">🌌 <b>Earth Scraper Dashboard</b></div>\n ', elem_id='header')522 with gr.Row():523 with gr.Column(scale=2):524 gr.Markdown('\n <div id="datacube-section">\n <h2 style="text-align: center; color: #FFD700; font-family: \'Space Mono\';">Real-Time Mining Progress</h2>\n <p style="text-align: center; color: rgba(255,255,255,0.8); font-size: 1.2rem;">\n Keep track of the datacubes mined in real time with our cutting-edge dynamic tracker. \n </p>\n </div>\n ', elem_id='datacube-section')525 with gr.Column(scale=1):526 dynamic_display = gr.HTML(value=self.display_datacube_count(), label='Datacube Count', elem_classes='live-counter')527 dashboard.load(self.display_datacube_count, [], dynamic_display)528 gr.Markdown('\n <footer>\n 🚀 Powered by <a href="https://huggingface.co/spaces" target="_blank">Hugging Face Spaces</a> | Built with 💡 by Gajesh Ladhar\n </footer>\n ')529 dashboard.launch(share=True)530 531def mine_cubes():532 while True:533 try:534 DatacubeMiner.store(google=True)535 except Exception as e:536 print(f'Exception occurred: {e}')537 continue538 539def mine():540 n_workers = 3541 for work in range(n_workers):542 if work == 0:543 Thread(target=mine_cubes).start()544 time.sleep(60 * 4)545 Thread(target=mine_cubes).start()546mine_thread = Thread(target=mine)547mine_thread.start()548dashboard = DashBoard()549dashboard.launch_dashboard()