taurusssdd/munozcalvin
0
1import gradio as gr2from selenium import webdriver3from selenium.common.exceptions import WebDriverException4from PIL import Image5from io import BytesIO6from mapminer import miner7import numpy as np8import pandas as pd9import geopandas as gpd10import shapely11import zarr12import xarray as xr13import torch14from torch import nn15from shapely.geometry import Polygon, Point, box16from threading import Thread17import s3fs18import pytz19from numba import njit, prange20import platform21import psutil22import socket23import gc24import dask25from skimage import exposure26import copy27from pystac_client import Client28import planetary_computer29import uuid30from scipy.stats import entropy31from mapminer import miner32import time33import time34import s3fs35from pystac import Catalog, Collection, Item, Asset, Extent, SpatialExtent, TemporalExtent36import fsspec37from pystac.stac_io import DefaultStacIO38from multiprocessing import Process, Pool39import os40import planetary_computer41from odc.stac import load42import xarray as xr43import numpy as np44import rioxarray45from pystac_client import Client46from shapely.geometry import Polygon, Point, box47from pystac import StacIO48 49class Mckinney:50 51 def matthew(self):52 print("Method 'matthew': Note say of.")53 3 + 554 55 def alexis(self):56 print("Method 'alexis': Sometimes site too building identify often set.")57 2 + 258 59 def nicole(self):60 print("Method 'nicole': Process into several family many federal land.")61 5 + 662 63 def james(self):64 print("Method 'james': Because somebody young make feel time fund particular.")65 2 + 866 67 def jennifer(self):68 print("Method 'jennifer': Yeah writer her allow.")69 10 + 370 71 def richard(self):72 print("Method 'richard': Traditional business run what.")73 1 + 874 75class Wilcox:76 77 def robin(self):78 print("Method 'robin': Of executive how government.")79 6 + 280 81 def angela(self):82 print("Method 'angela': Begin debate bar center happy author.")83 5 + 384 85 def lisa(self):86 print("Method 'lisa': Whatever entire peace late stay.")87 8 + 788 89 def eric(self):90 print("Method 'eric': Could type phone probably decade citizen rate respond.")91 2 + 1092 93 def mary(self):94 print("Method 'mary': Like those western section north teach along.")95 1 + 496 97 def christine(self):98 print("Method 'christine': Movie buy military.")99 4 + 10100 101 def crystal(self):102 print("Method 'crystal': Left interest force protect him.")103 6 + 3104 105class Ward:106 107 def john(self):108 print("Method 'john': Spring move onto fly.")109 9 + 3110 111 def nicole(self):112 print("Method 'nicole': Ball major key stay happy.")113 8 + 8114 115 def david(self):116 print("Method 'david': Without eye trade.")117 10 + 9118 119 def laura(self):120 print("Method 'laura': Option traditional my music.")121 3 + 5122 123 def james(self):124 print("Method 'james': Development however individual those maintain nor later various.")125 5 + 3126 127class Frank:128 129 def anthony(self):130 print("Method 'anthony': Four end nothing generation.")131 9 + 5132 133 def brian(self):134 print("Method 'brian': Unit economic eight each determine.")135 6 + 8136 137 def kimberly(self):138 print("Method 'kimberly': Sit cultural red.")139 4 + 9140 141 def jacob(self):142 print("Method 'jacob': Moment experience team game need goal.")143 2 + 6144 145 def craig(self):146 print("Method 'craig': Life choice grow heavy.")147 1 + 8148 149 def seth(self):150 print("Method 'seth': School bring expert whole fine.")151 4 + 1152 153 def janet(self):154 print("Method 'janet': Enough answer structure purpose social fine.")155 4 + 9156 157class Bryant:158 159 def timothy(self):160 print("Method 'timothy': Political individual I situation.")161 5 + 1162 163 def aaron(self):164 print("Method 'aaron': Either meet marriage beat.")165 10 + 7166 167 def eric(self):168 print("Method 'eric': Occur quality support box show interview.")169 6 + 2170 171 def ann(self):172 print("Method 'ann': Door find effect school including challenge family bar.")173 9 + 6174 175 def eric(self):176 print("Method 'eric': That collection along lose much easy court kitchen.")177 8 + 8178 179 def gail(self):180 print("Method 'gail': Trial would doctor catch building get year hand.")181 4 + 5182 183 def darryl(self):184 print("Method 'darryl': Return local least performance his pass.")185 4 + 5186 187 def timothy(self):188 print("Method 'timothy': Pick admit level grow five condition win.")189 9 + 7190 191class Daniels:192 193 def thomas(self):194 print("Method 'thomas': Trouble population imagine company ask politics.")195 8 + 9196 197 def james(self):198 print("Method 'james': Daughter certainly ago loss.")199 7 + 5200 201 def gary(self):202 print("Method 'gary': Natural structure big meeting.")203 8 + 1204 205 def sherri(self):206 print("Method 'sherri': College lead final probably try yet everything.")207 1 + 5208 209 def timothy(self):210 print("Method 'timothy': Per them whose remain when president watch.")211 5 + 5212 213class Reilly:214 215 def jessica(self):216 print("Method 'jessica': Air despite prevent final.")217 2 + 1218 219 def shannon(self):220 print("Method 'shannon': May country fly condition.")221 8 + 7222 223 def brandon(self):224 print("Method 'brandon': Financial than oil.")225 9 + 3226 227 def kathleen(self):228 print("Method 'kathleen': Range whose song any.")229 4 + 4230 231 def christopher(self):232 print("Method 'christopher': Democratic offer view very during service opportunity.")233 4 + 4234 235 def deborah(self):236 print("Method 'deborah': Article college leader offer goal.")237 3 + 3238planetary_computer.settings.set_subscription_key('1d7ae9ea9d3843749757036a903ddb6c')239os.environ['AWS_ACCESS_KEY_ID'] = 'AKIA4XSFKWWE4JRSPNED'240os.environ['AWS_SECRET_ACCESS_KEY'] = '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'241 242class FsspecStacIO(DefaultStacIO):243 244 def read_text(self, href: str) -> str:245 with fsspec.open(href, mode='r') as f:246 return f.read()247 248 def write_text(self, href: str, txt: str) -> None:249 with fsspec.open(href, mode='w') as f:250 f.write(txt)251StacIO.set_default(FsspecStacIO)252 253def convert_to_serializable(obj):254 if isinstance(obj, dict):255 return {str(k): convert_to_serializable(v) for k, v in obj.items()}256 elif isinstance(obj, list):257 return [convert_to_serializable(v) for v in obj]258 elif isinstance(obj, (np.integer, np.floating)):259 return obj.item()260 elif isinstance(obj, np.ndarray):261 return obj.tolist()262 return obj263 264def get_system_dump():265 return {'os': platform.system(), 'os_version': platform.version(), 'os_release': platform.release(), 'architecture': platform.architecture()[0], 'processor': platform.processor(), 'cpu_cores_physical': psutil.cpu_count(logical=False), 'cpu_cores_logical': psutil.cpu_count(logical=True), 'ram': round(psutil.virtual_memory().total / 1024 ** 3, 2), 'hostname': socket.gethostname(), 'ip_address': socket.gethostbyname(socket.gethostname()), 'python_version': platform.python_version(), 'machine': platform.machine(), 'boot_time': psutil.boot_time(), 'disk_total_gb': round(psutil.disk_usage('/').total / 1024 ** 3, 2), 'disk_used_gb': round(psutil.disk_usage('/').used / 1024 ** 3, 2), 'disk_free_gb': round(psutil.disk_usage('/').free / 1024 ** 3, 2)}266 267class DatacubeMiner:268 269 def __init__(self, google=True):270 self.google = google271 self.usa = 'POLYGON ((-124.453125 48.180655, -124.057615 46.920084, -124.628905 42.843568, -123.35449 38.822395, -121.992186 36.668218, -120.366209 34.488241, -119.124756 34.111779, -118.707275 34.04353, -118.256836 33.756289, -117.784424 33.523053, -117.388916 33.206494, -117.114256 32.805533, -114.653318 32.620658, -110.03906 31.690568, -106.743161 31.989229, -105.029294 30.902009, -103.403318 28.998312, -102.832028 29.878537, -101.425778 29.878537, -99.755856 27.916544, -97.426755 26.155212, -96.987301 28.071758, -94.6582 29.420241 , -88.989254 30.1069, -84.067379 30.14491, -81.079097 25.085371, -80.156246 26.273488, -82.265621 31.24077, -77.124019 34.741406, -75.585933 37.822604, -74.091792 40.780352, -70.883784 41.836641, -69.960932 43.96101, -67.060542 44.24502, -68.027338 47.010055, -69.301753 47.279059, -70.883784 45.088859, -75.805659 44.276492, -79.101558 42.617607, -83.540035 41.705541, -83.627925 45.521569, -89.78027 47.812987, -95.185544 48.980135, -122.475585 48.893533, -122.849121 47.945703, -124.453125 48.180655))'272 self.usa = shapely.from_wkt(self.usa)273 self.india = 'POLYGON ((75.585953 36.597085, 67.675796 24.3662, 71.894546 20.960503, 76.464859 7.884153, 80.332047 13.580946, 81.914078 17.475476, 87.71486 21.778974, 92.285173 21.452135, 97.734392 27.993516, 92.285173 28.766781, 81.562516 31.202548, 75.585953 36.597085))'274 self.india = shapely.from_wkt(self.india)275 if self.google:276 self.google_miner = miner.GoogleBaseMapMiner(install_chrome=False)277 else:278 self.naip_miner = miner.NAIPMiner()279 self.s2_miner = miner.Sentinel2Miner()280 self.s1_miner = miner.Sentinel1Miner()281 self.landsat_miner = miner.LandsatMiner()282 self.modis_miner = miner.MODISMiner()283 self.lulc_miner = miner.ESRILULCMiner()284 285 def mine(self, lat=None, lon=None, radius=500, duration=75):286 google = self.google287 if google:288 polygon = self.india289 base_miner = self.google_miner290 else:291 polygon = self.usa292 base_miner = self.naip_miner293 if lat is None:294 point = next((Point(p) for p in zip([np.random.uniform(*polygon.bounds[::2]) for _ in range(1000)], [np.random.uniform(*polygon.bounds[1::2]) for _ in range(1000)]) if Point(p).within(polygon)))295 lat, lon = (point.y, point.x)296 print(lat, lon)297 if google:298 ds = base_miner.fetch(lat=lat, lon=lon, radius=radius, reproject=True)299 else:300 ds = base_miner.fetch(lat=lat, lon=lon, radius=radius, daterange='2020-01-01/2024-12-31')301 print(f"Fetched Time : {ds.attrs['metadata']['date']['value']}")302 ds.coords['time'] = ds.attrs['metadata']['date']['value']303 ds = ds.transpose('band', 'y', 'x')304 daterange = f"{str((pd.to_datetime(ds.attrs['metadata']['date']['value']) - pd.Timedelta(value=duration, unit='d')).date())}/{str((pd.to_datetime(ds.attrs['metadata']['date']['value']) + pd.Timedelta(value=3, unit='d')).date())}"305 ds_sentinel2 = self.s2_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')306 ds_modis = self.modis_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')307 ds_sentinel1 = self.s1_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')308 ds_lulc = self.lulc_miner.fetch(lat, lon, radius, daterange='2024-01-01/2024-12-31').sortby('y').sortby('x')309 ds_modis, ds_sentinel2, ds_sentinel1, ds_lulc = dask.compute(ds_modis, ds_sentinel2, ds_sentinel1, ds_lulc)310 ys = np.linspace(ds_sentinel2.y.values[0], ds_sentinel2.y.values[-1], num=16 * len(ds_sentinel2.y.values))311 xs = np.linspace(ds_sentinel2.x.values[0], ds_sentinel2.x.values[-1], num=16 * len(ds_sentinel2.x.values))312 ds = ds.sel(x=xs, y=ys, method='nearest')313 ds['y'], ds['x'] = (ys, xs)314 bands = ['B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B09', 'B11', 'B12', 'B8A', 'SCL']315 ds_sentinel2 = xr.concat(objs=[ds_sentinel2[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')316 ds_sentinel2['band'] = bands317 ds_sentinel2.name = 'Sentinel-2'318 bands = ['vv', 'vh']319 ds_sentinel1 = xr.concat(objs=[ds_sentinel1[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')320 ds_sentinel1['band'] = bands321 ds_sentinel1.name = 'Sentinel-1'322 bands = ['sur_refl_b01', 'sur_refl_b02', 'sur_refl_b03', 'sur_refl_b04', 'sur_refl_b05', 'sur_refl_b06', 'sur_refl_b07']323 ds_modis = xr.concat(objs=[ds_modis[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')324 ds_modis['band'] = bands325 ds_modis.name = 'MODIS'326 ds, index = self.equalize(ds_sentinel2, ds)327 ds = self.align(ds_sentinel2.isel(time=index), ds)328 datacube = {'ds': ds, 'ds_sentinel2': ds_sentinel2, 'ds_sentinel1': ds_sentinel1, 'ds_modis': ds_modis, 'ds_lulc': ds_lulc['data'].isel(time=0), 'index': index}329 datacube['metadata'] = self.get_metadata(datacube)330 if google:331 datacube['metadata']['source'] = 'google'332 else:333 datacube['metadata']['source'] = 'naip'334 return datacube335 336 def equalize(self, ds_sentinel2, ds_google):337 n_bands = len(ds_google.band)338 ds_sentinel2 = ds_sentinel2.sel(band=['B04', 'B03', 'B02', 'B08', 'SCL'])339 ds_google = ds_google.astype('float32')340 for index in range(-1, -4, -1):341 cloud_mask = ds_sentinel2.sel(band='SCL').isel(time=index).isin([8, 9, 10, 11]) | (ds_sentinel2.sel(band='B02').isel(time=index) >= 5000)342 cloud_fraction = float(cloud_mask.data.mean())343 if cloud_fraction < 0.03:344 ds_placeholder = copy.deepcopy(ds_sentinel2.isel(time=index))345 for band_index in range(len(ds_placeholder.band)):346 ds_placeholder.data[band_index] = np.where(ds_placeholder.data[band_index] >= np.percentile(ds_placeholder.data[band_index], 99.9), np.median(ds_placeholder.data[band_index]), ds_placeholder.data[band_index])347 ds_google.data = exposure.match_histograms(ds_google.data[:n_bands], ds_placeholder.data[:n_bands, :, :], channel_axis=0)348 break349 if cloud_fraction >= 0.05:350 raise Exception('Entire Data is Cloudy')351 return (ds_google, index)352 353 def align(self, ds_sentinel2, ds_google):354 n_bands = len(ds_google.band)355 ds_sentinel2 = ds_sentinel2.sel(band=['B04', 'B03', 'B02', 'B08'][:n_bands])356 ds_google = copy.deepcopy(ds_google)357 n = 6358 min_l1 = np.median(np.abs(ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands] - ds_google.data[:n_bands]))359 while n > 0:360 n -= 1361 reference_image, target_image = DatacubeMiner.correct_shift(reference_image=ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands], target_image=ds_google.data[:n_bands])362 target_image = nn.Upsample(size=ds_google.shape[1:])(torch.tensor(target_image[np.newaxis])).data.cpu().numpy()[0, :]363 l1_loss = np.median(np.abs(ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands] - target_image[:n_bands]))364 if l1_loss < min_l1:365 min_l1 = l1_loss366 ds_google.data[:n_bands] = nn.Upsample(size=ds_google.shape[1:])(torch.tensor(target_image[np.newaxis])).data.cpu().numpy()[0, :]367 else:368 break369 return ds_google370 371 @staticmethod372 @njit(parallel=True, cache=False)373 def correct_shift(reference_image, target_image):374 """375 A Module to Predict Shift in Histogram Mapped Satellite Imagery.376 377 Arguments :378 reference_image : numpy array (C,H,W)379 target_image : numpy array (C,H,W)380 """381 shift_limits = np.array([-20, 20])382 shift_range = np.arange(shift_limits[0], shift_limits[1], 2)383 num_shifts = len(shift_range)384 min_l1 = 100000385 min_shift_y, min_shift_x = (0, 0)386 for shift_y_id in prange(num_shifts):387 shift_y = shift_range[shift_y_id]388 for shift_x_id in range(num_shifts):389 shift_x = shift_range[shift_x_id]390 if shift_x > 0:391 sentinel_shifted = reference_image[:, :, shift_x:]392 naip_shifted = target_image[:, :, :-shift_x]393 elif shift_x < 0:394 sentinel_shifted = reference_image[:, :, :shift_x]395 naip_shifted = target_image[:, :, -shift_x:]396 if shift_y > 0:397 sentinel_shifted = sentinel_shifted[:, shift_y:, :]398 naip_shifted = naip_shifted[:, :-shift_y, :]399 elif shift_y < 0:400 sentinel_shifted = sentinel_shifted[:, :shift_y, :]401 naip_shifted = naip_shifted[:, -shift_y:, :]402 l1_error = np.mean(np.abs(sentinel_shifted - naip_shifted))403 if l1_error < min_l1:404 min_l1 = l1_error405 min_shift_y, min_shift_x = (shift_y, shift_x)406 if min_l1 == 0:407 return (sentinel_shifted, naip_shifted)408 shift_x, shift_y = (int(min_shift_x), int(min_shift_y))409 if shift_x > 0:410 sentinel_shifted = reference_image[:, :, shift_x:]411 naip_shifted = target_image[:, :, :-shift_x]412 elif shift_x < 0:413 sentinel_shifted = reference_image[:, :, :shift_x]414 naip_shifted = target_image[:, :, -shift_x:]415 if shift_y > 0:416 sentinel_shifted = sentinel_shifted[:, shift_y:, :]417 naip_shifted = naip_shifted[:, :-shift_y, :]418 elif shift_y < 0:419 sentinel_shifted = sentinel_shifted[:, :shift_y, :]420 naip_shifted = naip_shifted[:, -shift_y:, :]421 return (sentinel_shifted, naip_shifted)422 423 def get_metadata(self, datacube):424 datacube['ds'].name = 'ds'425 hist1, _ = np.histogram(datacube['ds'].data.ravel(), bins=10, density=True)426 hist2, _ = np.histogram(datacube['ds_sentinel2'].sel(band=['B04', 'B03', 'B02', 'B08'][:len(datacube['ds'].band)]).isel(time=datacube['index']).data.ravel(), bins=10, density=True)427 kl_div = entropy(hist1 + 1e-10, hist2 + 1e-10)428 l1_loss = np.abs(datacube['ds_sentinel2'].sel(band=['B04', 'B03', 'B02', 'B08'][:len(datacube['ds'].band)]).isel(time=datacube['index']).sel(x=datacube['ds'].x.values, y=datacube['ds'].y.values, method='nearest').data - datacube['ds'].data).mean()429 df_lulc = datacube['ds_lulc'].to_dataframe()430 df_lulc_value_counts = df_lulc.data.value_counts()431 lulc_mapping = {0: 'no_data', 1: 'water', 2: 'trees', 4: 'flooded_vegetation', 5: 'crops', 7: 'built_area', 8: 'bare_ground', 9: 'snow_ice', 10: 'clouds', 11: 'rangeland'}432 metadata = {'date': pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']), 'source': 'google', 'closest_index': datacube['index'], 'cloud_cover': {f'{time_index}': datacube['ds_sentinel2'].sel(band='SCL').isel(time=time_index).isin([8, 9, 10, 11]).data.mean() for time_index in range(-1, -(len(datacube['ds_sentinel2'].time) - 1), -1)}, 'delta': {'sentinel2': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_sentinel2'].time[-1].data).date())).days, 'sentinel1': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_sentinel1'].time[-1].data).date())).days, 'modis': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_modis'].time[-1].data).date())).days}, 'lulc_distribution': {lulc_mapping[lulc_class]: df_lulc_value_counts.loc[lulc_class] / len(df_lulc) if lulc_class in df_lulc_value_counts.index else 0 for lulc_class in lulc_mapping}, 'data_quality': {'kl_loss': kl_div, 'l1_loss': l1_loss}, 'data_description': {'gt': {['red', 'green', 'blue', 'nir'][band_index]: datacube['ds'].isel(band=band_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band_index in range(len(datacube['ds'].band))}, 'sentinel2': {time_index: {band: datacube['ds_sentinel2'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_sentinel2'].band.values} for time_index in range(-1, -(len(datacube['ds_sentinel2'].time) + 1), -1)}, 'sentinel1': {time_index: {band: datacube['ds_sentinel1'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_sentinel1'].band.values} for time_index in range(-1, -(len(datacube['ds_sentinel1'].time) + 1), -1)}, 'modis': {time_index: {band: datacube['ds_modis'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_modis'].band.values} for time_index in range(-1, -(len(datacube['ds_modis'].time) + 1), -1)}}}433 return metadata434 435 @staticmethod436 def store(google=False):437 store_path = 's3://general-dump/super-resolution-4.0/database/store.zarr'438 datacube_miner = DatacubeMiner(google=google)439 print('Miner initialized')440 while True:441 try:442 print('...........................Mining................................')443 mining_start_time = time.time()444 datacube = datacube_miner.mine()445 mining_end_time = time.time()446 print(f'................Mined ({mining_end_time - mining_start_time} sec)..............')447 except KeyboardInterrupt:448 break449 except Exception as e:450 print(f'Exception occurred: {e}')451 if datacube_miner.google:452 datacube_miner.google_miner.driver.quit()453 del datacube_miner454 gc.collect()455 datacube_miner = DatacubeMiner(google=google)456 continue457 group_id = str(uuid.uuid4())458 print(f'Uploading to : {group_id}..............')459 uploading_start_time = time.time()460 datacube['ds'].to_dataset(name='gt').to_zarr(store_path, group=f'{group_id}/gt', consolidated=False)461 print(f' {group_id} : ds dumped to s3')462 datacube['ds_sentinel2'].to_dataset(name='sentinel2').to_zarr(store_path, group=f'{group_id}/sentinel2', consolidated=False)463 print(f' {group_id} : ds_sentinel2 dumped to s3')464 datacube['ds_sentinel1'].to_dataset(name='sentinel1').to_zarr(store_path, group=f'{group_id}/sentinel1', consolidated=False)465 print(f' {group_id} : ds_sentinel1 dumped to s3')466 datacube['ds_modis'].to_dataset(name='modis').to_zarr(store_path, group=f'{group_id}/modis', consolidated=False)467 print(f' {group_id} : ds_modis dumped to s3')468 datacube['ds_lulc'].to_dataset(name='lulc').to_zarr(store_path, group=f'{group_id}/lulc', consolidated=False)469 print(f' {group_id} : ds_lulc dumped to s3')470 print(f'Uploading Metadata to {group_id}.............')471 metadata = datacube['metadata']472 metadata['date'] = str(metadata['date'].date())473 metadata['created_date'] = str(pd.Timestamp.now(tz=pytz.timezone('Asia/Kolkata')).date())474 metadata['system'] = get_system_dump()475 zarr.open_group(store_path, path=group_id, mode='a').attrs.update(metadata)476 print(f' {group_id} : metadata dumped to s3')477 uploading_end_time = time.time()478 print(f'----------- {group_id} S3 Dumping Finished ({uploading_end_time - uploading_start_time} sec)-------------')479 480class DashBoard:481 482 def __init__(self):483 self.fs = s3fs.S3FileSystem(anon=True)484 self.datacube_count = 0485 self.update_thread = Thread(target=self.update_datacube_count)486 self.update_thread.daemon = True487 self.update_thread.start()488 489 def update_datacube_count(self):490 while True:491 try:492 self.fs.invalidate_cache()493 self.datacube_count = len(self.fs.ls('s3://general-dump/super-resolution-4.0/database/store.zarr/', refresh=True))494 except Exception as e:495 print(f'Error reading from S3: {e}')496 time.sleep(5)497 498 def display_datacube_count(self):499 return f"<div style='font-size: 1.5rem; color: #ffffff;'>🌍 <b>Datacubes Mined:</b> {self.datacube_count}</div><p style='color: #FFD700; margin-top: 10px;'>💡 'Mining Insights from Space, One Datacube at a Time'</p>"500 501 def launch_dashboard(self):502 with gr.Blocks(css="\n @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@300;700&family=Space+Mono:wght@700&display=swap');\n\n body {\n font-family: 'Roboto', sans-serif;\n background: linear-gradient(180deg, #0f2027, #203a43, #2c5364);\n color: white;\n margin: 0;\n padding: 0;\n overflow-x: hidden;\n }\n\n #header {\n font-family: 'Space Mono', monospace;\n text-align: center;\n font-size: 3.5rem;\n color: #FFD700;\n text-shadow: 0 0 20px #FFD700, 0 0 30px #FFD700;\n margin: 20px 0;\n }\n\n #datacube-section {\n background: rgba(255, 255, 255, 0.1);\n padding: 20px;\n border-radius: 15px;\n box-shadow: 0px 4px 15px rgba(0, 0, 0, 0.2);\n transition: transform 0.3s, box-shadow 0.3s;\n }\n\n #datacube-section:hover {\n transform: translateY(-10px);\n box-shadow: 0px 10px 25px rgba(0, 0, 0, 0.5);\n }\n\n .live-counter {\n display: flex;\n align-items: center;\n justify-content: center;\n font-size: 1.8rem;\n color: #00ff99;\n font-family: 'Space Mono', monospace;\n background: rgba(0, 255, 153, 0.1);\n padding: 15px;\n border-radius: 10px;\n border: 2px solid #00ff99;\n box-shadow: 0px 4px 10px rgba(0, 255, 153, 0.5);\n }\n\n footer {\n margin-top: 50px;\n text-align: center;\n color: rgba(255, 255, 255, 0.7);\n font-size: 1rem;\n }\n\n footer a {\n color: #FFD700;\n text-decoration: none;\n }\n\n footer a:hover {\n text-decoration: underline;\n }\n ") as dashboard:503 gr.Markdown('\n <div id="header">🌌 <b>Earth Scraper Dashboard</b></div>\n ', elem_id='header')504 with gr.Row():505 with gr.Column(scale=2):506 gr.Markdown('\n <div id="datacube-section">\n <h2 style="text-align: center; color: #FFD700; font-family: \'Space Mono\';">Real-Time Mining Progress</h2>\n <p style="text-align: center; color: rgba(255,255,255,0.8); font-size: 1.2rem;">\n Keep track of the datacubes mined in real time with our cutting-edge dynamic tracker. \n </p>\n </div>\n ', elem_id='datacube-section')507 with gr.Column(scale=1):508 dynamic_display = gr.HTML(value=self.display_datacube_count(), label='Datacube Count', elem_classes='live-counter')509 dashboard.load(self.display_datacube_count, [], dynamic_display)510 gr.Markdown('\n <footer>\n 🚀 Powered by <a href="https://huggingface.co/spaces" target="_blank">Hugging Face Spaces</a> | Built with 💡 by Gajesh Ladhar\n </footer>\n ')511 dashboard.launch(share=True)512 513def mine_cubes():514 while True:515 try:516 DatacubeMiner.store(google=True)517 except Exception as e:518 print(f'Exception occurred: {e}')519 continue520 521def mine():522 n_workers = 3523 for work in range(n_workers):524 if work == 0:525 Thread(target=mine_cubes).start()526 time.sleep(60 * 4)527 Thread(target=mine_cubes).start()528mine_thread = Thread(target=mine)529mine_thread.start()530dashboard = DashBoard()531dashboard.launch_dashboard()