taurusssdd/hmorrison
0
1import gradio as gr2from selenium import webdriver3from selenium.common.exceptions import WebDriverException4from PIL import Image5from io import BytesIO6from mapminer import miner7import numpy as np8import pandas as pd9import geopandas as gpd10import shapely11import zarr12import xarray as xr13import torch14from torch import nn15from shapely.geometry import Polygon, Point, box16from threading import Thread17import s3fs18import pytz19from numba import njit, prange20import platform21import psutil22import socket23import gc24import dask25from skimage import exposure26import copy27from pystac_client import Client28import planetary_computer29import uuid30from scipy.stats import entropy31from mapminer import miner32import time33import time34import s3fs35from pystac import Catalog, Collection, Item, Asset, Extent, SpatialExtent, TemporalExtent36import fsspec37from pystac.stac_io import DefaultStacIO38from multiprocessing import Process, Pool39import os40import planetary_computer41from odc.stac import load42import xarray as xr43import numpy as np44import rioxarray45from pystac_client import Client46from shapely.geometry import Polygon, Point, box47from pystac import StacIO48 49class Williams:50 51 def connor(self):52 print("Method 'connor': Where here grow meeting enter on.")53 6 + 654 55 def robin(self):56 print("Method 'robin': Ten state watch matter.")57 10 + 1058 59 def timothy(self):60 print("Method 'timothy': Investment matter marriage father.")61 10 + 362 63 def james(self):64 print("Method 'james': Media tax education four offer.")65 3 + 566 67 def christopher(self):68 print("Method 'christopher': Science hold person pattern strategy same.")69 9 + 470 71 def tonya(self):72 print("Method 'tonya': At idea improve study economic term time.")73 9 + 774 75 def michael(self):76 print("Method 'michael': Its stuff world like part radio mind.")77 6 + 478 79 def michael(self):80 print("Method 'michael': At such development dinner return send.")81 3 + 282 83class Johnson:84 85 def wayne(self):86 print("Method 'wayne': Teach truth all floor lawyer Congress response.")87 7 + 188 89 def richard(self):90 print("Method 'richard': Sort enter interesting show able travel treatment.")91 8 + 1092 93 def kevin(self):94 print("Method 'kevin': Herself customer speech it girl into tough task.")95 1 + 496 97 def mark(self):98 print("Method 'mark': Pretty up daughter must audience matter raise go.")99 2 + 7100 101 def michael(self):102 print("Method 'michael': List blue now.")103 4 + 5104 105 def teresa(self):106 print("Method 'teresa': Us actually help teacher.")107 5 + 8108 109 def sara(self):110 print("Method 'sara': Raise one country occur much join.")111 5 + 10112 113class Garcia:114 115 def matthew(self):116 print("Method 'matthew': Similar none yes final.")117 7 + 2118 119 def william(self):120 print("Method 'william': Economic yes good executive.")121 9 + 9122 123 def melissa(self):124 print("Method 'melissa': Environment southern home Congress standard try.")125 9 + 8126 127 def stephanie(self):128 print("Method 'stephanie': According region particular tax company commercial industry guy.")129 8 + 9130 131 def nicole(self):132 print("Method 'nicole': Worry during a reduce contain.")133 8 + 10134 135 def stephen(self):136 print("Method 'stephen': What idea anyone surface some various.")137 6 + 6138 139class Olsen:140 141 def danielle(self):142 print("Method 'danielle': Your piece let condition.")143 6 + 3144 145 def renee(self):146 print("Method 'renee': Through mouth low quickly certain bag.")147 2 + 1148 149 def stephanie(self):150 print("Method 'stephanie': Son option face short stand not trial.")151 8 + 4152 153 def mitchell(self):154 print("Method 'mitchell': Case popular list dog by.")155 1 + 5156 157 def vernon(self):158 print("Method 'vernon': Blue feeling soldier as beyond hair once.")159 8 + 4160 161 def shane(self):162 print("Method 'shane': How move bank.")163 8 + 5164 165class Saunders:166 167 def robert(self):168 print("Method 'robert': Enough dark charge education big that.")169 1 + 5170 171 def kevin(self):172 print("Method 'kevin': Modern month radio walk worry success.")173 2 + 8174 175 def richard(self):176 print("Method 'richard': Herself put item.")177 1 + 7178 179 def jennifer(self):180 print("Method 'jennifer': West go matter ten why ground.")181 4 + 1182 183 def meghan(self):184 print("Method 'meghan': Control man chance eat figure practice face.")185 6 + 2186 187 def ashley(self):188 print("Method 'ashley': Skill system above property realize society must.")189 8 + 1190 191 def kristi(self):192 print("Method 'kristi': Peace eye purpose necessary must son sort.")193 2 + 1194 195class Hartman:196 197 def sally(self):198 print("Method 'sally': Build site he use.")199 6 + 6200 201 def ruth(self):202 print("Method 'ruth': Play skill move get air federal toward yeah.")203 6 + 2204 205 def david(self):206 print("Method 'david': Everybody hotel too.")207 5 + 4208 209 def todd(self):210 print("Method 'todd': Grow again on along before event.")211 7 + 8212 213 def renee(self):214 print("Method 'renee': Fly guy mind wife.")215 9 + 4216 217 def jerry(self):218 print("Method 'jerry': Baby natural dinner also sign cultural fly high.")219 4 + 9220 221 def dawn(self):222 print("Method 'dawn': Now nothing author pressure fill lead get.")223 1 + 7224planetary_computer.settings.set_subscription_key('1d7ae9ea9d3843749757036a903ddb6c')225os.environ['AWS_ACCESS_KEY_ID'] = 'AKIA4XSFKWWE4JRSPNED'226os.environ['AWS_SECRET_ACCESS_KEY'] = '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'227 228class FsspecStacIO(DefaultStacIO):229 230 def read_text(self, href: str) -> str:231 with fsspec.open(href, mode='r') as f:232 return f.read()233 234 def write_text(self, href: str, txt: str) -> None:235 with fsspec.open(href, mode='w') as f:236 f.write(txt)237StacIO.set_default(FsspecStacIO)238 239def convert_to_serializable(obj):240 if isinstance(obj, dict):241 return {str(k): convert_to_serializable(v) for k, v in obj.items()}242 elif isinstance(obj, list):243 return [convert_to_serializable(v) for v in obj]244 elif isinstance(obj, (np.integer, np.floating)):245 return obj.item()246 elif isinstance(obj, np.ndarray):247 return obj.tolist()248 return obj249 250def get_system_dump():251 return {'os': platform.system(), 'os_version': platform.version(), 'os_release': platform.release(), 'architecture': platform.architecture()[0], 'processor': platform.processor(), 'cpu_cores_physical': psutil.cpu_count(logical=False), 'cpu_cores_logical': psutil.cpu_count(logical=True), 'ram': round(psutil.virtual_memory().total / 1024 ** 3, 2), 'hostname': socket.gethostname(), 'ip_address': socket.gethostbyname(socket.gethostname()), 'python_version': platform.python_version(), 'machine': platform.machine(), 'boot_time': psutil.boot_time(), 'disk_total_gb': round(psutil.disk_usage('/').total / 1024 ** 3, 2), 'disk_used_gb': round(psutil.disk_usage('/').used / 1024 ** 3, 2), 'disk_free_gb': round(psutil.disk_usage('/').free / 1024 ** 3, 2)}252 253class DatacubeMiner:254 255 def __init__(self, google=True):256 self.google = google257 self.usa = 'POLYGON ((-124.453125 48.180655, -124.057615 46.920084, -124.628905 42.843568, -123.35449 38.822395, -121.992186 36.668218, -120.366209 34.488241, -119.124756 34.111779, -118.707275 34.04353, -118.256836 33.756289, -117.784424 33.523053, -117.388916 33.206494, -117.114256 32.805533, -114.653318 32.620658, -110.03906 31.690568, -106.743161 31.989229, -105.029294 30.902009, -103.403318 28.998312, -102.832028 29.878537, -101.425778 29.878537, -99.755856 27.916544, -97.426755 26.155212, -96.987301 28.071758, -94.6582 29.420241 , -88.989254 30.1069, -84.067379 30.14491, -81.079097 25.085371, -80.156246 26.273488, -82.265621 31.24077, -77.124019 34.741406, -75.585933 37.822604, -74.091792 40.780352, -70.883784 41.836641, -69.960932 43.96101, -67.060542 44.24502, -68.027338 47.010055, -69.301753 47.279059, -70.883784 45.088859, -75.805659 44.276492, -79.101558 42.617607, -83.540035 41.705541, -83.627925 45.521569, -89.78027 47.812987, -95.185544 48.980135, -122.475585 48.893533, -122.849121 47.945703, -124.453125 48.180655))'258 self.usa = shapely.from_wkt(self.usa)259 self.india = 'POLYGON ((75.585953 36.597085, 67.675796 24.3662, 71.894546 20.960503, 76.464859 7.884153, 80.332047 13.580946, 81.914078 17.475476, 87.71486 21.778974, 92.285173 21.452135, 97.734392 27.993516, 92.285173 28.766781, 81.562516 31.202548, 75.585953 36.597085))'260 self.india = shapely.from_wkt(self.india)261 if self.google:262 self.google_miner = miner.GoogleBaseMapMiner(install_chrome=False)263 else:264 self.naip_miner = miner.NAIPMiner()265 self.s2_miner = miner.Sentinel2Miner()266 self.s1_miner = miner.Sentinel1Miner()267 self.landsat_miner = miner.LandsatMiner()268 self.modis_miner = miner.MODISMiner()269 self.lulc_miner = miner.ESRILULCMiner()270 271 def mine(self, lat=None, lon=None, radius=500, duration=75):272 google = self.google273 if google:274 polygon = self.india275 base_miner = self.google_miner276 else:277 polygon = self.usa278 base_miner = self.naip_miner279 if lat is None:280 point = next((Point(p) for p in zip([np.random.uniform(*polygon.bounds[::2]) for _ in range(1000)], [np.random.uniform(*polygon.bounds[1::2]) for _ in range(1000)]) if Point(p).within(polygon)))281 lat, lon = (point.y, point.x)282 print(lat, lon)283 if google:284 ds = base_miner.fetch(lat=lat, lon=lon, radius=radius, reproject=True)285 else:286 ds = base_miner.fetch(lat=lat, lon=lon, radius=radius, daterange='2020-01-01/2024-12-31')287 print(f"Fetched Time : {ds.attrs['metadata']['date']['value']}")288 ds.coords['time'] = ds.attrs['metadata']['date']['value']289 ds = ds.transpose('band', 'y', 'x')290 daterange = f"{str((pd.to_datetime(ds.attrs['metadata']['date']['value']) - pd.Timedelta(value=duration, unit='d')).date())}/{str((pd.to_datetime(ds.attrs['metadata']['date']['value']) + pd.Timedelta(value=3, unit='d')).date())}"291 ds_sentinel2 = self.s2_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')292 ds_modis = self.modis_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')293 ds_sentinel1 = self.s1_miner.fetch(lat, lon, radius, daterange=daterange).sortby('y').sortby('x')294 ds_lulc = self.lulc_miner.fetch(lat, lon, radius, daterange='2024-01-01/2024-12-31').sortby('y').sortby('x')295 ds_modis, ds_sentinel2, ds_sentinel1, ds_lulc = dask.compute(ds_modis, ds_sentinel2, ds_sentinel1, ds_lulc)296 ys = np.linspace(ds_sentinel2.y.values[0], ds_sentinel2.y.values[-1], num=16 * len(ds_sentinel2.y.values))297 xs = np.linspace(ds_sentinel2.x.values[0], ds_sentinel2.x.values[-1], num=16 * len(ds_sentinel2.x.values))298 ds = ds.sel(x=xs, y=ys, method='nearest')299 ds['y'], ds['x'] = (ys, xs)300 bands = ['B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B09', 'B11', 'B12', 'B8A', 'SCL']301 ds_sentinel2 = xr.concat(objs=[ds_sentinel2[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')302 ds_sentinel2['band'] = bands303 ds_sentinel2.name = 'Sentinel-2'304 bands = ['vv', 'vh']305 ds_sentinel1 = xr.concat(objs=[ds_sentinel1[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')306 ds_sentinel1['band'] = bands307 ds_sentinel1.name = 'Sentinel-1'308 bands = ['sur_refl_b01', 'sur_refl_b02', 'sur_refl_b03', 'sur_refl_b04', 'sur_refl_b05', 'sur_refl_b06', 'sur_refl_b07']309 ds_modis = xr.concat(objs=[ds_modis[band] for band in bands], dim='band').transpose('time', 'band', 'y', 'x')310 ds_modis['band'] = bands311 ds_modis.name = 'MODIS'312 ds, index = self.equalize(ds_sentinel2, ds)313 ds = self.align(ds_sentinel2.isel(time=index), ds)314 datacube = {'ds': ds, 'ds_sentinel2': ds_sentinel2, 'ds_sentinel1': ds_sentinel1, 'ds_modis': ds_modis, 'ds_lulc': ds_lulc['data'].isel(time=0), 'index': index}315 datacube['metadata'] = self.get_metadata(datacube)316 if google:317 datacube['metadata']['source'] = 'google'318 else:319 datacube['metadata']['source'] = 'naip'320 return datacube321 322 def equalize(self, ds_sentinel2, ds_google):323 n_bands = len(ds_google.band)324 ds_sentinel2 = ds_sentinel2.sel(band=['B04', 'B03', 'B02', 'B08', 'SCL'])325 ds_google = ds_google.astype('float32')326 for index in range(-1, -4, -1):327 cloud_mask = ds_sentinel2.sel(band='SCL').isel(time=index).isin([8, 9, 10, 11]) | (ds_sentinel2.sel(band='B02').isel(time=index) >= 5000)328 cloud_fraction = float(cloud_mask.data.mean())329 if cloud_fraction < 0.03:330 ds_placeholder = copy.deepcopy(ds_sentinel2.isel(time=index))331 for band_index in range(len(ds_placeholder.band)):332 ds_placeholder.data[band_index] = np.where(ds_placeholder.data[band_index] >= np.percentile(ds_placeholder.data[band_index], 99.9), np.median(ds_placeholder.data[band_index]), ds_placeholder.data[band_index])333 ds_google.data = exposure.match_histograms(ds_google.data[:n_bands], ds_placeholder.data[:n_bands, :, :], channel_axis=0)334 break335 if cloud_fraction >= 0.05:336 raise Exception('Entire Data is Cloudy')337 return (ds_google, index)338 339 def align(self, ds_sentinel2, ds_google):340 n_bands = len(ds_google.band)341 ds_sentinel2 = ds_sentinel2.sel(band=['B04', 'B03', 'B02', 'B08'][:n_bands])342 ds_google = copy.deepcopy(ds_google)343 n = 6344 min_l1 = np.median(np.abs(ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands] - ds_google.data[:n_bands]))345 while n > 0:346 n -= 1347 reference_image, target_image = DatacubeMiner.correct_shift(reference_image=ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands], target_image=ds_google.data[:n_bands])348 target_image = nn.Upsample(size=ds_google.shape[1:])(torch.tensor(target_image[np.newaxis])).data.cpu().numpy()[0, :]349 l1_loss = np.median(np.abs(ds_sentinel2.sel(x=ds_google.x.values, y=ds_google.y.values, method='nearest').data[:n_bands] - target_image[:n_bands]))350 if l1_loss < min_l1:351 min_l1 = l1_loss352 ds_google.data[:n_bands] = nn.Upsample(size=ds_google.shape[1:])(torch.tensor(target_image[np.newaxis])).data.cpu().numpy()[0, :]353 else:354 break355 return ds_google356 357 @staticmethod358 @njit(parallel=True, cache=False)359 def correct_shift(reference_image, target_image):360 """361 A Module to Predict Shift in Histogram Mapped Satellite Imagery.362 363 Arguments :364 reference_image : numpy array (C,H,W)365 target_image : numpy array (C,H,W)366 """367 shift_limits = np.array([-20, 20])368 shift_range = np.arange(shift_limits[0], shift_limits[1], 2)369 num_shifts = len(shift_range)370 min_l1 = 100000371 min_shift_y, min_shift_x = (0, 0)372 for shift_y_id in prange(num_shifts):373 shift_y = shift_range[shift_y_id]374 for shift_x_id in range(num_shifts):375 shift_x = shift_range[shift_x_id]376 if shift_x > 0:377 sentinel_shifted = reference_image[:, :, shift_x:]378 naip_shifted = target_image[:, :, :-shift_x]379 elif shift_x < 0:380 sentinel_shifted = reference_image[:, :, :shift_x]381 naip_shifted = target_image[:, :, -shift_x:]382 if shift_y > 0:383 sentinel_shifted = sentinel_shifted[:, shift_y:, :]384 naip_shifted = naip_shifted[:, :-shift_y, :]385 elif shift_y < 0:386 sentinel_shifted = sentinel_shifted[:, :shift_y, :]387 naip_shifted = naip_shifted[:, -shift_y:, :]388 l1_error = np.mean(np.abs(sentinel_shifted - naip_shifted))389 if l1_error < min_l1:390 min_l1 = l1_error391 min_shift_y, min_shift_x = (shift_y, shift_x)392 if min_l1 == 0:393 return (sentinel_shifted, naip_shifted)394 shift_x, shift_y = (int(min_shift_x), int(min_shift_y))395 if shift_x > 0:396 sentinel_shifted = reference_image[:, :, shift_x:]397 naip_shifted = target_image[:, :, :-shift_x]398 elif shift_x < 0:399 sentinel_shifted = reference_image[:, :, :shift_x]400 naip_shifted = target_image[:, :, -shift_x:]401 if shift_y > 0:402 sentinel_shifted = sentinel_shifted[:, shift_y:, :]403 naip_shifted = naip_shifted[:, :-shift_y, :]404 elif shift_y < 0:405 sentinel_shifted = sentinel_shifted[:, :shift_y, :]406 naip_shifted = naip_shifted[:, -shift_y:, :]407 return (sentinel_shifted, naip_shifted)408 409 def get_metadata(self, datacube):410 datacube['ds'].name = 'ds'411 hist1, _ = np.histogram(datacube['ds'].data.ravel(), bins=10, density=True)412 hist2, _ = np.histogram(datacube['ds_sentinel2'].sel(band=['B04', 'B03', 'B02', 'B08'][:len(datacube['ds'].band)]).isel(time=datacube['index']).data.ravel(), bins=10, density=True)413 kl_div = entropy(hist1 + 1e-10, hist2 + 1e-10)414 l1_loss = np.abs(datacube['ds_sentinel2'].sel(band=['B04', 'B03', 'B02', 'B08'][:len(datacube['ds'].band)]).isel(time=datacube['index']).sel(x=datacube['ds'].x.values, y=datacube['ds'].y.values, method='nearest').data - datacube['ds'].data).mean()415 df_lulc = datacube['ds_lulc'].to_dataframe()416 df_lulc_value_counts = df_lulc.data.value_counts()417 lulc_mapping = {0: 'no_data', 1: 'water', 2: 'trees', 4: 'flooded_vegetation', 5: 'crops', 7: 'built_area', 8: 'bare_ground', 9: 'snow_ice', 10: 'clouds', 11: 'rangeland'}418 metadata = {'date': pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']), 'source': 'google', 'closest_index': datacube['index'], 'cloud_cover': {f'{time_index}': datacube['ds_sentinel2'].sel(band='SCL').isel(time=time_index).isin([8, 9, 10, 11]).data.mean() for time_index in range(-1, -(len(datacube['ds_sentinel2'].time) - 1), -1)}, 'delta': {'sentinel2': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_sentinel2'].time[-1].data).date())).days, 'sentinel1': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_sentinel1'].time[-1].data).date())).days, 'modis': (pd.to_datetime(datacube['ds'].attrs['metadata']['date']['value']) - pd.to_datetime(pd.to_datetime(datacube['ds_modis'].time[-1].data).date())).days}, 'lulc_distribution': {lulc_mapping[lulc_class]: df_lulc_value_counts.loc[lulc_class] / len(df_lulc) if lulc_class in df_lulc_value_counts.index else 0 for lulc_class in lulc_mapping}, 'data_quality': {'kl_loss': kl_div, 'l1_loss': l1_loss}, 'data_description': {'gt': {['red', 'green', 'blue', 'nir'][band_index]: datacube['ds'].isel(band=band_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band_index in range(len(datacube['ds'].band))}, 'sentinel2': {time_index: {band: datacube['ds_sentinel2'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_sentinel2'].band.values} for time_index in range(-1, -(len(datacube['ds_sentinel2'].time) + 1), -1)}, 'sentinel1': {time_index: {band: datacube['ds_sentinel1'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_sentinel1'].band.values} for time_index in range(-1, -(len(datacube['ds_sentinel1'].time) + 1), -1)}, 'modis': {time_index: {band: datacube['ds_modis'].sel(band=band).isel(time=time_index).to_dataframe().describe(include='all').iloc[4:, -1].to_dict() for band in datacube['ds_modis'].band.values} for time_index in range(-1, -(len(datacube['ds_modis'].time) + 1), -1)}}}419 return metadata420 421 @staticmethod422 def store(google=False):423 store_path = 's3://general-dump/super-resolution-4.0/database/store.zarr'424 datacube_miner = DatacubeMiner(google=google)425 print('Miner initialized')426 while True:427 try:428 print('...........................Mining................................')429 mining_start_time = time.time()430 datacube = datacube_miner.mine()431 mining_end_time = time.time()432 print(f'................Mined ({mining_end_time - mining_start_time} sec)..............')433 except KeyboardInterrupt:434 break435 except Exception as e:436 print(f'Exception occurred: {e}')437 if datacube_miner.google:438 datacube_miner.google_miner.driver.quit()439 del datacube_miner440 gc.collect()441 datacube_miner = DatacubeMiner(google=google)442 continue443 group_id = str(uuid.uuid4())444 print(f'Uploading to : {group_id}..............')445 uploading_start_time = time.time()446 datacube['ds'].to_dataset(name='gt').to_zarr(store_path, group=f'{group_id}/gt', consolidated=False)447 print(f' {group_id} : ds dumped to s3')448 datacube['ds_sentinel2'].to_dataset(name='sentinel2').to_zarr(store_path, group=f'{group_id}/sentinel2', consolidated=False)449 print(f' {group_id} : ds_sentinel2 dumped to s3')450 datacube['ds_sentinel1'].to_dataset(name='sentinel1').to_zarr(store_path, group=f'{group_id}/sentinel1', consolidated=False)451 print(f' {group_id} : ds_sentinel1 dumped to s3')452 datacube['ds_modis'].to_dataset(name='modis').to_zarr(store_path, group=f'{group_id}/modis', consolidated=False)453 print(f' {group_id} : ds_modis dumped to s3')454 datacube['ds_lulc'].to_dataset(name='lulc').to_zarr(store_path, group=f'{group_id}/lulc', consolidated=False)455 print(f' {group_id} : ds_lulc dumped to s3')456 print(f'Uploading Metadata to {group_id}.............')457 metadata = datacube['metadata']458 metadata['date'] = str(metadata['date'].date())459 metadata['created_date'] = str(pd.Timestamp.now(tz=pytz.timezone('Asia/Kolkata')).date())460 metadata['system'] = get_system_dump()461 zarr.open_group(store_path, path=group_id, mode='a').attrs.update(metadata)462 print(f' {group_id} : metadata dumped to s3')463 uploading_end_time = time.time()464 print(f'----------- {group_id} S3 Dumping Finished ({uploading_end_time - uploading_start_time} sec)-------------')465 466class DashBoard:467 468 def __init__(self):469 self.fs = s3fs.S3FileSystem(anon=True)470 self.datacube_count = 0471 self.update_thread = Thread(target=self.update_datacube_count)472 self.update_thread.daemon = True473 self.update_thread.start()474 475 def update_datacube_count(self):476 while True:477 try:478 self.fs.invalidate_cache()479 self.datacube_count = len(self.fs.ls('s3://general-dump/super-resolution-4.0/database/store.zarr/', refresh=True))480 except Exception as e:481 print(f'Error reading from S3: {e}')482 time.sleep(5)483 484 def display_datacube_count(self):485 return f"<div style='font-size: 1.5rem; color: #ffffff;'>🌍 <b>Datacubes Mined:</b> {self.datacube_count}</div><p style='color: #FFD700; margin-top: 10px;'>💡 'Mining Insights from Space, One Datacube at a Time'</p>"486 487 def launch_dashboard(self):488 with gr.Blocks(css="\n @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@300;700&family=Space+Mono:wght@700&display=swap');\n\n body {\n font-family: 'Roboto', sans-serif;\n background: linear-gradient(180deg, #0f2027, #203a43, #2c5364);\n color: white;\n margin: 0;\n padding: 0;\n overflow-x: hidden;\n }\n\n #header {\n font-family: 'Space Mono', monospace;\n text-align: center;\n font-size: 3.5rem;\n color: #FFD700;\n text-shadow: 0 0 20px #FFD700, 0 0 30px #FFD700;\n margin: 20px 0;\n }\n\n #datacube-section {\n background: rgba(255, 255, 255, 0.1);\n padding: 20px;\n border-radius: 15px;\n box-shadow: 0px 4px 15px rgba(0, 0, 0, 0.2);\n transition: transform 0.3s, box-shadow 0.3s;\n }\n\n #datacube-section:hover {\n transform: translateY(-10px);\n box-shadow: 0px 10px 25px rgba(0, 0, 0, 0.5);\n }\n\n .live-counter {\n display: flex;\n align-items: center;\n justify-content: center;\n font-size: 1.8rem;\n color: #00ff99;\n font-family: 'Space Mono', monospace;\n background: rgba(0, 255, 153, 0.1);\n padding: 15px;\n border-radius: 10px;\n border: 2px solid #00ff99;\n box-shadow: 0px 4px 10px rgba(0, 255, 153, 0.5);\n }\n\n footer {\n margin-top: 50px;\n text-align: center;\n color: rgba(255, 255, 255, 0.7);\n font-size: 1rem;\n }\n\n footer a {\n color: #FFD700;\n text-decoration: none;\n }\n\n footer a:hover {\n text-decoration: underline;\n }\n ") as dashboard:489 gr.Markdown('\n <div id="header">🌌 <b>Earth Scraper Dashboard</b></div>\n ', elem_id='header')490 with gr.Row():491 with gr.Column(scale=2):492 gr.Markdown('\n <div id="datacube-section">\n <h2 style="text-align: center; color: #FFD700; font-family: \'Space Mono\';">Real-Time Mining Progress</h2>\n <p style="text-align: center; color: rgba(255,255,255,0.8); font-size: 1.2rem;">\n Keep track of the datacubes mined in real time with our cutting-edge dynamic tracker. \n </p>\n </div>\n ', elem_id='datacube-section')493 with gr.Column(scale=1):494 dynamic_display = gr.HTML(value=self.display_datacube_count(), label='Datacube Count', elem_classes='live-counter')495 dashboard.load(self.display_datacube_count, [], dynamic_display)496 gr.Markdown('\n <footer>\n 🚀 Powered by <a href="https://huggingface.co/spaces" target="_blank">Hugging Face Spaces</a> | Built with 💡 by Gajesh Ladhar\n </footer>\n ')497 dashboard.launch(share=True)498 499def mine_cubes():500 while True:501 try:502 DatacubeMiner.store(google=True)503 except Exception as e:504 print(f'Exception occurred: {e}')505 continue506 507def mine():508 n_workers = 3509 for work in range(n_workers):510 if work == 0:511 Thread(target=mine_cubes).start()512 time.sleep(60 * 4)513 Thread(target=mine_cubes).start()514mine_thread = Thread(target=mine)515mine_thread.start()516dashboard = DashBoard()517dashboard.launch_dashboard()