CoolFace
Datasetpublic

novaia/world-heightmaps-360px

World Heightmaps 360px This is a dataset of 360x360 Earth heightmaps generated from SRTM 1 Arc-Second Global. Each heightmap is labelled according to its latitude and longitude. There are 573,995 samples. Method Convert GeoTIFFs into PNGs with Python and Rasterio. import rasterio import matplotlib.pyplot as plt import os input_directory = '...' output_directory = '...' file_list = os.listdir(input_directory) for i in range(len(file_list)): image =… See the full description on the dataset page: https://huggingface.co/datasets/novaia/world-heightmaps-360px.

sourceHugging Faceapache-2.0updated 3y agoView on Hugging Face
3likes374downloads
Dataset Card

World Heightmaps 360px

This is a dataset of 360x360 Earth heightmaps generated from SRTM 1 Arc-Second Global. Each heightmap is labelled according to its latitude and longitude. There are 573,995 samples.

Method

  1. 1.Convert GeoTIFFs into PNGs with Python and Rasterio.
python
import rasterio
import matplotlib.pyplot as plt
import os

input_directory = '...'
output_directory = '...'
file_list = os.listdir(input_directory)

for i in range(len(file_list)):
    image = rasterio.open(input_directory + file_list[i])
    plt.imsave(output_directory + file_list[i][0:-4] + '.png', image.read(1), cmap='gray')
  1. 1.Split PNGs into 100 patches with Split Image.
python
from split_image import split_image
import os

input_directory = '...'
output_directory = '...'
file_list = os.listdir(input_directory)

for i in range(len(file_list)):
    split_image(input_directory + file_list[i], 10, 10, should_square=True, should_cleanup=False, output_dir=output_directory)
  1. 1.Hand pick a dataset of corrupted and uncorrupted heightmaps then train a discriminator to automatically filter the whole dataset.
  1. 1.Compile images into parquet files.
python
import pyarrow as pa
import pyarrow.parquet as pq
import pandas as pd
from PIL import Image
import os
import io
import json

samples_per_file = 6_000

root_dir = 'data/datasets/world-heightmaps-360px-png'
df = pd.read_csv(os.path.join(root_dir, 'metadata.csv'))
df = df.sample(frac=1).reset_index(drop=True)

def save_table(image_data, table_number):
    print(f'Entries in table {table_number}: {len(image_data)}')
    schema = pa.schema(
        fields=[
            ('heightmap', pa.struct([('bytes', pa.binary()), ('path', pa.string())])),
            ('latitude', pa.string()),
            ('longitude', pa.string())
        ],
        metadata={
            b'huggingface': json.dumps({
                'info': {
                    'features': {
                        'heightmap': {'_type': 'Image'},
                        'latitude': {'_type': 'Value', 'dtype': 'string'},
                        'longitude': {'_type': 'Value', 'dtype': 'string'}
                    }
                }
            }).encode('utf-8')
        }
    )

    table = pa.Table.from_pylist(image_data, schema=schema)
    pq.write_table(table, f'data/world-heightmaps-360px-parquet/{str(table_number).zfill(4)}.parquet')

image_data = []
samples_in_current_file = 0
current_file_number = 0
for i, row in df.iterrows():
    if samples_in_current_file >= samples_per_file:
        save_table(image_data, current_file_number)
        image_data = []
        samples_in_current_file = 0
        current_file_number += 1
    samples_in_current_file += 1
    image_path = row['file_name']
    with Image.open(os.path.join(root_dir, image_path)) as image:
        image_bytes = io.BytesIO()
        image.save(image_bytes, format='PNG')
        image_dict = {
            'heightmap': {
                'bytes': image_bytes.getvalue(),
                'path': image_path
            },
            'latitude': str(row['latitude']),
            'longitude': str(row['longitude'])
        }
        image_data.append(image_dict)

save_table(image_data, current_file_number)