Datasetpublic

novaia/world-heightmaps-256px

World Heightmaps 256px This is a dataset of 256x256 Earth heightmaps generated from SRTM 1 Arc-Second Global. Each heightmap is labelled according to its latitude and longitude. There are 573,995 samples. It is the same as World Heightmaps 360px but downsampled to 256x256. Method Convert GeoTIFFs into PNGs with Rasterio. import rasterio import matplotlib.pyplot as plt import os input_directory = '...' output_directory = '...' file_list =… See the full description on the dataset page: https://huggingface.co/datasets/novaia/world-heightmaps-256px.

sourceHugging Faceapache-2.0updated 3y agoView on Hugging Face

1likes560downloads

Dataset Card

World Heightmaps 256px

This is a dataset of 256x256 Earth heightmaps generated from SRTM 1 Arc-Second Global. Each heightmap is labelled according to its latitude and longitude. There are 573,995 samples. It is the same as World Heightmaps 360px but downsampled to 256x256.

Method

1.Convert GeoTIFFs into PNGs with Rasterio.

python

import rasterio
import matplotlib.pyplot as plt
import os

input_directory = '...'
output_directory = '...'
file_list = os.listdir(input_directory)

for i in range(len(file_list)):
    image = rasterio.open(input_directory + file_list[i])
    plt.imsave(output_directory + file_list[i][0:-4] + '.png', image.read(1), cmap='gray')

1.Split PNGs into 100 patches with Split Image.

python

from split_image import split_image
import os

input_directory = '...'
output_directory = '...'
file_list = os.listdir(input_directory)

for i in range(len(file_list)):
    split_image(input_directory + file_list[i], 10, 10, should_square=True, should_cleanup=False, output_dir=output_directory)

1.Hand pick a dataset of corrupted and uncorrupted heightmaps then train a discriminator to automatically filter the whole dataset.

1.Downsample from 360x360 to 256x256 with Pillow and the Lanczos resampling method.

python

import glob
from PIL import Image

paths = glob.glob('world-heightmaps-360px-png/data/*/*')

for file_name in paths:
    image = Image.open(file_name)
    if image.width == 256:
        continue
    print(file_name)
    image = image.resize((256, 256), resample=Image.LANCZOS)
    image.save(file_name)

1.Compile images into parquet files.

python

import pyarrow as pa
import pyarrow.parquet as pq
import pandas as pd
from PIL import Image
import os
import io
import json

samples_per_file = 10_000

root_dir = 'data/datasets/world-heightmaps-256px-png'
df = pd.read_csv(os.path.join(root_dir, 'metadata.csv'))
df = df.sample(frac=1).reset_index(drop=True)

def save_table(image_data, table_number):
    print(f'Entries in table {table_number}: {len(image_data)}')
    schema = pa.schema(
        fields=[
            ('heightmap', pa.struct([('bytes', pa.binary()), ('path', pa.string())])),
            ('latitude', pa.string()),
            ('longitude', pa.string())
        ],
        metadata={
            b'huggingface': json.dumps({
                'info': {
                    'features': {
                        'heightmap': {'_type': 'Image'},
                        'latitude': {'_type': 'Value', 'dtype': 'string'},
                        'longitude': {'_type': 'Value', 'dtype': 'string'}
                    }
                }
            }).encode('utf-8')
        }
    )

    table = pa.Table.from_pylist(image_data, schema=schema)
    pq.write_table(table, f'data/world-heightmaps-256px-parquet/{str(table_number).zfill(4)}.parquet')

image_data = []
samples_in_current_file = 0
current_file_number = 0
for i, row in df.iterrows():
    if samples_in_current_file >= samples_per_file:
        save_table(image_data, current_file_number)
        image_data = []
        samples_in_current_file = 0
        current_file_number += 1
    samples_in_current_file += 1
    image_path = row['file_name']
    with Image.open(os.path.join(root_dir, image_path)) as image:
        image_bytes = io.BytesIO()
        image.save(image_bytes, format='PNG')
        image_dict = {
            'heightmap': {
                'bytes': image_bytes.getvalue(),
                'path': image_path
            },
            'latitude': str(row['latitude']),
            'longitude': str(row['longitude'])
        }
        image_data.append(image_dict)

save_table(image_data, current_file_number)