neoneye/base64-decode-v1
Dataset: Base64 decode version1 This dataset is for improving base64 decoding capabilities. The number of bytes that are in the base64 encoded data spans between 0..127 bytes. GPT 4o is great at base64 decoding. However llama3 is terrible at base64 decoding. Short examples of what data.jsonl looks like: {"instruction": "Transform base64 to HEX", "input": "464pNBlIObA=", "output": "e3ae2934194839b0"} {"instruction": "Decode Base64 to json", "input": "NQ==", "output": "[53]"}… See the full description on the dataset page: https://huggingface.co/datasets/neoneye/base64-decode-v1.
044
1from random_data import generate_random_byte_array2import json3import os4import random5import base646 7def base64_encode_byte_array(byte_array):8 return base64.b64encode(byte_array).decode('utf-8')9 10def generate_dataset_item(seed):11 length = random.Random(seed + 1000).randint(0, 127)12 byte_array = generate_random_byte_array(length=length, seed=seed + 1001)13 14 output_formats = [15 'hex', 16 'json'17 ]18 output_format = random.Random(seed + 1002).choice(output_formats)19 20 names_hex = [21 'Hexadecimal',22 'hexadecimal',23 'hex',24 'Hex',25 'HEX',26 ]27 names_json = [28 'Json',29 'json',30 'JSON',31 ]32 33 name_output = None34 if output_format == 'hex':35 name_output = random.Random(seed + 1003).choice(names_hex)36 else:37 if output_format == 'json':38 name_output = random.Random(seed + 1004).choice(names_json)39 40 name_inputs = [41 'base64',42 'Base64',43 'BASE64',44 ]45 name_input = random.Random(seed + 1005).choice(name_inputs)46 47 instructions = [48 f'Decode {name_input} to {name_output}',49 f'decode {name_input} to {name_output}',50 f'convert {name_input} to {name_output}',51 f'Convert {name_input} to {name_output}',52 f'Transform {name_input} to {name_output}',53 f'transform {name_input} to {name_output}',54 f'Change {name_input} to {name_output}',55 f'change {name_input} to {name_output}',56 f'{name_input} to {name_output}',57 f'{name_output} from {name_input}',58 ]59 60 instruction = random.Random(seed + 1006).choice(instructions)61 62 input = base64_encode_byte_array(byte_array)63 64 output = None65 if output_format == 'hex':66 output = byte_array.hex()67 else:68 if output_format == 'json':69 output = json.dumps(list(byte_array), separators=(',', ':'))70 71 dict = {72 'instruction': instruction,73 'input': input,74 'output': output75 }76 return dict77 78def generate_dataset(max_num_samples=1000, max_byte_size=1024*1024, seed_start=0):79 dataset = []80 dataset_byte_size = 081 for i in range(max_num_samples):82 item = generate_dataset_item(seed_start + i)83 bytes = len(json.dumps(item))84 if dataset_byte_size + bytes > max_byte_size:85 break86 dataset_byte_size += bytes87 dataset.append(item)88 return dataset89 90dataset = generate_dataset(91 max_num_samples=50000,92 max_byte_size=1024*1024*20,93)94 95# Save dataset to file96filename = 'data.jsonl'97with open(filename, 'w') as f:98 for item in dataset:99 f.write(json.dumps(item) + '\n')100 101# Summary102file_size = os.path.getsize(filename)103print(f"Generated {len(dataset)} samples, saved to {filename}, file size: {file_size} bytes.")104 105