dataunitylab/json-schema
JSON Schema Dataset This dataset consists of a collection of JSON Schema documents collected from GitHub by searching using the Sourcegraph API. Step 1: Find a list of JSON Schema paths The Sourcegraph code search API is used to find files with a .json extension and containing {\n "$schema": "https://json-schema.org/". This is somewhat restrictive, but still manages to find a large number of schemas. pipenv run python slurp.py --outfile repos.csv… See the full description on the dataset page: https://huggingface.co/datasets/dataunitylab/json-schema.
2205
1import json2import os3from pathlib import Path4import re5import sys6from urllib.request import urlretrieve7 8import fasttext9import tqdm10 11 12LANG_THRESHOLD = 0.113FASTTEXT_MODEL_URL = (14 "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"15)16JSON_SCHEMA_KEYWORDS = {17 "$anchor",18 "$comment",19 "$defs",20 "$dynamicAnchor",21 "$dynamicRef",22 "$id",23 "$recursiveAnchor",24 "$recursiveRef",25 "$ref",26 "$schema",27 "$vocabulary",28 "additionalItems",29 "additionalProperties",30 "allOf",31 "anyOf",32 "const",33 "contains",34 "contentEncoding",35 "contentMediaType",36 "contentSchema",37 "definitions",38 "dependencies",39 "dependentRequired",40 "dependentSchemas",41 "description",42 "disallow",43 "divisibleBy",44 "else",45 "enum",46 "exclusiveMaximum",47 "exclusiveMinimum",48 "extends",49 "format",50 "id",51 "if",52 "items",53 "maxContains",54 "maximum",55 "maxItems",56 "maxLength",57 "maxProperties",58 "minContains",59 "minimum",60 "minItems",61 "minLength",62 "minProperties",63 "multipleOf",64 "not",65 "oneOf",66 "pattern",67 "patternProperties",68 "prefixItems",69 "properties",70 "propertyNames",71 "required",72 "then",73 "title",74 "type",75 "unevaluatedItems",76 "unevaluatedProperties",77 "uniqueItems",78}79 80IGNORE_KEYWORDS = {81 "$id",82 "$schema",83 "$vocabulary",84 "format",85 "pattern",86 "type",87}88 89 90# Adapted from https://stackoverflow.com/a/37697078/12369591def identifier_split(id_str):92 return id_str93 return " ".join(94 re.sub("([A-Z][a-z]+)", r"_\1", re.sub("([A-Z]+)", r"_\1", id_str)).split("_")95 )96 97 98def collect_text(schema):99 """Generate a string of text from a schema, ignoring keywords"""100 text = ""101 102 if isinstance(schema, dict):103 for k, v in schema.items():104 # Ignore some keywords completely105 if k in IGNORE_KEYWORDS:106 continue107 108 # If the key is not a keyword, include it109 if k not in JSON_SCHEMA_KEYWORDS:110 text += " " + identifier_split(k)111 text += collect_text(v)112 113 elif isinstance(schema, list):114 text += " ".join(collect_text(v) for v in schema)115 116 elif isinstance(schema, str):117 # Include any found string values118 text += " " + schema119 120 return text.replace("\n", " ")121 122 123def get_languages(text):124 return {l.split("_")[-1]: p for (l, p) in zip(*model.predict(text, k=5))}125 126 127if __name__ == "__main__":128 # Download the language model if needed129 if not os.path.isfile("lid.176.bin"):130 urlretrieve(FASTTEXT_MODEL_URL, "lid.176.bin")131 model = fasttext.load_model("lid.176.bin")132 133 files = list(Path("valid_data").rglob("*.json"))134 for f in tqdm.tqdm(files):135 if not f.is_file():136 continue137 138 schema = json.load(f.open(encoding="utf-8"))139 schema_str = collect_text(schema)140 langs = get_languages(schema_str)141 top_lang, prob = max(langs.items(), key=lambda x: x[1])142 if prob < LANG_THRESHOLD:143 top_lang = None144 obj = {145 "repository": "/".join(f.parts[1:3]),146 "commit": f.parts[3],147 "path": str(Path(*f.parts[4:])),148 "language": top_lang,149 "languages": langs,150 }151 json.dump(obj, sys.stdout)152 sys.stdout.write("\n")153