enigmare/v2-crawler

1{2  "targetExamples": 10000,3  "preparedOutputPath": "output/sft/sft_prepared.jsonl",4  "seed": 42,5  "deduplication": {6    "enabled": true,7    "exactDeduplication": true,8    "normalizedDeduplication": true,9    "semanticDeduplication": true,10    "similarityThreshold": 0.8811  },12  "sources": {13    "stackoverflow": {14      "enabled": true,15      "percentage": 100,16      "tags": [17        "javascript",18        "typescript",19        "reactjs",20        "next.js",21        "node.js",22        "angular",23        "svelte",24        "nuxt.js",25        "vite",26        "tailwind-css",27        "redux",28        "tanstack-query",29        "zod",30        "prisma",31        "mongoose",32        "mongodb",33        "postgresql",34        "mysql",35        "git"36      ],37      "maxQuestionsPerTag": 1000,38      "minScore": 1,39      "acceptedAnswersOnly": false40    },41    "github": {42      "enabled": false,43      "percentage": 40,44      "repositories": ["facebook/react", "vercel/next.js", "python/cpython"],45      "contentTypes": ["issues", "discussions"],46      "state": "closed",47      "labels": ["bug", "question"],48      "minComments": 1,49      "maxIssuesPerRepo": 1050    }51  }52}53