enigmare/v2-crawler

1{2  "sites": [3    "https://nextjs.org/docs",4    "https://react.dev/learn",5    "https://www.typescriptlang.org/docs/",6    "https://developer.mozilla.org/en-US/docs/Web",7    "https://nodejs.org/en/learn",8    "https://expressjs.com/en/guide/",9    "https://docs.nestjs.com/",10    "https://fastapi.tiangolo.com/",11    "https://docs.djangoproject.com/en/stable/",12    "https://flask.palletsprojects.com/en/stable/",13    "https://spring.io/guides",14    "https://laravel.com/docs",15    "https://rubyonrails.org/docs",16 17    "https://huggingface.co/docs/transformers/index",18    "https://huggingface.co/docs/datasets/index",19    "https://huggingface.co/docs/diffusers/index",20    "https://huggingface.co/docs/peft/index",21    "https://huggingface.co/docs/trl/index",22    "https://huggingface.co/docs/accelerate/index",23    "https://python.langchain.com/docs/",24    "https://docs.llamaindex.ai/en/stable/",25    "https://docs.vllm.ai/en/latest/",26    "https://docs.pytorch.org/tutorials/",27    "https://pytorch.org/vision/stable/index.html",28    "https://pytorch.org/audio/stable/index.html",29    "https://lightning.ai/docs/pytorch/stable/",30    "https://www.tensorflow.org/guide",31    "https://scikit-learn.org/stable/user_guide.html",32    "https://docs.jax.dev/en/latest/",33    "https://pandas.pydata.org/docs/getting_started/",34    "https://pandas.pydata.org/docs/user_guide/index.html",35    "https://numpy.org/learn/",36    "https://numpy.org/doc/stable/user/",37    "https://docs.scipy.org/doc/scipy/",38    "https://docs.ray.io/",39    "https://www.deepspeed.ai/tutorials/",40    "https://triton-lang.org/main/index.html",41    "https://onnxruntime.ai/docs/",42 43    "https://supabase.com/docs",44    "https://www.prisma.io/docs",45    "https://orm.drizzle.team/docs/overview",46    "https://tanstack.com/query/latest/docs/framework/react/overview",47    "https://docs.astro.build/en/getting-started/",48    "https://svelte.dev/docs/",49    "https://tailwindcss.com/docs",50    "https://bun.sh/docs",51    "https://developers.cloudflare.com/workers/",52    "https://qdrant.tech/documentation/",53    "https://ziglang.org/documentation/master/",54 55    "https://www.postgresql.org/docs/",56    "https://www.mongodb.com/docs/",57    "https://redis.io/docs/latest/",58    "https://sqlite.org/docs.html",59    "https://surrealdb.com/docs",60    "https://duckdb.org/docs/",61    "https://clickhouse.com/docs/en/",62    "https://www.elastic.co/guide/index.html",63    "https://docs.opensearch.org/latest/",64    "https://neon.com/docs/introduction",65    "https://planetscale.com/docs",66    "https://docs.turso.tech/introduction",67 68    "https://docs.docker.com/get-started/",69    "https://kubernetes.io/docs/home/",70    "https://kubernetes.io/docs/tutorials/",71    "https://developer.hashicorp.com/terraform/tutorials",72    "https://docs.ansible.com/",73    "https://nginx.org/en/docs/",74    "https://prometheus.io/docs/introduction/overview/",75    "https://www.envoyproxy.io/docs/envoy/latest/",76    "https://istio.io/latest/docs/",77    "https://spark.apache.org/docs/latest/",78    "https://kafka.apache.org/documentation/",79    "https://nightlies.apache.org/flink/flink-docs-stable/",80    "https://cassandra.apache.org/doc/latest/",81    "https://grpc.io/docs/",82    "https://protobuf.dev/",83    "https://graphql.org/learn/",84 85    "https://docs.github.com/en/actions",86    "https://docs.github.com/en/get-started",87    "https://docs.github.com/en/rest",88    "https://docs.gitlab.com/",89    "https://docs.aws.amazon.com/",90    "https://docs.aws.amazon.com/s3/",91    "https://cloud.google.com/docs",92    "https://learn.microsoft.com/en-us/azure/",93    "https://vercel.com/docs",94    "https://docs.netlify.com/",95    "https://render.com/docs",96    "https://docs.railway.app/",97 98    "https://doc.rust-lang.org/book/",99    "https://doc.rust-lang.org/std/",100    "https://doc.rust-lang.org/cargo/",101    "https://go.dev/doc/",102    "https://pkg.go.dev/std",103    "https://en.cppreference.com/w/",104    "https://llvm.org/docs/",105    "https://docs.oracle.com/en/java/",106    "https://kotlinlang.org/docs/home.html",107    "https://elixir-lang.org/getting-started/introduction.html",108    "https://www.lua.org/manual/5.4/",109    "https://docs.julialang.org/en/v1/",110    "https://cmake.org/documentation/",111    "https://git-scm.com/doc",112    "https://www.gnu.org/software/bash/manual/",113 114    "https://docs.soliditylang.org/en/latest/",115    "https://ethereum.org/en/developers/docs/",116    "https://book.getfoundry.sh/",117    "https://hardhat.org/docs",118    "https://docs.ethers.org/v6/",119 120    "https://docs.npmjs.com/",121    "https://yarnpkg.com/getting-started",122    "https://pnpm.io/motivation",123    "https://webpack.js.org/concepts/",124    "https://esbuild.github.io/",125    "https://eslint.org/docs/latest/",126    "https://prettier.io/docs/en/index.html",127 128    "https://playwright.dev/docs/intro",129    "https://docs.cypress.io/guides/overview/why-cypress",130    "https://docs.pytest.org/en/stable/",131    "https://vitest.dev/guide/",132 133    "https://developer.flutterwave.com/",134    "https://docs.stripe.com/",135    "https://developer.paypal.com/",136    "https://docs.adyen.com/",137    "https://developer.squareup.com/docs",138 139    "https://auth0.com/docs/",140    "https://clerk.com/docs",141    "https://firebase.google.com/docs/auth",142    "https://developer.okta.com/",143 144    "https://www.twilio.com/docs/",145    "https://www.twilio.com/docs/sendgrid",146    "https://resend.com/docs/introduction",147    "https://developers.brevo.com/docs/getting-started",148 149    "https://firebase.google.com/docs/",150    "https://appwrite.io/docs",151    "https://docs.convex.dev/",152    "https://developers.cloudflare.com/r2/",153 154    "https://developers.openai.com/api/docs",155    "https://ai.google.dev/api",156    "https://platform.claude.com/docs/en/home",157    "https://docs.cohere.com/",158    "https://docs.mistral.ai/",159    "https://console.groq.com/docs/overview",160    "https://docs.together.ai/intro",161 162    "https://developer.atlassian.com/",163    "https://developers.google.com/",164    "https://developers.facebook.com/docs/development"165  ],166  "maxPages": 2500,167  "maxDepth": 8,168  "concurrency": 16,169  "siteConcurrency": 8,170  "requestDelayMs": 80,171  "respectRobotsTxt": true,172  "sameDomainOnly": true,173  "scopedPathOnly": true,174  "fresh": false,175  "retryCount": 3,176  "requestTimeoutMs": 15000,177  "maxErrorsPerDomain": 20,178  "outputDir": "output/raw/documentation"179}180