# ========================= # Global defaults # ========================= User-agent: * Crawl-delay: 5 # Block APIs Disallow: /apis/ # Allow core content Allow: / Allow: /blogs/ Allow: /blogs/* # Block crawler explosions (deep paths) Disallow: /blogs/*/*/*/*/ Disallow: /blogs/*/*/*/*/*/ Disallow: /blogs/*/*/*/*/*/*/ # Block internal & utility paths (controlled) Disallow: /wp-admin/ Disallow: /admin/ Disallow: /login/ # Allow only root-level ourpackages Allow: /ourpackages/ Disallow: /*/ourpackages/ # ========================= # Google # ========================= User-agent: Googlebot Allow: / Crawl-delay: 2 # ========================= # Bing # ========================= User-agent: Bingbot Allow: / Crawl-delay: 2 # ========================= # OpenAI GPTBot # ========================= User-agent: GPTBot Allow: / Disallow: /apis/ Disallow: /admin/ Disallow: /login/ Disallow: /wp-admin/ Disallow: /blogs/*/*/*/*/ Disallow: /blogs/*/*/*/*/*/ # ========================= # Anthropic Claude-Web # ========================= User-agent: Claude-Web Allow: / Disallow: /apis/ Disallow: /admin/ Disallow: /login/ Disallow: /wp-admin/ # ========================= # Perplexity # ========================= User-agent: PerplexityBot Allow: / Disallow: /apis/ Disallow: /admin/ Disallow: /login/ Disallow: /wp-admin/ # ========================= # Google-Extended (AI Overviews) # ========================= User-agent: Google-Extended Allow: / Disallow: /apis/ Disallow: /admin/ Disallow: /login/ Disallow: /wp-admin/ # ========================= # Block aggressive SEO bots # ========================= User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / # ========================= # Sitemap # ========================= Sitemap: https://www.docgenie.in/sitemap.xml