# robots.txt for mubashirhassan.com

User-agent: *
Allow: /

# Sitemap locations
Sitemap: https://www.mubashirhassan.com/sitemap_index.xml
Sitemap: https://www.mubashirhassan.com/sitemap.xml

# LLMs.txt for AI crawlers
# See: https://www.mubashirhassan.com/llms.txt

# Crawl delay (optional, in seconds)
Crawl-delay: 1

# Allow all major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

# AI Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: GoogleOther
Allow: /

# Block specific paths if needed (currently allowing all)
# Disallow: /admin/
# Disallow: /private/
# Disallow: /snippets/

# Exclude template files from indexing
Disallow: /location-page-template.html
Disallow: /country-hub-template.html

# Block malformed location URLs with double cities
Disallow: /locations/*/*/*/

# Block duplicate country pages
Disallow: /locations/pakistan/pakistan/
Disallow: /locations/saudi-arabia/saudi-arabia/
Disallow: /locations/uk/uk/
Disallow: /locations/uae/uae/
Disallow: /locations/qatar/qatar/
Disallow: /locations/canada/canada/
Disallow: /locations/usa/usa/

# Block invalid nested country paths
Disallow: /locations/pakistan/uae/
Disallow: /locations/pakistan/saudi-arabia/
Disallow: /locations/usa/pakistan/
Disallow: /locations/saudi-arabia/usa/

# Block search and feed URLs
Disallow: /search/
Disallow: /feed/
Disallow: */feed/rss2/

# Block unnecessary paths
Disallow: /pages/
Disallow: /tr/
Disallow: /snippets/