User-agent: *
Allow: /

# Block sensitive backend routes & database webhooks
Disallow: /api/auth/
Disallow: /api/paddle-webhook
Disallow: /api/latest-license

# Allow public agent endpoints
Allow: /api/agent/

# Discovery & Agent Resources
Sitemap: https://cheapcrawler.com/sitemap.xml
# LLM Context: https://cheapcrawler.com/llms.txt
# Agent Purchasing API: https://cheapcrawler.com/api/agent/buy
# Documentation: https://cheapcrawler.com/docs/