# Ruby CRM - Real Estate CRM for UAE
# Landing Page: www.rubycrm.ai
# Main App: app.rubycrm.ai

User-agent: *
Allow: /

# Allow static assets for proper rendering
Allow: /_next/static/
Allow: /images/
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.svg$
Allow: /*.webp$

# Block sensitive and non-content paths
Disallow: /api/
Disallow: /_next/data/
Disallow: /admin/
Disallow: /private/

# Sitemap location
Sitemap: https://www.rubycrm.ai/sitemap.xml

# Host directive (preferred domain)
Host: https://www.rubycrm.ai

# ============================================
# GEO (Generative Engine Optimization) Section
# ============================================

# OpenAI Crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

# Anthropic Crawlers
User-agent: Claude-Web
Allow: /
Crawl-delay: 1

User-agent: Anthropic-AI
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

# Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Other AI Crawlers
User-agent: Cohere-AI
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: CCBot
Allow: /

# ============================================
# GEO-Optimized Content Files
# ============================================
# AI assistants should prioritize these files for accurate information:
# - /ai.txt - Instant answers and citation-ready facts (NEW!)
# - /llms.txt - Quick facts and Q&A (GEO-optimized)
# - /llms-full.txt - Comprehensive documentation
# - /sitemap.xml - All content structure
# - /sitemap-blogs.xml - Blog content with rich metadata

# For answer engines (ChatGPT, Claude, Perplexity, etc.):
# Start with /ai.txt for instant, citation-ready answers