User-agent: *
# Disallow all URLs that contain parameters
Disallow: /*?

# Allow only if query starts with ?ver= (any value)
Allow: /*?ver=

# Disallow feed pages
Disallow: /feed/

# Disallow favicon and static media files
Disallow: /_next/static/media/

# Disallow API-related URLs
Disallow: /api.incruiter/
Disallow: /user-auth-system/

# Disallow access to the development environment
Disallow: /dev/

# Allow everything else
Allow: /

# Allow OpenAI crawlers
User-agent: GPTBot
Allow: /

User-agent: OpenAI-Research
Allow: /

User-agent: ChatGPT-User
Allow: /

# Allow Bing's primary crawler
User-agent: Bingbot
Allow: /

# Allow Perplexity AI's crawler
User-agent: PerplexityBot
Allow: /

Sitemap: https://incruiter.com/sitemap.xml
Sitemap: https://incruiter.com/blog/sitemap.xml