# Robots.txt for GoodSpace.ai
# Updated: 2025-11-16
# Synchronized with Cloudflare Worker bot detection for prerendering

# ============================================
# DEFAULT ACCESS FOR ALL CRAWLERS
# ============================================
User-agent: *
Allow: /

# Block utility pages and flows
Disallow: /create-job-post/
Disallow: /auth/
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /reset-password
Disallow: /verify/
Disallow: /dashboard/
Disallow: /account/
Disallow: /settings/
Disallow: /onboarding/
Disallow: /payment/
Disallow: /checkout/
Disallow: /api/
Disallow: /admin/

# Block user profiles (except curated ones)
Disallow: /users/
Allow: /users/featured/



# ============================================
# TRADITIONAL SEARCH ENGINES - ALLOW WITH CRAWL DELAY
# ============================================
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

User-agent: Yandex
Allow: /
Crawl-delay: 2

# ============================================
# AI/LLM CRAWLERS - ALLOW FOR AI-POWERED SEARCH & DISCOVERY
# ============================================
# OpenAI Bots
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic Bots
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# Google AI Bots
User-agent: Google-Extended
Allow: /

# Microsoft AI Bots
User-agent: BingChatBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Other AI Crawlers
User-agent: YouBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

# ============================================
# SOCIAL MEDIA BOTS - ALLOW FOR LINK PREVIEWS
# ============================================
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Telegram
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Pinterest
Allow: /

User-agent: PinterestBot
Allow: /

User-agent: redditbot
Allow: /

User-agent: Tumblr
Allow: /

User-agent: SkypeUriPreview
Allow: /

User-agent: Flipboard
Allow: /

User-agent: vkShare
Allow: /

User-agent: Viber
Allow: /

User-agent: KakaoTalk
Allow: /

User-agent: WeChat
Allow: /

User-agent: Line
Allow: /

# ============================================
# SEO & ANALYTICS TOOLS - CONTROLLED ACCESS
# ============================================
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: DotBot
Allow: /
Crawl-delay: 10

User-agent: Majestic
Allow: /
Crawl-delay: 10

User-agent: MJ12bot
Allow: /
Crawl-delay: 10

User-agent: SerpstatBot
Allow: /
Crawl-delay: 10

User-agent: Screaming Frog SEO Spider
Allow: /
Crawl-delay: 5

User-agent: GTmetrix
Allow: /
Crawl-delay: 5

User-agent: Pingdom
Allow: /
Crawl-delay: 5

User-agent: Site24x7
Allow: /
Crawl-delay: 5

User-agent: BlexBot
Allow: /
Crawl-delay: 10

User-agent: SeoStar
Allow: /
Crawl-delay: 10

User-agent: AlexaBot
Allow: /
Crawl-delay: 10

# ============================================
# REGIONAL SEARCH ENGINES
# ============================================
User-agent: Sogou
Allow: /
Crawl-delay: 2

User-agent: 360Spider
Allow: /
Crawl-delay: 2

User-agent: Naver
Allow: /
Crawl-delay: 2

User-agent: SeznamBot
Allow: /
Crawl-delay: 2

User-agent: Mail.RU
Allow: /
Crawl-delay: 2

User-agent: DaumOa
Allow: /
Crawl-delay: 2

User-agent: Qwantify
Allow: /
Crawl-delay: 2

User-agent: PetalSearch
Allow: /
Crawl-delay: 2

# ============================================
# DEVELOPER TOOLS & VALIDATORS
# ============================================
User-agent: W3C_Validator
Allow: /

User-agent: Chrome-Lighthouse
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: Google Page Speed
Allow: /

User-agent: PageSpeed
Allow: /

User-agent: Lighthouse
Allow: /

User-agent: PTST
Allow: /

User-agent: validator.nu
Allow: /

# ============================================
# APPLE & AMAZON SERVICES
# ============================================
User-agent: Applebot
Allow: /
Crawl-delay: 1

User-agent: Apple-Applebot
Allow: /
Crawl-delay: 1

User-agent: Amazonbot
Allow: /
Crawl-delay: 2

User-agent: Alexabot
Allow: /
Crawl-delay: 2

# ============================================
# OTHER IMPORTANT BOTS
# ============================================
User-agent: Rogerbot
Allow: /
Crawl-delay: 2

User-agent: Embedly
Allow: /

User-agent: Quora link preview
Allow: /

User-agent: Showyoubot
Allow: /

User-agent: Outbrain
Allow: /

User-agent: developers.google.com/+/web/snippet
Allow: /

User-agent: Nuzzel
Allow: /

User-agent: Bitlybot
Allow: /

User-agent: bitrix link preview
Allow: /

User-agent: XING-contenttabreceiver
Allow: /

User-agent: integration-test
Allow: /

# ============================================
# SITEMAP
# ============================================
Sitemap: https://goodspace.ai/sitemap_index.xml