# ============================================================
# Trossen Robotics — robots.txt
# Last updated: April 2026
# Purpose: Keep public product, research, documentation, and support content
# crawlable for search engines and AI discovery while suppressing low-value
# Wix parameter URLs and selected low-quality bots.
# ============================================================

# ----- Default Rules -----
# Allow public crawling. Suppress Wix lightbox parameter URLs.
User-agent: *
Allow: /
Disallow: /*?lightbox=

# ----- Google Ads Bots -----
# Allow ad landing page validation, but exclude Wix internal partial/pro-gallery endpoints.
User-agent: AdsBot-Google-Mobile
Allow: /
Disallow: /*?lightbox=
Disallow: /_partials*
Disallow: /pro-gallery-webapp/v1/galleries/*

User-agent: AdsBot-Google
Allow: /
Disallow: /*?lightbox=
Disallow: /_partials*
Disallow: /pro-gallery-webapp/v1/galleries/*

# ----- Search and AI Discovery / Training Crawlers -----
# Shared disallow is repeated here because specific user-agent groups do not inherit
# User-agent:* rules.
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: GPTBot
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Amzn-SearchBot
Allow: /
Disallow: /*?lightbox=

# ----- Crawl Delay for Aggressive SEO Crawlers -----
# Crawl-delay is non-standard and not supported by Google, but some crawlers may respect it.
User-agent: DotBot
Allow: /
Disallow: /*?lightbox=
Crawl-delay: 10

User-agent: AhrefsBot
Allow: /
Disallow: /*?lightbox=
Crawl-delay: 10

# ----- Block Spam / Low-Quality Bots -----
User-agent: PetalBot
Disallow: /

# ----- Sitemap -----
Sitemap: https://www.trossenrobotics.com/sitemap.xml