User-agent: *
Disallow: /shopping/
Disallow: /login/
Disallow: /*?keyword=
Disallow: /*?keyword_this_page=
Disallow: /*/ezsearch_results.asp?cat1=*
Disallow: /*/?sort=*
Allow: /*lmode=10
Allow: /shop/
Allow: /product/
Allow: /category/
Allow: /images/
Allow: /css/
Allow: /js/

User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /
 
User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Yandex
Allow: /

User-agent: Sogou
Allow: /

User-agent: FacebookBot
Allow: /


User-agent: facebot
Allow: /

User-agent: ia_archiver
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: CCBot
Allow: /
 
User-agent: webcrawler
Disallow: /shopping/
 
User-agent: DuckAssistBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: PerplexityBot
Allow: /
Disallow: /shopping/

User-agent: ChatGPT-User
Allow: /

User-agent: cohere-ai
Allow: /
Disallow: /login/




User-agent: botify
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: SemrushBot
User-agent: proximic
User-agent: 360Spider
User-agent: RavenCrawler
User-agent: GPTBot
User-agent: anthropic-ai
User-agent: Omgilibot
User-agent: Omgili
User-agent: Diffbot
User-agent: amazonbot
User-agent: AI2Bot
User-agent: Bytespider
User-agent: Kangaroo Bot
User-agent: PanguBot
User-agent: ImagesiftBot
User-agent: cohere-training-data-crawler
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: YouBot
User-agent: BLEXBot
User-agent: spbot
User-agent: archive.org_bot
User-agent: CCBot
User-agent: SeekportBot
User-agent: XoviBot
User-agent: CocCocbot
User-agent: DotBot
User-agent: Exabot
User-agent: FreshBot
Disallow: /
 
Sitemap: https://www.compsource.com/sitemaps/siteindex1.xml