User-agent: *
Disallow:
Crawl-Delay: 10

User-agent: 008
User-agent: BLEXBot
User-agent: MJ12bot
User-agent: Goodzer
User-agent: AhrefsBot
User-agent: spbot
User-agent: dotbot
User-agent: DotBot
User-agent: turnitinbot
User-agent: SeznamBot
User-Agent: The Knowledge AI
User-agent: CheckMarkNetwork
User-agent: Ever Accountable
User-agent: SeekportBot
User-agent: ClaudeBot
User-agent: MauiBot
User-agent: houzzbot
User-agent: Baiduspider
User-agent: Baiduspider-image
User-Agent: serpstatbot
User-agent: Sogou blog
User-agent: Sogou inst spider
User-agent: Sogou News Spider
User-agent: Sogou Orion spider
User-agent: Sogou spider2
User-agent: Sogou web spider
User-agent: Uptimebot
User-agent: Yandex
User-agent: YandexMobileBot
User-agent: ZoominfoBot
User-agent: MegaIndex.ru
User-agent: AlphaSeoBot-SA
User-agent: proximic
User-agent: Amazonbot
User-agent: PetalBot
User-agent: Re-re Studio
User-agent: barkrowler
User-agent: SiteAuditBot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Bytespider
User-agent: DataForSeoBot
User-agent: IonCrawl
User-agent: Nutch
# Need to allow the below bot, OAI-SearchBot, in order for our merchants products to show up in open ai. Im leaving the bot in the file, commented out, in case someone else notices the bot and wants to block it
# User-agent: OAI-SearchBot
User-agent: GPTBot
User-agent: ChatGPT-User
Disallow: /

# See if Semrush can behave - March 2023.
# Update Nov 2023, no, they can't behave.
User-agent: SemrushBot-BA
User-agent: SemrushBot
Disallow: /

Sitemap: /sitemap.xml