User-agent: *
Allow: /

# The push endpoints live on the Cloudflare Worker, not on this domain.
# Googlebot parses inline JS and speculatively crawls path-like string
# literals, so it tries /push/subscribe here and gets a 404. Nothing to crawl.
Disallow: /push/

# Server-side only: the reader-vote endpoint and its SQLite store.
Disallow: /votes.php
Disallow: /data/

# The affiliate click redirector — no SEO value, wastes crawl budget.
Disallow: /go/

# Crawlers that return no visitors and cost the most to serve. These are also
# refused at the door by botguard.php - this is the polite request, that is the
# lock. Both exist because the well-behaved ones stop when asked, and the rest
# never read this file at all.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SERankingBacklinksBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

# Everything else that is neither a search engine nor named above: pace yourself.
# Google ignores crawl-delay (it is tuned in Search Console), which is exactly
# the intent - the crawler we WANT is unaffected.
User-agent: *
Crawl-delay: 5

Sitemap: https://aigirls.vip/sitemap.xml