# ihatepdf.cv robots.txt
User-agent: *
Allow: /

# Block non-content directories
Disallow: /api/
Disallow: /admin/
Disallow: /src/
# Internal regression bench for the compression engine — never index.
Disallow: /compress-bench

# FIX: Block UTM and referral parameter URLs from being indexed.
# These were appearing as duplicate pages in Google Search Console.
Disallow: /*?ref=
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?utm_term=
Disallow: /*?utm_content=

# FIX: Block the literal ad template URL that Google was crawling.
# Caused by the SearchAction schema (now removed from index.html).
Disallow: /*?tool={

# Block non-essential build/config files
Disallow: /sw.js

# Allow crawling of main assets
Allow: /assets/
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.png$
Allow: /*.svg$
Allow: /*.webp$

# FIX: Updated sitemap URL from ihatepdf.cv to www.ihatepdf.cv
# to match the canonical domain. Mismatched sitemap domain confuses
# Google about which version of the site is authoritative.
Sitemap: https://www.ihatepdf.cv/sitemap.xml

# AI search and assistant crawlers — explicitly welcome.
#
# Two kinds are named here. Index crawlers (GPTBot, OAI-SearchBot, ClaudeBot,
# Claude-SearchBot, PerplexityBot, CCBot) build the indexes answer engines cite
# from. User-triggered fetchers (ChatGPT-User, Claude-User, Perplexity-User)
# load a page live while someone waits for an answer — a crawl delay on those
# only makes the answer slower or makes the engine give up on the page.
# Google-Extended and Applebot-Extended are opt-in tokens, not crawlers: they
# say whether Gemini and Apple Intelligence may use what Googlebot and Applebot
# already fetch.
#
# The retired anthropic-ai and Claude-Web tokens were replaced by ClaudeBot,
# Claude-User and Claude-SearchBot, and no crawl delays apply: a static site
# on a CDN is not at risk from these request rates.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
Allow: /
Disallow: /compress-bench

# A plain-text map of the site for language models: https://www.ihatepdf.cv/llms.txt

# Google bots
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

# Bing bot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yandex
User-agent: Yandex
Allow: /
Crawl-delay: 2

# Common bots
User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /
Crawl-delay: 2