# VehiLeaks robots.txt
#
# Policy (golden middle): search indexing YES, AI answers/citation YES,
# AI model training NO. Only the compliant crawlers of the major model
# vendors are named here — exotic/non-compliant scrapers (Bytespider & co)
# ignore robots.txt anyway and are handled by Cloudflare at the edge.
#
# NOTE: while Cloudflare's "Managed robots.txt" feature is ON for this zone,
# Cloudflare prepends its own managed block, which may re-block bots this
# file intentionally allows (e.g. Google-Extended). Turn the managed feature
# OFF so this file is the single source of truth.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Disallow: /report/
# Locale-prefixed report pages (/uk/report/, /pl/report/, …) — the plain
# /report/ rule only matches from the root, so the wildcard is needed too.
Disallow: /*/report/
# Logged-in area (reports cabinet lives at /account/reports/).
Disallow: /account
Disallow: /vehicle/Listing-
# Локальні префікси того самого простору (/pl/vehicle/Listing-…): правило
# від кореня їх не матчить, потрібен wildcard-двійник (як у /*/report/).
Disallow: /*/vehicle/Listing-
Disallow: /*?*
# Sitelinks-searchbox SearchAction target (?vin=VIN redirects to the vehicle
# page). Longer pattern outranks the generic /*?* block above.
Allow: /*?vin=
# UTM-tagged VDP/homepage URLs (vinassessment cross-promo, mailings, ads)
# accumulated in GSC as "Blocked by robots.txt" (1.6K+ URLs, validation
# stuck on Failed forever). Letting Google crawl them reveals the
# self-canonical to the clean page and drains the backlog. SCOPED on
# purpose: a global "Allow: /*?utm_source=" would out-rank the SHORTER
# Disallows above by longest-match and reopen /report/, /account, /users
# etc. for any ?utm_source= link (mailings link /report/ with UTM!).
# Public VIN pages accept any first UTM key, so campaign/medium/source
# ordering does not stop crawlers from reading the clean canonical. Only
# explicit public VDP prefixes are allowed; unknown first query parameters
# and account/report/search routes remain blocked. Attribution is preserved.
# Each Allow has a longer Listing-space denial to retain its exclusion.
Allow: /vehicle/*?utm_
Disallow: /vehicle/Listing-*?utm_
Allow: /en/vehicle/*?utm_
Disallow: /en/vehicle/Listing-*?utm_
Allow: /uk/vehicle/*?utm_
Disallow: /uk/vehicle/Listing-*?utm_
Allow: /lt/vehicle/*?utm_
Disallow: /lt/vehicle/Listing-*?utm_
Allow: /lv/vehicle/*?utm_
Disallow: /lv/vehicle/Listing-*?utm_
Allow: /ee/vehicle/*?utm_
Disallow: /ee/vehicle/Listing-*?utm_
Allow: /pl/vehicle/*?utm_
Disallow: /pl/vehicle/Listing-*?utm_
Allow: /kz/vehicle/*?utm_
Disallow: /kz/vehicle/Listing-*?utm_
Allow: /uz/vehicle/*?utm_
Disallow: /uz/vehicle/Listing-*?utm_
Allow: /ru/vehicle/*?utm_
Disallow: /ru/vehicle/Listing-*?utm_
Allow: /?utm_source=
# The other public marketing targets UTM'd links actually point to
# (vinassessment nav + Brevo mailings) — without these the "Blocked by
# robots.txt" backlog keeps refilling from live link sources.
Allow: /car_stickers?utm_source=
Allow: /sample_reports/*?utm_source=
Allow: /guides*?utm_source=
Allow: /free-vin-check*?utm_source=
# Рекламні клік-URL (?gclid=, ?gad_source=): без винятку Ads-цілі були
# некраулабельні для Googlebot ("Destination not crawlable" б'є по Quality
# Score) — CF-виняток для них зробили 2026-08-02, robots лишався закритим.
# СКОУПЛЕНО так само, як utm вище: глобальний Allow: /*?gclid= перевідкрив
# би /report/ і /account для будь-якого лінка з ?gclid.
Allow: /vehicle/*?gclid=
Allow: /vehicle/*?gad_source=
Disallow: /vehicle/Listing-*?gclid=
Disallow: /vehicle/Listing-*?gad_source=
Allow: /?gclid=
Allow: /?gad_source=
# Lazy turbo-frame секції VDP (?lazy_section=ownership|history|market|catalog):
# рендерер Google (WRS) поважає robots.txt для СУБресурсів — заблокований
# фрагмент = Google не бачить секції взагалі. Allow довший за /*?*, тож
# перемагає; контр-Disallow тримає Listing-простір закритим (як для utm).
Allow: /vehicle/*?lazy_section=
Disallow: /vehicle/Listing-*?lazy_section=
# Хешовані асети з cache-buster (?v=…) підпадали під /*?* — рендер без
# шрифту/стилів і «blocked resource» у GSC. Найдовший збіг → дозволено.
Allow: /vite/assets/
Allow: /assets/
# QR-код сторінки (<img src="/qr?data=…">) — теж ресурс рендера.
Allow: /qr?data=
Disallow: /*?show_modal_for=
Disallow: /users
Disallow: /devise
Disallow: /sign_in
Disallow: /sign_up
Disallow: /password
Disallow: /billing
Disallow: /checkout
Disallow: /payment_intents
# Free AI-citable API (JSON facts + canonical_url). Longer pattern outranks
# the generic /api block below; explicit market_price line is a safety net
# for naive first-match crawlers (query-string endpoint).
Allow: /api/ai
Allow: /api/ai/market_price
Disallow: /api
Disallow: /admin
Disallow: /backend
Disallow: /customer_backend
Disallow: /express_authentications
# Merchant Center fetches the product feed as Googlebot and obeys robots.txt,
# so the feed URL itself must stay reachable — the longer Allow outranks the
# Disallow. Everything else under /merchant/ stays out of the index.
Allow: /merchant/products.xml
Disallow: /merchant/

# Ahrefs SEO crawler — honest and robots-compliant, but heavy (~7.9k req/h
# across our zones). Crawl-delay: 30 throttles it to ~120 req/h while keeping
# the site in the Ahrefs index (useful for our own backlink analytics).
# Named group = AhrefsBot leaves the "*" group, so the Disallows are restated
# (the UTM/gclid Allow exceptions are deliberately NOT repeated — those exist
# only to drain the Google/Bing GSC backlog; for Ahrefs less crawl is the
# point).
User-agent: AhrefsBot
Crawl-delay: 30
Disallow: /report/
Disallow: /*/report/
Disallow: /account
Disallow: /vehicle/Listing-
Disallow: /*?*
Disallow: /users
Disallow: /devise
Disallow: /sign_in
Disallow: /sign_up
Disallow: /password
Disallow: /billing
Disallow: /checkout
Disallow: /payment_intents
Disallow: /api
Disallow: /admin
Disallow: /backend
Disallow: /customer_backend
Disallow: /express_authentications
Disallow: /merchant/

# AI SEARCH / CITATION crawlers — ALLOWED (they read live pages to cite us in
# ChatGPT/Claude/Perplexity/Gemini answers; they do NOT train on our data).
# Gemini grounding uses Google-Extended, which is deliberately NOT blocked
# anywhere in this file; Google AI Mode/Overviews use the standard Googlebot.
# Per the robots spec a named group does NOT inherit `User-agent: *` rules, so
# the sensitive-path Disallows are repeated here. The crawl-budget rules
# (/vehicle/Listing-, /*?*) are intentionally NOT repeated — on-demand
# citation fetchers have no crawl budget to waste.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: DuckAssistBot
User-agent: Meta-ExternalFetcher
User-agent: MistralAI-User
Allow: /
Disallow: /report/
Disallow: /*/report/
Disallow: /account
Disallow: /users
Disallow: /devise
Disallow: /sign_in
Disallow: /sign_up
Disallow: /password
Disallow: /billing
Disallow: /checkout
Disallow: /payment_intents
# Free AI-citable API (JSON facts + canonical_url). Longer pattern outranks
# the generic /api block below; explicit market_price line is a safety net
# for naive first-match crawlers (query-string endpoint).
Allow: /api/ai
Allow: /api/ai/market_price
Disallow: /api
Disallow: /admin
Disallow: /backend
Disallow: /customer_backend
Disallow: /express_authentications
Disallow: /merchant/

# AI TRAINING crawlers of the major model vendors — allowed ONLY on the
# marketing/guide pages: brand recall inside model weights is welcome, the
# 125M-vehicle data corpus is not (it stays ai-train=no via the Disallow: /).
# Longest-match rule: the specific Allows below beat the catch-all Disallow.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: meta-externalagent
User-agent: Applebot-Extended
Content-Signal: search=no, ai-input=no, ai-train=yes
# Training bots have no freshness deadline, so 1 req/s is plenty. Added after
# ClaudeBot hit sibling sites at ~11 r/s (2026-08-17); nginx also enforces
# 2 r/s for ClaudeBot with 429 + Retry-After as the hard backstop.
Crawl-delay: 1
Allow: /$
Allow: /guides
Allow: /free-vin-check
Allow: /vehicle-history-report
Allow: /check-vin-history
Allow: /lookup-vehicle-history-by-vin
Allow: /license-plate-lookup
Allow: /compare-vehicle-reports
Allow: /how-to/
Disallow: /

Sitemap: https://vehileaks.com/sitemap_index.xml