# Presencis robots.txt
# https://presencis.com/robots.txt
# This static file IS the production robots.txt — published with the site into
# the presencis-site bucket that the Paris front door serves.
#
# STRATEGY: Block AI training crawlers. Welcome AI search crawlers.
# GPTBot = OpenAI TRAINING scraper -> BLOCK
# OAI-SearchBot = ChatGPT SEARCH citations -> ALLOW
# ClaudeBot = Anthropic TRAINING scraper -> BLOCK
# See internal SEO crawler policy.
#
# RFC 9309 §2.2.1: a crawler obeys ONLY the group that names it. So every named
# group that is allowed in carries the full private-area block below; a
# Disallow placed only under `*` would not apply to Googlebot, Bingbot and the
# other named search crawlers. scripts/test/robots-policy.test.mjs enforces it.

# ── AI Training Crawlers — BLOCKED ─────────────────────
# These crawlers collect content for model training, not search.
# Blocking establishes legal standing under CFAA (US) / Computer Misuse Act (UK).
#
# AI-TRAINING variants of otherwise-welcome crawlers are blocked granularly so the
# SEARCH / share-card sibling stays allowed (added 2026-07-21 after a crawler-
# visibility audit found Google-Extended served 200 while GPTBot/ClaudeBot 403'd):
#   Google-Extended  = Gemini/Vertex training (decoupled from Googlebot search — blocking does NOT affect ranking)
#   Applebot-Extended = Apple AI training (Applebot search stays WELCOME below)
#   meta-externalagent = Meta AI training (facebookexternalhit share-card fetcher stays WELCOME)
#
# One group (RFC 9309 allows several user-agent lines per group). `Disallow: /`
# already covers everything; the private block is repeated so that every named
# group carries it, as the policy test requires of all groups.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DataForSEO
User-agent: ByteSpider
User-agent: PetalBot
User-agent: Bytedance
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
Disallow: /
Disallow: /api/
Disallow: /app/
Disallow: /app
Disallow: /sys/
Disallow: /sys
Disallow: /admin/
Disallow: /admin
Disallow: /auth/
Disallow: /onboarding
Disallow: /accept-invite
Disallow: /reset-password
Disallow: /feed
Disallow: /dashboard
Disallow: /settings
Disallow: /billing
Disallow: /projects/
Disallow: /builder
Disallow: /builder-v2
Disallow: /ProjectBuilderV2
Disallow: /legacy
Disallow: /light
Disallow: /internal/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=

# ── AI Search Crawlers — WELCOME ──────────────────────
# These crawlers power AI search features that cite sources.
# We WANT to appear in AI-generated answers with attribution.

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /app
Disallow: /sys/
Disallow: /sys
Disallow: /admin/
Disallow: /admin
Disallow: /auth/
Disallow: /onboarding
Disallow: /accept-invite
Disallow: /reset-password
Disallow: /feed
Disallow: /dashboard
Disallow: /settings
Disallow: /billing
Disallow: /projects/
Disallow: /builder
Disallow: /builder-v2
Disallow: /ProjectBuilderV2
Disallow: /legacy
Disallow: /light
Disallow: /internal/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=
Crawl-delay: 1

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /app
Disallow: /sys/
Disallow: /sys
Disallow: /admin/
Disallow: /admin
Disallow: /auth/
Disallow: /onboarding
Disallow: /accept-invite
Disallow: /reset-password
Disallow: /feed
Disallow: /dashboard
Disallow: /settings
Disallow: /billing
Disallow: /projects/
Disallow: /builder
Disallow: /builder-v2
Disallow: /ProjectBuilderV2
Disallow: /legacy
Disallow: /light
Disallow: /internal/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=
Crawl-delay: 1

User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /app
Disallow: /sys/
Disallow: /sys
Disallow: /admin/
Disallow: /admin
Disallow: /auth/
Disallow: /onboarding
Disallow: /accept-invite
Disallow: /reset-password
Disallow: /feed
Disallow: /dashboard
Disallow: /settings
Disallow: /billing
Disallow: /projects/
Disallow: /builder
Disallow: /builder-v2
Disallow: /ProjectBuilderV2
Disallow: /legacy
Disallow: /light
Disallow: /internal/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /app
Disallow: /sys/
Disallow: /sys
Disallow: /admin/
Disallow: /admin
Disallow: /auth/
Disallow: /onboarding
Disallow: /accept-invite
Disallow: /reset-password
Disallow: /feed
Disallow: /dashboard
Disallow: /settings
Disallow: /billing
Disallow: /projects/
Disallow: /builder
Disallow: /builder-v2
Disallow: /ProjectBuilderV2
Disallow: /legacy
Disallow: /light
Disallow: /internal/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=
Crawl-delay: 1

# ── Search Engines — WELCOME (fast crawling) ──────────

User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /app
Disallow: /sys/
Disallow: /sys
Disallow: /admin/
Disallow: /admin
Disallow: /auth/
Disallow: /onboarding
Disallow: /accept-invite
Disallow: /reset-password
Disallow: /feed
Disallow: /dashboard
Disallow: /settings
Disallow: /billing
Disallow: /projects/
Disallow: /builder
Disallow: /builder-v2
Disallow: /ProjectBuilderV2
Disallow: /legacy
Disallow: /light
Disallow: /internal/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /app/
Disallow: /app
Disallow: /sys/
Disallow: /sys
Disallow: /admin/
Disallow: /admin
Disallow: /auth/
Disallow: /onboarding
Disallow: /accept-invite
Disallow: /reset-password
Disallow: /feed
Disallow: /dashboard
Disallow: /settings
Disallow: /billing
Disallow: /projects/
Disallow: /builder
Disallow: /builder-v2
Disallow: /ProjectBuilderV2
Disallow: /legacy
Disallow: /light
Disallow: /internal/
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=
Crawl-delay: 1

# ── Default (all other crawlers) ──────────────────────

User-agent: *
Allow: /
Crawl-delay: 2

# Sitemaps (static files in the site bucket)
Sitemap: https://presencis.com/sitemap-index.xml

# LLM-readable content (llmstxt.org spec)
# llms.txt = index of the explanation pages (guides, overviews, synthesis,
# comparisons, questions). The full-text article dump that sat beside it was
# retired with the verbatim article/recital copies (corpus cut, 2026-10-02).
# NOTE: `LLMs-Txt:` is NOT a standard robots.txt directive — it is ignored by all
# crawlers and tripped Lighthouse's robots.txt validator, so it is kept as a comment
# only. llms.txt -> https://presencis.com/llms.txt

# ── Private / Authenticated Areas ──────────────────────
# Synced with routes.json privateRoutes + app-only paths
Disallow: /api/
Disallow: /app/
Disallow: /app
Disallow: /sys/
Disallow: /sys
Disallow: /admin/
Disallow: /admin
Disallow: /auth/
Disallow: /onboarding
Disallow: /accept-invite
Disallow: /reset-password
Disallow: /feed
Disallow: /dashboard
Disallow: /settings
Disallow: /billing
Disallow: /projects/
Disallow: /builder
Disallow: /builder-v2
Disallow: /ProjectBuilderV2
Disallow: /legacy
Disallow: /light
Disallow: /internal/

# ── Block URL parameters (tracking, referral) ─────────
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*source=

# ── Allow important public pages explicitly ────────────
Allow: /
Allow: /login
Allow: /pricing
Allow: /how-it-works
Allow: /why-presencis
Allow: /demo
Allow: /savings
Allow: /docs
Allow: /education
Allow: /about
Allow: /blog
Allow: /tutorials
Allow: /regulations/
Allow: /research/
Allow: /tools/
Allow: /verify
Allow: /academic
Allow: /llms.txt
Allow: /security
Allow: /industry/
Allow: /capabilities/
Allow: /contact
Allow: /careers
Allow: /partners
Allow: /policies/
Allow: /terms
Allow: /privacy
Allow: /press
Allow: /your-data
Allow: /investors
Allow: /changelog
Allow: /accessibility
Allow: /cookies
Allow: /dpa