# https://www.robotstxt.org/robotstxt.html
#
# ─── Content Signals (Sec2010) ───────────────────────────────────────────────
#
# `Content-Signal` declares what we permit content to be USED for, which is a
# separate question from what a crawler is allowed to FETCH. Syntax and
# vocabulary come from the IETF AI Preferences (aipref) working group draft;
# see https://contentsignals.org/.
#
#   ai-train  — using the content to train a model
#   search    — building a classic search index that links back
#   ai-input  — retrieving the content to ground a generated answer (RAG)
#
# ⚠️ All three are deliberately YES, and the default example everyone copies
# (`ai-train=no, search=yes, ai-input=no`) would be actively harmful HERE.
# MesColis is not a publisher protecting an archive; it is a small brand whose
# measured problem is the opposite one — 0 mentions across 15 AI answers to
# buyer-intent questions, while ShipStation, Shippo and Stallion get named.
# `ai-input=no` would tell the assistants to stop using our pages to ground
# exactly the answers we are trying to appear in, and `ai-train=no` would ask
# the models not to learn the brand exists. Cloudflare's own contentsignals.org
# publishes yes/yes/yes for the same reason.
#
# Revisit if that calculus changes (e.g. if licensing content becomes a revenue
# line). Until then, being cited is worth more than being withheld.
User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /auth
Disallow: /api/
# Sec1052 — per-shipment, per-invite, internal-flow pages should not be indexed
Disallow: /track/
Disallow: /onboarding
# Sec1725 — /pos MUST stay anchored. robots.txt matches by PREFIX, not by path
# segment, so a bare "Disallow: /pos" also blocked /post-office-locator (Sec1242)
# on both hosts — the only two CRITICAL rows in the GSC index panel, while
# sitemap.ts kept submitting that URL at priority 0.7. Never un-anchor these.
Disallow: /pos$
Disallow: /pos/
Disallow: /accept-invite

# AI crawlers — explicitly welcome, no restrictions on marketing pages
User-agent: GPTBot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /auth
Disallow: /api/
# Sec1052 — per-shipment, per-invite, internal-flow pages should not be indexed
Disallow: /track/
Disallow: /onboarding
# Sec1725 — /pos MUST stay anchored. robots.txt matches by PREFIX, not by path
# segment, so a bare "Disallow: /pos" also blocked /post-office-locator (Sec1242)
# on both hosts — the only two CRITICAL rows in the GSC index panel, while
# sitemap.ts kept submitting that URL at priority 0.7. Never un-anchor these.
Disallow: /pos$
Disallow: /pos/
Disallow: /accept-invite

User-agent: ClaudeBot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /auth
Disallow: /api/
# Sec1052 — per-shipment, per-invite, internal-flow pages should not be indexed
Disallow: /track/
Disallow: /onboarding
# Sec1725 — /pos MUST stay anchored. robots.txt matches by PREFIX, not by path
# segment, so a bare "Disallow: /pos" also blocked /post-office-locator (Sec1242)
# on both hosts — the only two CRITICAL rows in the GSC index panel, while
# sitemap.ts kept submitting that URL at priority 0.7. Never un-anchor these.
Disallow: /pos$
Disallow: /pos/
Disallow: /accept-invite

User-agent: PerplexityBot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /auth
Disallow: /api/
# Sec1052 — per-shipment, per-invite, internal-flow pages should not be indexed
Disallow: /track/
Disallow: /onboarding
# Sec1725 — /pos MUST stay anchored. robots.txt matches by PREFIX, not by path
# segment, so a bare "Disallow: /pos" also blocked /post-office-locator (Sec1242)
# on both hosts — the only two CRITICAL rows in the GSC index panel, while
# sitemap.ts kept submitting that URL at priority 0.7. Never un-anchor these.
Disallow: /pos$
Disallow: /pos/
Disallow: /accept-invite

User-agent: Google-Extended
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /auth
Disallow: /api/
# Sec1052 — per-shipment, per-invite, internal-flow pages should not be indexed
Disallow: /track/
Disallow: /onboarding
# Sec1725 — /pos MUST stay anchored. robots.txt matches by PREFIX, not by path
# segment, so a bare "Disallow: /pos" also blocked /post-office-locator (Sec1242)
# on both hosts — the only two CRITICAL rows in the GSC index panel, while
# sitemap.ts kept submitting that URL at priority 0.7. Never un-anchor these.
Disallow: /pos$
Disallow: /pos/
Disallow: /accept-invite

# Sec2009 — the LIVE-FETCH AI agents, named explicitly.
#
# The four groups above cover the INDEXING crawlers (GPTBot, ClaudeBot,
# PerplexityBot, Google-Extended). Each vendor also runs a separate agent that
# fetches a page in real time, while a human is mid-conversation asking
# something like "best shipping software in Canada" — and those were not named
# anywhere in this file. They were still allowed, via `User-agent: *`, but an
# omission next to four explicit siblings reads like a decision rather than an
# oversight. `crawler-agents.ts` already treats them as the most valuable
# traffic we get; this file now says the same thing.
#
# robots.txt allows stacking User-agent lines before one rule block, so this
# grants all six the identical rules the groups above use. Keep the Disallow
# list in sync with those groups — a group that matches an agent REPLACES the
# `*` group for it, so anything missing here is silently allowed.
#
# ⚠️ None of this reaches these agents if an edge rule answers first, and for a
# while one did: Sec2009 recorded Cloudflare returning 403 to every one of them
# on both brands, so they never saw this file.
#
# ✅ RE-MEASURED 27 Aug 2026 (Sec2032) — that block is NOT firing any more. All
# six agents plus GPTBot, ClaudeBot, PerplexityBot and Googlebot fetch a live
# blog post and get **200** on both hosts.
#
# ⚠️ Caveat, so nobody over-reads this: the probe spoofed the User-Agent from an
# ordinary IP. Cloudflare's AI-crawler rules can match VERIFIED bots (UA + owned
# IP range), and such a rule would still 403 the real crawler while letting this
# probe through. So the UA-level block is gone; a verified-bot rule is not ruled
# out. Confirm in the dashboard (Security → Bots → AI Scrapers and Crawlers)
# before treating AI access as fully proven — and note this is a Cloudflare
# change, never a repo change.
#
# 📌 This note replaced a flat "Cloudflare returns 403" claim that had already
# stopped being true. Same rot as the Supabase redirect-URL item that sat marked
# as a launch blocker for weeks after it was fixed: a stale ⚠️ in a file every
# session reads is worse than no note, because it sends people to fix what works.
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Perplexity-User
User-agent: Applebot-Extended
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Disallow: /dashboard
Disallow: /admin
Disallow: /auth
Disallow: /api/
# Sec1052 — per-shipment, per-invite, internal-flow pages should not be indexed
Disallow: /track/
Disallow: /onboarding
# Sec1725 — /pos MUST stay anchored. robots.txt matches by PREFIX, not by path
# segment, so a bare "Disallow: /pos" also blocked /post-office-locator (Sec1242)
# on both hosts — the only two CRITICAL rows in the GSC index panel, while
# sitemap.ts kept submitting that URL at priority 0.7. Never un-anchor these.
Disallow: /pos$
Disallow: /pos/
Disallow: /accept-invite

# Sitemap + LLM-friendly product description
# Sec1051 — both sitemap URLs are www (canonical host after Sec1032 apex→www
# 308). Submitting the apex URL caused Google to follow the redirect on every
# fetch and report "Duplicate, Google chose different canonical than user"
# in the apex GSC property. The www variants align with brand.baseUrl.
Sitemap: https://www.mescolis.ca/sitemap.xml
Sitemap: https://www.shipsmarter.ca/sitemap.xml