# Yuki — robots.txt
# Public pages are open to every crawler, AI answer engines included. The
# signed-in app, payment links, auth callbacks and utility screens stay out.
#
# ONE group for everyone: the wildcard and every named crawler below share
# the same rules. (A crawler that matches a named group ignores the `*`
# group entirely — so separate "Allow: /" groups for AI bots used to let them
# crawl /app, /pay and auth callbacks that `*` was meant to keep out.)
#
# Rules are EXACT paths or explicit subtrees, never bare prefixes: the old
# `Disallow: /trip` also blocked any future /trip-planner… page, and
# `/account` would have blocked /accounting…. `$` anchors the end of the
# URL (Google, Bing and most majors support it); `/x?` covers /x with a
# query string; `/x/` covers the subtree.

# ── AI search & answer engines: ALLOWED ──
# Yuki WANTS to be found and cited by AI assistants — ChatGPT, Claude, Gemini,
# Perplexity, Copilot, Apple Intelligence, Grok and others. These user-agents
# power AI search results, live citations and answer engines, so we let them in
# — to every public page, under the same exclusions as everyone else.
#
# Notes on some of the names below:
# Naver's crawler is called Yeti — Naver is ~55-60% of search in Korea and an
# IndexNow participant, so pings already reach it.
# Yahoo's own crawler is Slurp. Note Yahoo! Japan runs on Google's index rather than
# this one, so Japanese coverage comes from ranking in Google.
# Mojeek — a small independent UK index (genuinely its own crawl, not a
# Google/Bing reseller).
# Baidu — ~60% of search in China. A foreign-hosted site without an ICP
# licence ranks poorly there, but being crawlable costs nothing and the
# alternative is being invisible by omission rather than by ranking.
# The other Chinese engines, for the same reason.
# Coc Coc (Vietnam) and Seznam (Czech) both run their own index.
# Brave Search runs its own index and several AI assistants query it. Brave
# does not publish a stable crawler token the way the others do, so there is
# no specific entry to write — the wildcard at the top already admits it.

# NOTE: `User-agent: *` in the group below already admits every crawler,
# including ones not listed. They are named anyway for two reasons:
# a few agents are governed by their own directive rather than the wildcard
# (Google-Extended and Applebot-Extended control AI training separately from
# search), and naming them means a future tightening of the wildcard cannot
# silently lock answer engines out.
#
# There is no allow-list to join and no form to submit to for any AI
# assistant. They crawl what is reachable, or they read a conventional index.
# Being fast, linked and in Bing/Google is the whole lever.

User-agent: *
User-agent: Google-Extended
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Bingbot
User-agent: Google-CloudVertexBot
User-agent: Meta-ExternalAgent
User-agent: cohere-ai
User-agent: CCBot
User-agent: YouBot
User-agent: DuckAssistBot
User-agent: ImagesiftBot
User-agent: Diffbot
User-agent: MistralAI-User
User-agent: Bytespider
User-agent: Yandex
User-agent: SeznamBot
User-agent: Qwantify
User-agent: Yeti
User-agent: Slurp
User-agent: MojeekBot
User-agent: Baiduspider
User-agent: Baiduspider-render
User-agent: Sogou web spider
User-agent: 360Spider
User-agent: YisouSpider
User-agent: coccocbot-web
Allow: /
# Signed-in web app (client-rendered, per-user)
Disallow: /app$
Disallow: /app?
Disallow: /app/
# Retired pre-app pages: they 301 into /app (public/_redirects), which is
# never indexed, so there is nothing for a crawler to follow them to.
Disallow: /account$
Disallow: /account?
Disallow: /account/
Disallow: /goals$
Disallow: /goals?
Disallow: /goals/
Disallow: /retirement$
Disallow: /retirement?
Disallow: /retirement/
Disallow: /admin/
Disallow: /internal/
# Auth callbacks and one-time links (tokens live in these URLs)
Disallow: /auth/
Disallow: /reset-password
Disallow: /pay/
# Booking-link pages (per-person, noindex) and their manage links
Disallow: /book/
# Public polls (per-poll, noindex; the edit link's token is in the fragment)
Disallow: /poll/
Disallow: /pro?
Disallow: /group/join
# Return page of the phone apps' Stripe checkout (noindex)
Disallow: /checkout/
Disallow: /invite$
Disallow: /invite?
Disallow: /delegated$
Disallow: /delegated?
# Trip share links (proxied to the backend) — not /trip-planner… pages
Disallow: /trip$
Disallow: /trip?
Disallow: /trip/
# Old redirects
Disallow: /preview$
Disallow: /day-organizer$
Disallow: /api/
# Cloudflare internal endpoints (incl. the email-protection decode link).
# Bots get a 404 on /cdn-cgi/l/email-protection by design — keep them out so
# audits don't flag it as a "broken link". Real users' JS decodes it fine.
Disallow: /cdn-cgi/

# ── Social link-preview crawlers: ALLOWED EVERYWHERE (rich previews on share) ──
# Their own group on purpose: they only fetch URLs a person pasted into a
# chat, and trip / invite share links (/trip?…, /invite?…) must keep
# unfurling in WhatsApp, Slack and friends even though search crawlers are
# kept out of them above.
User-agent: facebookexternalhit
User-agent: FacebookBot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: WhatsApp
Allow: /

Sitemap: https://yukihq.com/sitemap.xml