# FridgeToForks — https://fridgetoforks.com
#
# EVERY AI CRAWLER IS WELCOME: search crawlers that cite us, live fetchers
# acting for a person, and training crawlers too (Scott, 2026-10-08: "I want to
# allow ALL LLMS!"). One group for all of them, so they all get the same rules:
# the public site is open; the signed-in app is not (it is noindex anyway, and
# keeping crawlers out of it keeps crawl budget on pages that exist to be found).
#
# Named here so the welcome is explicit; `*` covers everyone else.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: CCBot
User-agent: cohere-ai
User-agent: Bytespider
User-agent: *
Allow: /
# The ingredient landing pages (/recipes/find?ingredients=...) are listed in
# the sitemap and meant to be crawled and cited. `Allow: /recipes/` is longer,
# so it beats `Disallow: /recipe` (the signed-in recipe page) for them. (D308)
Allow: /recipes/
Disallow: /api/
Disallow: /app
Disallow: /kitchen
Disallow: /find
Disallow: /box
Disallow: /shopping
Disallow: /one-thing
Disallow: /recipe
Disallow: /admin
Disallow: /login
Disallow: /signup
Disallow: /forgot
Disallow: /reset
Disallow: /verify
# Link-preview images for shared recipes (D330). X's crawler obeys robots.txt
# for images, so under the /media/ disallow every share posted to X showed a
# grey placeholder (Scott, 2026-10-10). ABOVE the disallow on purpose: Google
# and X take the longest rule, simpler parsers take the first, and this order
# gives the same answer to both.
Allow: /media/og/
Disallow: /media/

# /mcp is the public Model Context Protocol endpoint (D187): POST-only JSON-RPC,
# nothing to index, but deliberately NOT disallowed so an assistant that reads
# this file can reach it. Documented at /assistants and in /llms.txt.
# The blog is at /articles and is meant to be crawled and cited.

Sitemap: https://fridgetoforks.com/sitemap.xml