# robots.txt – https://agentic.market
# Spec: RFC 9309 (https://www.rfc-editor.org/rfc/rfc9309)
# AI Crawl Control: https://developers.cloudflare.com/ai-crawl-control/
# Content Signals Policy: https://blog.cloudflare.com/content-signals-policy (CC0)

# --- Content Signals Policy ---
# As a condition of accessing this website, you agree to abide by the following content signals:
# (a) If a content-signal = yes, you may collect content for the corresponding use.
# (b) If a content-signal = no, you may not collect content for the corresponding use.
# (c) If the website operator does not include a content signal for a corresponding use,
#     the website operator neither grants nor restricts permission via content signal
#     with respect to the corresponding use.
# The content signals and their meanings are:
# search: building a search index and providing search results (e.g., returning hyperlinks
#         and short excerpts from your website's contents). Search does not include providing
#         AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval augmented generation,
#           grounding, or other real-time taking of content for generative AI search answers).
# ai-train: training or fine-tuning AI models.
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF RIGHTS UNDER
# ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS IN THE
# DIGITAL SINGLE MARKET.

# Default: allow all well-behaved crawlers
User-agent: *
Content-Signal: search=yes, ai-input=no, ai-train=no
Allow: /
Disallow: /api/
Disallow: /_next/

# --- AI Crawler Rules ---

# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /_next/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /_next/

# Anthropic
User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /_next/

User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /_next/

# Google AI
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /_next/

# Amazon
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /_next/

# Apple
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /_next/

# Common Crawl
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /_next/

# ByteDance
User-agent: Bytespider
Allow: /
Disallow: /api/
Disallow: /_next/

Sitemap: https://agentic.market/sitemap.xml