# here.now robots.txt
# We allow search and user-directed AI agents to discover public product, docs, and API metadata.
# We restrict common bulk AI training crawlers by default.

Content-Signal: search=yes
Content-Signal: ai-input=yes
Content-Signal: ai-train=no

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Disallow: /

User-agent: ByteSpider
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: *
Allow: /
Allow: /api/llms.txt
# og:image for default share cards (docs/features/site-og-share-cards.md).
# X/Twitterbot respects robots.txt for card-image fetches, so without this
# Allow, X cards render without their image. The endpoint only serves
# public sites' current-version screenshots.
Allow: /api/thumbnails/
Disallow: /api/
Disallow: /hn-admin-tool

Sitemap: https://here.now/sitemap.xml
Schemamap: https://here.now/schema-map.xml