User-agent: *
Allow: /
Allow: /blog/
Allow: /compare/
Allow: /guide/
Allow: /learn/
Allow: /sectors/
Allow: /company-profiles/
Allow: /news
Allow: /satellites
Allow: /live
Allow: /marketplace/
# Feeds are for readers and crawlers alike (2026-09-26): the rest of /api stays closed.
Allow: /api/feed/
# The free open data feeds (CC BY 4.0, 2026-09-26), listed in /llms.txt.
Allow: /api/open/
Allow: /api/open$
# SpaceNexus Daily media (2026-09-28): the podcast episodes (audio.m4a) and the
# video, poster and captions of PUBLISHED editions only (drafts 404).
Allow: /api/briefing/
# Share-card images (2026-10-12): og:image and twitter:image for most pages
# point at /api/og, and X and LinkedIn only show images robots.txt allows.
Allow: /api/og
Disallow: /api/
Disallow: /admin/
Disallow: /checkout/
Disallow: /account/
Disallow: /dashboard/
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /provider-dashboard/
Disallow: /advertise/dashboard
Disallow: /my-watchlists
Disallow: /desk
Disallow: /inbox
Disallow: /messages/
Disallow: /notifications/
Disallow: /reading-list
Disallow: /community/profile
Disallow: /space-stocks/filings?*type=
Crawl-delay: 1

# AI search and answer agents (2026-09-26, owner-approved "be the answer
# inside AI assistants"). Named so a future blanket rule can never catch them
# by accident: these fetch pages to cite them in ChatGPT, Claude, Perplexity,
# Siri, DuckDuckGo and Le Chat answers. A crawler that matches a named group
# ignores the * group, so the same public/private rules are repeated here
# (src/lib/__tests__/robots-ai.test.ts keeps them identical). No Crawl-delay:
# answer-time fetches should be quick.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Applebot
User-agent: DuckAssistBot
User-agent: MistralAI-User
Allow: /
Allow: /api/feed/
Allow: /api/open/
Allow: /api/open$
Allow: /api/briefing/
Allow: /api/og
Disallow: /api/
Disallow: /admin/
Disallow: /checkout/
Disallow: /account/
Disallow: /dashboard/
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /provider-dashboard/
Disallow: /advertise/dashboard
Disallow: /my-watchlists
Disallow: /desk
Disallow: /inbox
Disallow: /messages/
Disallow: /notifications/
Disallow: /reading-list
Disallow: /community/profile
Disallow: /space-stocks/filings?*type=

# AI model-training crawlers and training opt-out tokens. ALLOWED by default
# (2026-09-26): they are how base models learn SpaceNexus exists, and the
# content is free to read anyway. The owner can opt out of training by
# replacing the Allow/Disallow lines of this group with "Disallow: /" without
# touching search or answer visibility above.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
Allow: /
Allow: /api/feed/
Allow: /api/open/
Allow: /api/open$
Allow: /api/briefing/
Allow: /api/og
Disallow: /api/
Disallow: /admin/
Disallow: /checkout/
Disallow: /account/
Disallow: /dashboard/
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /provider-dashboard/
Disallow: /advertise/dashboard
Disallow: /my-watchlists
Disallow: /desk
Disallow: /inbox
Disallow: /messages/
Disallow: /notifications/
Disallow: /reading-list
Disallow: /community/profile
Disallow: /space-stocks/filings?*type=
Crawl-delay: 1

Sitemap: https://spacenexus.us/sitemap/0.xml
Sitemap: https://spacenexus.us/sitemap/1.xml
Sitemap: https://spacenexus.us/sitemap/2.xml
Sitemap: https://spacenexus.us/sitemap/3.xml
Sitemap: https://spacenexus.us/jobs-sitemap.xml
Sitemap: https://spacenexus.us/news-sitemap.xml