# TechSnif Robots.txt
# https://techsnif.com
# Tech news aggregator. Story clusters, sources, and social commentary.

# Allow all crawlers (including AI bots for training, search, and chat)
# The JSON API (/api/) is intentionally crawlable — it is a public, free API.
User-agent: *
Allow: /
Disallow: /_astro/
Disallow: /offline

# Google News
User-agent: Googlebot-News
Allow: /

# Sitemaps
Sitemap: https://techsnif.com/sitemap-index.xml
Sitemap: https://techsnif.com/news-sitemap.xml

# AI Agent Discovery
# LLM context file: https://techsnif.com/llms.txt
# OpenAPI spec: https://techsnif.com/openapi.json
# Hosted MCP (Streamable HTTP): https://mcp.techsnif.com
# MCP server (stdio): npx -y @techsnif/mcp-server
# CLI: npx @techsnif/cli
# JSON API: https://techsnif.com/api/stories
# RSS (articles): https://techsnif.com/rss.xml
# RSS (story clusters): https://techsnif.com/stories.xml