mirror of
https://github.com/waynesutton/markdown-site.git
synced 2026-01-12 12:19:18 +00:00
34 lines
455 B
Plaintext
34 lines
455 B
Plaintext
# robots.txt for markdown sync framework
|
|
# https://www.robotstxt.org/
|
|
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# Sitemaps
|
|
Sitemap: https://www.markdown.fast/sitemap.xml
|
|
|
|
# AI and LLM crawlers
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: ChatGPT-User
|
|
Allow: /
|
|
|
|
User-agent: Claude-Web
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: PerplexityBot
|
|
Allow: /
|
|
|
|
User-agent: Applebot-Extended
|
|
Allow: /
|
|
|
|
# Cache directive
|
|
Crawl-delay: 1
|