# Digital Karma Web - Robots.txt
# Updated: April 21, 2026

User-agent: *
Allow: /
Allow: /data/federation-map.json
Allow: /ai/*.json
Allow: /llm.txt
Allow: /llm.json
Allow: /datasets/
Allow: /entities/
Allow: /federation/
Allow: /specs/

# Block sensitive directories and files
Disallow: /deploy/
Disallow: /.git/
Disallow: /.github/

# Block backup and system files
Disallow: *.log
Disallow: *.sh

# Crawl rate
Crawl-delay: 1

Sitemap: https://www.digitalkarmaweb.com/sitemap.xml

# LLM discovery
# llm.txt: https://www.digitalkarmaweb.com/llm.txt
# Optional AI sitemap companion: https://www.digitalkarmaweb.com/ai/sitemap.json