# robots.txt for https://lma-consultinggroup.com/
# Policy: allow normal search indexing and AI search/retrieval.
# Block AI training/model-development crawlers where separate controls exist.
# Note: robots.txt is a request to crawlers, not a security control.

# Sitemaps
Sitemap: https://lma-consultinggroup.com/sitemap_index.xml
Sitemap: https://www.lma-consultinggroup.com/sitemap_index.xml
Sitemap: https://www.lma-consultinggroup.com/sitemap.xml
Sitemap: https://www.lma-consultinggroup.com/post-sitemap.xml
Sitemap: https://www.lma-consultinggroup.com/page-sitemap.xml

# llms.txt locations
# https://lma-consultinggroup.com/llms.txt
# https://lma-consultinggroup.com/.well-known/llms.txt

# --- Allow AI search and user-requested retrieval ---

User-agent: OAI-SearchBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: Perplexity-User
Disallow:

User-agent: Claude-SearchBot
Disallow:

User-agent: Claude-User
Disallow:

# Apple search crawler. This supports Apple search experiences such as Siri, Safari, and Spotlight.
User-agent: Applebot
Disallow:

# Optional site audit crawler
User-agent: SiteAuditBot
Disallow:

# --- Block AI training/model-development crawlers ---

# OpenAI training crawler
User-agent: GPTBot
Disallow: /

# Google control for Gemini/Vertex AI training and grounding usage
User-agent: Google-Extended
Disallow: /

# Anthropic training/model-development crawler
User-agent: ClaudeBot
Disallow: /

# Legacy Anthropic-style crawler names seen in older logs/blocklists
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Apple model-training control
User-agent: Applebot-Extended
Disallow: /

# Meta AI crawler
User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# Common Crawl, often used as a source dataset for AI training
User-agent: CCBot
Disallow: /

# ByteDance/TikTok crawler commonly included in AI crawler blocklists
User-agent: Bytespider
Disallow: /

# --- Default rule ---
# Allow standard search engines and other crawlers unless blocked above.
# Removed crawl-delay so search engines are not unnecessarily slowed after the redevelopment.

User-agent: *
Disallow: