# robots.txt for Chelmark Staffing Ltd
# https://www.chelmark.co.uk
# Last updated: 2026-04-26

# -------------------------------------------------------
# Standard Search Engine Crawlers — Full Access
# -------------------------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: facebot
Allow: /

User-agent: ia_archiver
Allow: /

# -------------------------------------------------------
# AI Search & Retrieval Crawlers — Explicitly Allowed
# These bots power AI-driven search and answer engines
# (ChatGPT, Perplexity, Claude, Gemini, Copilot, etc.)
# -------------------------------------------------------

# OpenAI — ChatGPT browsing, GPT training, OAI Search
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude AI
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Google AI — Gemini, Vertex AI, Google AI Overviews
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

# Apple — Siri, Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Meta AI — Llama, Meta AI assistant
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Meta-WebIndexer
Allow: /

# Amazon — Alexa, Bedrock AI
User-agent: Amazonbot
Allow: /

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /

# You.com AI Search
User-agent: YouBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Diffbot (structured data extraction for AI)
User-agent: Diffbot
Allow: /

# Common Crawl (open dataset used by many AI models)
User-agent: CCBot
Allow: /

# Webz.io (AI data provider)
User-agent: Webz.io
Allow: /

# ICC-Crawler (AI research)
User-agent: ICC-Crawler
Allow: /

# -------------------------------------------------------
# All Other Crawlers — Default Allow
# -------------------------------------------------------

User-agent: *
Allow: /
Disallow: /api/
Disallow: /api/trpc/
Disallow: /_core/

# -------------------------------------------------------
# Sitemaps
# -------------------------------------------------------

Sitemap: https://www.chelmark.co.uk/sitemap.xml

# -------------------------------------------------------
# LLMs.txt — AI/LLM Optimised Content Index
# Provides structured site content for AI consumption
# See https://llmstxt.org for the specification
# -------------------------------------------------------

# LLMs.txt: https://www.chelmark.co.uk/llms.txt
# LLMs Full: https://www.chelmark.co.uk/llms-full.txt