# robots.txt — Clevertel SA Group
# Updated: 2026-06-11 (added blog sitemap, API and dash blocks, AI bot controls)

User-agent: *
Allow: /

# Block internal/system directories — never index these
Disallow: /_inc/
Disallow: /_seo/
Disallow: /_tools/
Disallow: /_seo_patches/
Disallow: /portal.php
Disallow: /portal/

# Block API endpoints — JSON, not pages
Disallow: /api/

# Block blog diagnostic file if it ever gets left on the server
Disallow: /blog-diag.php

# Block paginated & search-query blog pages from indexing
# (canonical blog posts are in /blog/<slug>/ — index those, not these)
Disallow: /blog/?page=
Disallow: /blog/?q=

# Allow blog category filter pages — these act as topic index pages
Allow: /blog/?category=

# Block known AI training crawlers that don't respect content licensing
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: FacebookBot
Disallow: /

# Allow Google AdsBot (needed for Smart Campaigns)
User-agent: AdsBot-Google
Allow: /

# Allow Google image indexing
User-agent: Googlebot-Image
Allow: /assets/img/
Allow: /assets/blog-img/

# ── Sitemaps ──────────────────────────────────────────────────────────────────
# Master sitemapindex — submit this URL to Google Search Console & Bing
Sitemap: https://www.clevertel.co.za/sitemap.xml

# Blog sitemap listed separately for faster discovery after new posts
Sitemap: https://www.clevertel.co.za/blog/sitemap.xml