# robots.txt for Mindset Psychology
# https://mindsetpsychology.co.id
# Last updated: 2025-12-07

# ============================================
# ALLOW ALL GOOD SEARCH ENGINES
# ============================================

# Default rule: Allow all search engines to crawl the entire site
User-agent: *
Allow: /
Crawl-delay: 1

# ============================================
# DISALLOW NON-ESSENTIAL FILES
# ============================================

# Block access to build artifacts and system files
Disallow: /node_modules/
Disallow: /dist/
Disallow: /.git/
Disallow: /.vscode/

# Block access to configuration files
Disallow: /*.json$
Disallow: /*.config.js$
Disallow: /*.config.ts$

# Block access to source maps (not needed for SEO)
Disallow: /*.map$

# ============================================
# MAJOR SEARCH ENGINES (NO CRAWL-DELAY)
# ============================================

# Google
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

# Bing
User-agent: Bingbot
Allow: /

User-agent: BingPreview
Allow: /

# Yahoo
User-agent: Slurp
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yandex (Russia)
User-agent: Yandex
Allow: /

# Baidu (China)
User-agent: Baiduspider
Allow: /

# ============================================
# SEO TOOLS (ALLOW - USEFUL FOR MONITORING)
# ============================================

# Ahrefs - Allow (useful for backlink monitoring)
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

# Semrush - Allow (useful for SEO audit)
User-agent: SemrushBot
Allow: /
Crawl-delay: 2

# Moz - Allow (useful for domain authority tracking)
User-agent: rogerbot
Allow: /
Crawl-delay: 2

# ============================================
# BLOCK BAD BOTS (SCRAPERS & SPAMMERS)
# ============================================

# MJ12bot (Majestic) - Aggressive crawler
User-agent: MJ12bot
Disallow: /

# DotBot - Aggressive crawler
User-agent: DotBot
Disallow: /

# SeznamBot - Czech search engine (not needed for Indonesia)
User-agent: SeznamBot
Disallow: /

# Scrapers and content thieves
User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Wget
Disallow: /

User-agent: curl
Disallow: /

# ============================================
# BOT AI — DIPISAH BERDASARKAN TUJUANNYA (diubah 12 Agt 2026)
# ============================================
# Sebelumnya semua bot AI diblokir borongan. Itu ikut menutup bot yang dipicu MANUSIA:
# saat calon klien bertanya ke ChatGPT/Claude, asistennya mengambil halaman kita memakai
# ChatGPT-User / Claude-User. Log 11 Agt 2026: ChatGPT-User membuka /biaya,
# /artikel/biaya-konseling-online-indonesia-2026, /artikel/tes-psikologi-iq-jogja-daftar-harga
# — halaman berniat beli, ~77 kali/hari (±2.300/bulan). Sebagai pembanding, seluruh
# pencarian lokal cuma 210/bln (Solo) + 30/bln (Jogja). Memblokirnya = menutup kanal
# penemuan yang sudah lebih besar dari pencarian lokal.
#
# ⛔ JANGAN memblokir borongan lagi. Bedakan tiga jenis:
#   1. Melatih model      → DIBLOKIR (di bawah). Tidak ada trafik yang hilang.
#   2. Mengindeks jawaban → DIIZINKAN. Ini yang membuat kita BISA DIKUTIP di jawaban AI.
#   3. Dipicu pengguna    → DIIZINKAN. Ada calon klien yang sedang menunggu jawaban.
#
# ⚠️ Cloudflare menyuntik "Managed robots.txt" DI EDGE, di ATAS berkas ini — isinya sudah
#    memblokir GPTBot, ClaudeBot, CCBot, Google-Extended, Amazonbot, Applebot-Extended,
#    Bytespider, meta-externalagent. Jadi robots.txt yang dilihat crawler BUKAN berkas ini
#    saja. Cek yang sebenarnya dengan `curl https://mindsetpsychology.co.id/robots.txt`,
#    dan ubah kebijakan bot AI lewat DASBOR Cloudflare — mengedit berkas ini saja tidak cukup.

# --- (1) Pelatihan model: TETAP DIBLOKIR ---
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

# --- (2)+(3) Pengindeks jawaban AI & bot yang dipicu pengguna: SENGAJA DIIZINKAN ---
# ⛔ Jangan pindahkan ke daftar blokir di atas tanpa membaca alasannya.
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /node_modules/
Disallow: /dist/
Disallow: /.git/

# Facebook crawler (optional - block if you don't want FB to cache your pages)
# User-agent: facebookexternalhit
# Disallow: /

# ============================================
# SITEMAP LOCATION
# ============================================

Sitemap: https://mindsetpsychology.co.id/sitemap.xml

# ============================================
# NOTES FOR WEBMASTER
# ============================================
# - This robots.txt allows all major search engines to crawl the site
# - SEO tools (Ahrefs, Semrush, Moz) are allowed with crawl-delay
# - Bad bots and scrapers are blocked to save server resources
# - Bot AI pelatih model diblokir; pengindeks jawaban AI & bot yang dipicu pengguna
#   (ChatGPT-User dst) SENGAJA diizinkan — lihat alasannya di bagian BOT AI di atas
# - Sitemap is declared for faster indexing
# - Update this file when adding new pages or changing structure