# =========================
# Robots.txt for M S Mehta & Co
# =========================

User-agent: *
Allow: /
Allow: /llms.txt

# =========================
# WordPress Core Security
# =========================

Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /msm-login/
Disallow: /wp-login.php
Disallow: /wp-config.php
Disallow: /wp-config-sample.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# =========================
# Clean Up & Optimization
# =========================

Disallow: /cdn-cgi/
Disallow: /?s=
Disallow: /search/
Disallow: /?replytocom=
Disallow: /comments/feed/
Disallow: /trackback/
Disallow: /?preview=
Disallow: /*?preview=
Disallow: /author/
Disallow: /*?*utm_
Disallow: /*?gclid=
Disallow: /*?fbclid=

# =========================
# Tool Access (Critical)
# =========================

Allow: /container-tracking/
Allow: /cbm-calculator/
Allow: /customs-exchange-rate/

# =========================
# AI / LLM Crawlers
# =========================

# OpenAI (ChatGPT)
User-agent: OAI-SearchBot
Allow: /
User-agent: GPTBot
Allow: /

# Google (Gemini / Google AI)
User-agent: Google-Extended
Allow: /
User-agent: Googlebot
Allow: /

# Microsoft (Bing)
User-agent: bingbot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

# Apple (Siri / Spotlight)
User-agent: Applebot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# =========================
# Sitemaps
# =========================

Sitemap: https://msmehta.in/sitemap.xml