# LedgerConnect - AI-Driven Accounting Automation
# Sitemap: https://ledgerconnect.in/sitemap.xml

# Allow Google and Bing to crawl the entire site
User-agent: Googlebot
Allow: /
Allow: /blog
Allow: /blog/
Allow: /solutions/
Allow: /pricing
Allow: /sitemap.xml

User-agent: Bingbot
Allow: /
Allow: /blog
Allow: /blog/
Allow: /solutions/
Allow: /pricing
Allow: /sitemap.xml

# Allow other major search engines
User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Allow social media crawlers for previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Block AI training bots (while allowing search engines)
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: MetaAI
Disallow: /

# Block aggressive SEO crawlers
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Default rule - Allow all except specified above
User-agent: *
Allow: /

# Disallow sensitive directories
Disallow: /writable/
Disallow: /vendor/
Disallow: /system/
Disallow: /app/
Disallow: /modules/
Disallow: /.git/
Disallow: /backups/
Disallow: /logs/

# Logged-in app & admin (auth required) — hint crawlers not to waste crawl on login walls / private UI.
# Meta noindex on app layout is the main signal; this reduces unnecessary fetches.
Disallow: /admin/
Disallow: /bank-to-excel/
Disallow: /ecom-to-excel/
Disallow: /parties/
Disallow: /staff/
Disallow: /tally-connector/
Disallow: /tally-connector-v2/
Disallow: /pdf-to-image/
Disallow: /dashboard
Disallow: /my-plans
Disallow: /my-profile
Disallow: /my-videos
Disallow: /convert-pdf
Disallow: /verify-account
Disallow: /upload
Disallow: /history
Disallow: /validate/
Disallow: /generate/
Disallow: /refund-approval
# Note: sitemap.xml URLs (e.g. /bank-pdf-converter, /ecommerce-converter, /solutions/tally-connector)
# are not the same as disallowed app paths (/bank-to-excel/, /ecom-to-excel/, /tally-connector/).

# Allow public assets
Allow: /assets/
Allow: /metronic/
Allow: /public/

# Sitemap: static public/sitemap.xml (upload or deploy with site). Regenerate: php scripts/generate_public_sitemap.php
# Submit the full URL of your deployed site in Search Console, e.g.:
Sitemap: https://ledgerconnect.in/sitemap.xml