# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-content/uploads/*
Disallow: /tag/*
Disallow: /feed/*
Disallow: /privacy-policy/
Disallow: /terms-and-conditions/
Disallow: /?s=*
Disallow: /?wc-ajax=*
Disallow: /?preview=true
Disallow: /?v=*
Disallow: /wp-json/
Disallow: /cdn-cgi/

# Exception: Allow blog pages
Allow: /blogs/

Sitemap: https://www.smileformilesdental.com/sitemap_index.xml

# Google Adsbot
User-agent: adsbot-google
Disallow: /?preview=true

# Block Nutch
User-agent: Nutch
Disallow: /

# MJ12bot (Majestic)
User-agent: MJ12bot
Crawl-delay: 10
Disallow: /

# AhrefsBot (Backlink tool)
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /

# AhrefsSiteAudit (Site audit tool from Ahrefs)
User-agent: AhrefsSiteAudit
Crawl-delay: 10
Disallow: /

# SemrushBot (SEMrush SEO crawler)
User-agent: SemrushBot
Crawl-delay: 10
Disallow: /

# DotBot (used by Moz/Open Site Explorer)
User-agent: DotBot
Crawl-delay: 10
Disallow: /

# Screaming Frog SEO Spider
User-agent: Screaming Frog SEO Spider
Crawl-delay: 5
Disallow: /

# BLEXBot (SEO tool, aggressive)
User-agent: BLEXBot
Crawl-delay: 10
Disallow: /

# Googlebot (Google Search Crawler)
User-agent: Googlebot
Disallow:

# Bingbot (Microsoft Search)
User-agent: Bingbot
Crawl-delay: 5
Disallow:

# DuckDuckBot (DuckDuckGo)
User-agent: DuckDuckBot
Crawl-delay: 5
Disallow:

# YandexBot (Russian Search)
User-agent: YandexBot
Crawl-delay: 10
Disallow: /

# Baiduspider (Chinese Search)
User-agent: Baiduspider
Crawl-delay: 15
Disallow: /

# Facebook Crawler (for link previews)
User-agent: facebookexternalhit
Crawl-delay: 1
Disallow:

# Twitterbot (for X/Twitter link previews)
User-agent: Twitterbot
Crawl-delay: 1
Disallow:

# Pinterestbot (for pinning content)
User-agent: Pinterest
Crawl-delay: 1
Disallow:

# Applebot (used by Siri & Spotlight)
User-agent: Applebot
Crawl-delay: 5
Disallow:

# LinkedInBot (for link previews)
User-agent: LinkedInBot
Crawl-delay: 1
Disallow:

# ---------------------------
# END YOAST BLOCK