User-agent: *
Disallow: /cgi-bin/
Disallow: /stats/
Disallow: /blog/wp-admin/
# disallow the wp admin and includes directories as these contain WordPress software application data
Disallow: /blog/wp-includes/
Disallow: /blog/wp-content/
# categories are duplicate content, the crawlers will still find your posts
Disallow: /blog/trackback/
Disallow: /blog/page/
Disallow: /blog/search/
Disallow: /blog/comments/
# allow the medapartners google bot to determine adsense content
User-agent: Mediapartners-Google
Allow: /

# allow google to crawl all pages to measure page quality for AdWords
User-agent: Adsbot-Google
Allow: /

# allow the Google image crawler to crawl all pages
User-agent: Googlebot-Image
Allow: /

# allow the Google bot to craw all pages for its mobile index
User-agent: Googlebot-Mobile
Allow: /