User-agent: * Crawl-delay: 1 Disallow: /webstat/ Disallow: /cgi-bin Disallow: /wp-admin Disallow: /wp-includes Disallow: /wp-content/plugins Disallow: /wp-content/cache Allow: /wp-content/themes Disallow: /trackback Disallow: */trackback Disallow: /wp-trackback Disallow: /wp-feed Disallow: /comments Disallow: /wp-comments Disallow: /xmlrpc.php Disallow: */trackback # Disallow: */feed # Disallow: /feed/ Disallow: */comments # Disallow: /category # Disallow: /category/*/* Disallow: /tag # Disallow: /*?* # Disallow: /*? Disallow: /to/ Disallow: /&* Disallow: /page/ Disallow: /goto/ Disallow: /goto/* Sitemap: site.com/sitemap.xml.gz Host: site.com