User-agent: Scrapy
Allow: /

user-agent: *
Crawl-delay: 10
Sitemap: /sitemap_index.xml
Disallow: /*?s=
Disallow: /*?frm
Disallow: wp-content/plugins
Disallow: wp-content/themes
Disallow: wp-content/languages
Disallow: wp-content/cache
Disallow: wp-content/maintenance
Disallow: wp-content/upgrade
Disallow: wp-content/uploads/backwpup-*
Disallow: wp-content/uploads/complianz
Disallow: wp-content/uploads/formidable
Disallow: wp-content/uploads/ithemes-security
Disallow: wp-content/uploads/revslider
Disallow: wp-content/uploads/sass
Disallow: wp-content/uploads/w*
Disallow: wp-includes
Disallow: wp-admin
Disallow: wp-logs