User-agent: dotbot
Disallow: /

User-agent: Scrapy
Allow: /

User-agent: *
Crawl-delay: 2


#WP Import Export Rule
User-agent: *
Disallow: /wp-content/uploads/wp-import-export-lite/