User-agent: Scrapy
Allow: /

User-agent: Site24x7
Disallow: /

User-agent: Traackr.com
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SentiBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: squirrobot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: MojeekBot
Disallow: /

User-agent: epicbot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: BLP_bbot
Disallow: /

User-agent: Netvibes
Disallow: /

User-agent: Mediatoolkitbot
Crawl-Delay: 60

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 20