User-agent: Scrapy
Allow: /

User-agent: Scrapy
Allow: /

User-agent: *
Disallow:  /files/*.pdf$

User-agent: *
Disallow: /
User-agent: Googlebot
Disallow:
User-agent: bingbot
Disallow: /no-bing-crawl/
Disallow: wp-admin

User-agent: *
Disallow: /wp-login.php
Disallow: /wp-admin/
Crawl-delay: 600