User-agent: Scrapy
Allow: /

# BEGIN W3TC ROBOTS
User-agent: *
Disallow: /wp-content/cache/
# END W3TC ROBOTS