# blocked for extensive crawls without respecting crawl-delay
User-agent: Baiduspider
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: Baiduspider+
Disallow: /

User-agent: Googlebot
Disallow:

User-agent: PetalBot
Disallow: /

User-agent: Bingbot
Disallow: /
Disallow: /doc_view/pid*
Disallow: /pdf*

User-agent: *
Disallow: /doc_view/pid*
Disallow: /pdf*
Crawl-delay: 10

#added msnbot with more delay - was hitting hard with different ip's
User-agent: msnbot
Disallow: /
Disallow: /doc_view/pid*
Disallow: /pdf*
Crawl-delay: 40

Sitemap: https://citeseerx.ist.psu.edu/sitemap_index.xml