User-agent: *
Disallow: /cgi-bin/

User-agent: *
Disallow: /svn/

# Block the TurnitinBot because it crawls at a too high rate
User-agent: TurnitinBot
Disallow: /

# Block the Zhoof SpiderBot because it crawls at a too high rate
User-agent: Zhoof SpiderBot
Disallow: /

# Block the MJ12bot because it crawls at a too high rate
User-agent: MJ12bot
Disallow: /

# Block the Lipperhey Spider because it crawls at a too high rate
User-agent: Mozilla/5.0 (compatible; Lipperhey Spider; http://www.lipperhey.com/)
Disallow: /