# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: webcrawler
Disallow:

User-agent: lycra
Disallow: 

User-agent: *
Disallow: /tmp
Disallow: /logs
Disallow: /images
Disallow: /pdf
Disallow: /Flash
Disallow: /Mancomm
Disallow: /SearchVerity
Disallow: /360
Disallow: /cartoons
Disallow: /coologhan_files
Disallow: /DWT_Gary
Disallow: /mail
Disallow: /files