# see: http://en.wikipedia.org/wiki/Robots.txt

# no idea if and how this part below works, or conflicts with the rest
# see: http://forum.joomla.org/viewtopic.php?t=147947
# first: create the list with bad bots

SetEnvIfNoCase User-Agent "^EmailSiphon" bad_bot
SetEnvIfNoCase User-Agent "^EmailWolf" bad_bot
SetEnvIfNoCase User-Agent "^ExtractorPro" bad_bot
SetEnvIfNoCase User-Agent "^CherryPicker" bad_bot
SetEnvIfNoCase User-Agent "^NICErsPRO" bad_bot
SetEnvIfNoCase User-Agent "^Teleport" bad_bot
SetEnvIfNoCase User-Agent "^EmailCollector" bad_bot
SetEnvIfNoCase User-Agent "^LinkWalker" bad_bot
SetEnvIfNoCase User-Agent "^Zeus" bad_bot

SetEnvIfNoCase User-Agent "^Slurp" bad_bot
SetEnvIfNoCase User-Agent "^MSNbot" bad_bot
SetEnvIfNoCase User-Agent "^Scooter" bad_bot
SetEnvIfNoCase User-Agent "^MSIECcrawler" bad_bot
SetEnvIfNoCase User-Agent "^ia_archiver" bad_bot
SetEnvIfNoCase User-Agent "^WebZIP" bad_bot
SetEnvIfNoCase User-Agent "^WGet" bad_bot
SetEnvIfNoCase User-Agent "^WebCopier" bad_bot
SetEnvIfNoCase User-Agent "^WebStripper" bad_bot
SetEnvIfNoCase User-Agent "^TelePort" bad_bot
SetEnvIfNoCase User-Agent "^TelePortPro" bad_bot
SetEnvIfNoCase User-Agent "^SitesNagger" bad_bot
SetEnvIfNoCase User-Agent "^EmailCollector/1.0" bad_bot
SetEnvIfNoCase User-Agent "^Email" bad_bot
SetEnvIfNoCase User-Agent "^Email_Spider 2.0" bad_bot
SetEnvIfNoCase User-Agent "^WebEmailExtrac" bad_bot
SetEnvIfNoCase User-Agent "^WebEmailExtractor" bad_bot
SetEnvIfNoCase User-Agent "^Email_Extractor" bad_bot
SetEnvIfNoCase User-Agent "^EmailWolf1.00" bad_bot
SetEnvIfNoCase User-Agent "^advanced_email_extractor" bad_bot
SetEnvIfNoCase User-Agent "^webemailextrac.*" bad_bot
SetEnvIfNoCase User-Agent "^cherrypickerse/1.0" bad_bot
SetEnvIfNoCase User-Agent "^cherrypickerelite/1.0" bad_bot
SetEnvIfNoCase User-Agent "^httrack" bad_bot
SetEnvIfNoCase User-Agent "^microsoft.url.control" bad_bot
SetEnvIfNoCase User-Agent "^webstripper" bad_bot

Order Allow,Deny
Allow from all				# allows all bots acces to all directories and files in them
Deny from env=bad_bot	# except from the listed bad bots
# no idea if and how this part above works, or conflicts with the rest below


User-agent: *				# match all bots
Disallow: /test/			# keep all bots out of this directory (overrules the allow-all above)
Disallow: /docs/			# keep all bots out of this directory (overrules the allow-all above)
Disallow: /work/basworkdownloads.html	# keep all bots out of this file (overrules the allow-all above)