User-agent: Googlebot
Allow: /

User-agent: *
Disallow: /

User-Agent: Web Link Validator
Disallow:/

User-Agent: W3C-checklink
Disallow:/

User-Agent: TYPO3-linkvalidator
Disallow:/


User-Agent: *linkvalidator
Disallow:/

User-Agent: LinkChecker
Disallow: /

User-agent: Yandex
Disallow: /


User-agent: MS Search 4.0 Robot 
Disallow: / 

User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot) Microsoft 
Disallow: / 

User-agent: Lycos/x.x
Disallow: / 


User-agent:check_http/v1.4.15+(nagios-plugins+1.4.15)
Disallow: / 

User-agent: *
Disallow: /wpad.dat
Disallow: /admin/
Disallow: /App_Browsers/
Disallow: /App_Code/
Disallow: /App_Data/
Disallow: /App_GlobalResources/
Disallow: /bin/
Disallow: /Components/
Disallow: /Config/
Disallow: /contest/
Disallow: /controls/
Disallow: /DesktopModules/
Disallow: /Documentation/
Disallow: /HttpModules/
Disallow: /images/
Disallow: /Install/
Disallow: /js/
Disallow: /Portals/
Disallow: /Providers/
# Preserve "Resources" for site friendly urls
Disallow: /Resources/ContentRotator/
Disallow: /Resources/ControlPanel/
Disallow: /Resources/Dashboard/
Disallow: /Resources/FeedBrowser/
Disallow: /Resources/OpenForceAd/
Disallow: /Resources/Search/
Disallow: /Resources/Shared/
Disallow: /Resources/SkinWidgets/
Disallow: /Resources/TabStrip/
Disallow: /Resources/Widgets/
Disallow: /User-Profile/	# Do not index user profiles
Disallow: /Contest/	# Do not index the old contest site
Disallow: /guidedtour/	# Do not index the old guidedtour