User-agent: Googlebot
Allow: /

User-agent: Feedfetcher-Google
Disallow: /

User-Agent: Web Link Validator
Disallow:/

User-Agent: W3C-checklink
Disallow:/

User-Agent: TYPO3-linkvalidator
Disallow:/

User-Agent: *linkvalidator
Disallow:/

User-Agent: LinkChecker
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: MS Search 4.0 Robot 
Disallow: / 

User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot) Microsoft 
Disallow: / 

User-agent: Lycos/x.x
Disallow: / 


User-agent:check_http/v1.4.15+(nagios-plugins+1.4.15)
Disallow: /