################################################################################################################################
# PLEASE NOTE: Any pages that you list here must be secured by other means if you don't want people to be able to view them as #
# some malicious users will look at a robots.txt file to try to find "hidden" or "secret" areas for confidential information.  #
################################################################################################################################
# Crawlers that are kind enough to obey, but which we'd rather not have unless they're feeding search engines.

User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot) Microsoft
Disallow: /

#
# Friendly, low-speed bots are most welcome to view article pages, but not dynamically-generated pages please.
# If your bot supports 'meta tags' and obey robots.txt, please let us know.
# Lastly we need to protect some folders from all user agents:
#
#If you donīt want your images folder to be indexed add

User-agent: *
Disallow: /images/

#this goes for all webcrawlers if they obey. Means a delay of 5 seconds between crawls
Crawl-delay: 5