# #######################################################################
#
# File    : robots.txt
# Version : 1.0
# Date    : 13 June 2008
# Site    : standrin.com
# Author  : TN [Kinesis Systems]
#
# General Notes:
#    - This file is intended to document a simple, global crawler policy.
#      It is not currently used to implement a per-crawler policy.
#    - This file may fail valdation by some validators that do not include
#      support for the common non-standard extension directives used below.
#      These non-standard directives are supported by a number of major
#      crawlers and for this reason they have been incuded here. Their
#      use is more benificial than their omission.
#
# Revision Notes:
#
# 09/04/09: Added disallow all policy for that pesky SurveyBot tosh from
#           domaintools.com. Pretty useless really as whilst this would
#           normaly be sufficient the website further qualifies that the
#           domain be submitted to them for removal. This will cause all
#           domains on a single IP address to be removed also. Further,
#           if there are multiple administrators, with just one who wishes
#           to be included then priority will be given to this one.
#
# #######################################################################


# =======================================================================
#
# Standard directives
#
# The directives listed in this section will be parsed by all conformant
# crawlers.
#
# =======================================================================
#
# -----------------------------------------------------------------------
# Targeted bots/crawlers
# -----------------------------------------------------------------------
User-agent: *    # Match all bots requesting robots.txt (this file)
Disallow: /assets/
Disallow: /errors/
Disallow: /library/
Disallow: /cs/
Allow: /assets/images/catalogue/
Crawl-delay: 5
Sitemap: http://www.standrin.com/sitemap.xml    # The URI of the sitemap file (this file can be GZipped i.e. sitemap.xml.gz).

# =======================================================================
# For that SurveyBot tosh
# =======================================================================
User-agent: SurveyBot
Disallow: /