# small robots.txt
# more information about this file can be found at
# http://www.robotstxt.org/wc/robots.html
# to stop a polite robot indexing an exampledir
# add a line like (delte the #'s)
# user-agent: polite-bot
# Disallow: /exampledir/
# a list of know bots can be found at
# http://www.robotstxt.org/wc/active/html/index.html
# see http://www.sxw.org.uk/computing/robots/check.html
# for syntax checking
User-agent: *
Crawl-Delay: 10
Disallow: /aggregator/
Disallow: /tracker/
Disallow: /comment/reply/
Disallow: /user/
Disallow: /en/user/
Disallow: /de/user/
Disallow: /fr/user/
Disallow: /taxonomy/
Disallow: /en/taxonomy/
Disallow: /de/taxonomy/
Disallow: /fr/taxonomy/
# Disallow: /files/  perche' fermarlo?
Disallow: /search/
# Nodes with aliases are accessible from 2 URLs:search engine bots
# will discover both over a period of time and this might lead to
# penalisation for duplicate content
Disallow: /node/
Disallow: /print/
Disallow: /send_email/
Disallow: /organization/governance/board/
Disallow: /en/organization/governance/board/
Disallow: /de/organization/governance/board/
Disallow: /fr/organization/governance/board/

# User-Agent: msnbot
# Disallow: /