# Made by Max to limit the amount of traffic produced by crawlers.

# At the time this file is written, the CPDL wiki contains about 30 thousands pages. Given the number of pages
# added every week (a small percentage of the total), it appears logical that each crawler does not need to visit
# the whole website more frequently than once a week. As there are about 600 thousand seconds in a week,
# it seems convenient to set a maximum visiting rate of 1 page every 600,000/30,000 = 20 seconds.


User-agent: *        # valid for any crawler
Crawl-delay: 20      # wait 20 seconds between successive requests

# This is added to avoid that the back-up BB is indexed by crawlers

User-agent: *        # valid for any crawler
Disallow: /forums1/  # do not index any pages in the forums1 subdirectory