# This allows the site to be spidered, but not certain directories
User-agent: *
Disallow: /cgi-bin/