# robots.txt
User-agent: *
Disallow: /news.html
Disallow: /documents/gg98101.pdf
Disallow: /documents/gg98101.txt
Disallow: /documents/gg97090.pdf
Disallow: /documents/gg97090.txt

Crawl-delay: 120