User-Agent: Googlebot
Disallow: /go.asp
Disallow: /site/
Disallow: /nl/item/
Disallow: /de/item/
Disallow: /en/item/
Disallow: /ru/item/
Disallow: /cs/item/
Disallow: /pl/item/
Disallow: /es/item/
Disallow: /fr/item/
Disallow: /nl/topnews/
Disallow: /de/topnews/
Disallow: /en/topnews/
Disallow: /ru/topnews/
Disallow: /cs/topnews/
Disallow: /pl/topnews/
Disallow: /es/topnews/
Disallow: /fr/topnews/
Disallow: /*/keywords-news/*
Disallow: /ru/news/
Disallow: /de/trends/
Disallow: /en/trends/
Disallow: /nl/trends/



User-Agent: MJ12bot
Disallow: /go.asp
Disallow: /site/
Disallow: /nl/item/
Disallow: /de/item/
Disallow: /en/item/
Disallow: /ru/item/
Disallow: /cs/item/
Disallow: /pl/item/
Disallow: /es/item/
Disallow: /fr/item/
Disallow: /de/default.asp?q=feedid:*
Disallow: /nl/default.asp?q=feedid:*
Disallow: /ru/default.asp?q=feedid:*
Disallow: /fr/default.asp?q=feedid:*
Disallow: /pl/default.asp?q=feedid:*
Disallow: /cs/default.asp?q=feedid:*
Disallow: /es/default.asp?q=feedid:*
Disallow: /en/default.asp?q=feedid:*


User-Agent: Mediapartners-Google
Disallow: /go.asp
Disallow: /site/
Disallow: /nl/item/
Disallow: /de/item/
Disallow: /en/item/
Disallow: /ru/item/
Disallow: /cs/item/
Disallow: /pl/item/
Disallow: /es/item/
Disallow: /fr/item/
Disallow: /nl/topnews/
Disallow: /de/topnews/
Disallow: /en/topnews/
Disallow: /ru/topnews/
Disallow: /cs/topnews/
Disallow: /pl/topnews/
Disallow: /es/topnews/
Disallow: /fr/topnews/

User-Agent: MagpieRSS
Disallow: /

User-Agent: MSNBot
Disallow: /go.asp
Disallow: /site/
Disallow: /nl/item/
Disallow: /de/item/
Disallow: /en/item/
Disallow: /ru/item/
Disallow: /cs/item/
Disallow: /pl/item/
Disallow: /es/item/
Disallow: /fr/item/

User-Agent: Slurp
Disallow: /go.asp
Disallow: /site/
Disallow: /nl/item/
Disallow: /de/item/
Disallow: /en/item/
Disallow: /ru/item/
Disallow: /cs/item/
Disallow: /pl/item/
Disallow: /es/item/
Disallow: /fr/item/
Crawl-delay: 2 


User-Agent: StackRambler
Disallow: /go.asp
Disallow: /site/
Disallow: /nl/item/
Disallow: /de/item/
Disallow: /en/item/
Disallow: /ru/item/
Disallow: /cs/item/
Disallow: /pl/item/
Disallow: /es/item/
Disallow: /fr/item/
Disallow: /ru/news/
Disallow: /ru/pressreleases/
Disallow: /ru/jobmarket/


User-Agent: Yandex
Disallow: /go.asp
Disallow: /site/
Disallow: /nl/item/
Disallow: /de/item/
Disallow: /en/item/
Disallow: /ru/item/
Disallow: /cs/item/
Disallow: /pl/item/
Disallow: /es/item/
Disallow: /fr/item/
Disallow: /ru/news/
Disallow: /ru/pressreleases/
Disallow: /ru/jobmarket/