# always allow adsense
User-agent: Mediapartners-Google*
Disallow:
User-agent: grub-client
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: NaverBot
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: e-SocietyRobot
Disallow: /
User-agent: PCNBot
Disallow: /
User-agent: aipbot
Disallow: /
User-agent: RedCarpet
Disallow: /
User-agent: OmniExplorer_Bot
Disallow: /
User-agent: PGBot
Disallow: /
User-agent: psbot
Disallow: /
User-agent: *
Disallow: /cgi-bin
Disallow: /texis,