User-agent: *
Disallow: /

User-agent: htdig
Disallow: /cgi-bin/
Disallow: /cat/

User-agent: Mini-Crawl 
Disallow: 

User-agent: gsa-crawler
Disallow: /cgi-bin/
Disallow: /cat/