User-Agent: Wget
Allow: /

User-Agent: *
Allow: /
Allow: /show
Allow: /stats
Disallow: /display
Disallow: /shuttle_log.jsp
Disallow: /shuttle*
Disallow: /production/*
Disallow: /subscribe/*
Disallow: /admin/*
Disallow: /admin.jsp
Disallow: /lpm/*
Disallow: /PWG/*