User-agent: Googlebot # Disallow all directories and files within Disallow: /cgi-bin/ Disallow: /xhtml/ Disallow: /mail/ Disallow: /js/ Disallow: /*.gif$ Allow: /*.html$