# All robots will spider the domain User-agent: * Disallow: # All robots will spider this folder: /to User-agent: * allow: /to # Disallow directory /images User-agent: * Disallow: /images # Disallow directory /css User-agent: * Disallow: /css # Disallow directory /tree User-agent: * Disallow: /tree # Disallow directory /test User-agent: * Disallow: /test # Disallow directory sendMail.php User-agent: * Disallow: sendMail.php # Disallow directory contact_email.html User-agent: * Disallow: contact_email.html