# robots.txt User-agent: Googlebot # google Disallow: / # block all. It gets stuck in biz directory User-agent: * Disallow: /ajax.cfm Disallow: /api.cfm Disallow: /content.cfm Disallow: /site/ajax.cfm Disallow: /site/api.cfm Disallow: /site/content.cfm Disallow: # Crawl-delay: 20 Allow: /client_images/*.jpg$ Allow: /client_images/*.jpeg$ Allow: /client_images/*.png$ Allow: /client_images/*.gif$ Allow: /client_images/*.JPG$ Allow: /client_images/*.JPEG$ Allow: /client_images/*.PNG$ Allow: /client_images/*.GIF$ Disallow: /client_images/