## GENERAL SETTINGS
## 
## Enable robots.txt rules for all crawlers
User-agent: *
Crawl-delay: 10
##
Sitemap: http://www.georgeandking.com.au/sitemap/
##
## DEVELOPMENT RELATED SETTINGS
## Do not crawl development files and folders: CVS, svn directories and dump files
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$
##
##
## GENERAL MAGENTO SETTINGS
## Do not crawl common Magento technical folders
Disallow: /admin/
Disallow: /app/
Disallow: /dev/
Disallow: /downloader/
Disallow: /errors/
Disallow: /facebook/
Disallow: /includes/
Disallow: /ioncube/
#Disallow: /js/
Disallow: /kiosk/
Disallow: /lib/
#Disallow: /media/
Disallow: /newsletter/
Disallow: /offline/
Disallow: /pkginfo/
Disallow: /shell/
Disallow: /var/
Disallow: /skin/frontend/default/shopper/images/model_preview/
Disallow: /blog/wp-content/gallery-bank/
Disallow: /skin/frontend/default/shopper/images/newsletter/
##
## Do not crawl common Magento files
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt
##
##
## MAGENTO SEO IMPROVEMENTS
## Do not crawl sub category pages that are sorted or filtered.
##Disallow: /*?dir*
##Disallow: /*?dir=desc
##Disallow: /*?dir=asc
##Disallow: /*?limit=all
##Disallow: /*?mode*
##
## Do not crawl 2-nd home page copy (example.com/index.php/). Uncomment only if you activated Magento SEO URLs.
##
Disallow: /index.php/
##
## Do not crawl links with session IDs
##
Disallow: /*?SID=
##
## Paths (no clean URLs)
##Disallow: /*.js$
##Disallow: /*.css$
##Disallow: /*.php$
##
## Do not crawl checkout and user account pages
Disallow: /checkout/
Disallow: /checkout/onepage/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
##
## Do not crawl seach pages and not-SEO optimized catalog links
Disallow: /catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /review/product/list/
##
##
## SERVER SETTINGS
## Do not crawl common server technical folders and files
Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php
## 
## 
## IMAGE CRAWLERS SETTINGS
## Extra: To disable Google and Bing image indexing
## User-agent: Googlebot-Image
## Disallow: /
## User-agent: msnbot-media
## Disallow: /