#=========================================================================================================================================
#
# robots.txt file for shehaal.com
# Created by Stuart Morris
# Email stuart [AT] shehaal [DOT] com
#
#=========================================================================================================================================



#=========================================================================================================================================
#
# Revision History
#
# Date		    Changes
# ----------------------------------------------------------------------------------------------------------------------------------------
# 2007-04-25	Created initial file
# 2007-07-06	Added /css/ directory; re-enabled archiving for Way Back Machine/ Alexa
# 2007-07-08	Once again disabled Way back Machine/ Alexa, as all old pages from domain have been reincluded
# 2007-07-10	Added /includes/ directory as SSI is now used
# 2009-08-17    Added WordPress directories
# 2009-08-23    Modified /cgi-bin entry to only block Awstats so that redirected external links can be crawled
#
#=========================================================================================================================================



# Prevent Web Archive (Way Back Machine)/ Alexa from crawling the site
# This is to have old content removed from the archive
User-agent: ia_archiver
Disallow: /

User-agent: *
#Disallow: /cgi-bin/ ## Commented out so redirected links can be followed
Disallow: /cgi-bin/awstats/
Disallow: /cgi-bin/jawstats/
Disallow: /cgi-bin/perldriver/
Disallow: /cgi-bin/phpinfo.php
Disallow: /comments/
Disallow: /css/
Disallow: /images/
Disallow: /includes/
Disallow: /scripts/
Disallow: /stats/
Disallow: /trackback/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: */comment-page-*/
Disallow: */trackback/
Disallow: /wp-*