Sitemap: http://www.iwda.org.au/au/sitemap.xml

User-agent: sitecheck.internetseer.com
Disallow: /
 
User-agent: wget
Disallow: /
 
User-agent: Zealbot
Disallow: /
 
User-agent: MSIECrawler
Disallow: /
 
User-agent: SiteSnagger
Disallow: /
 
User-agent: WebStripper
Disallow: /
 
User-agent: WebCopier
Disallow: /
 
User-agent: Fetch
Disallow: /
 
User-agent: Offline Explorer
Disallow: /
 
User-agent: Teleport
Disallow: /
 
User-agent: TeleportPro
Disallow: /
 
User-agent: WebZIP
Disallow: /
 
User-agent: linko
Disallow: /
 
User-agent: HTTrack
Disallow: /
 
User-agent: Microsoft.URL.Control
Disallow: /
 
User-agent: Xenu
Disallow: /
 
User-agent: larbin
Disallow: /
 
User-agent: libwww
Disallow: /
 
User-agent: ZyBORG
Disallow: /
 
User-agent: Download Ninja
Disallow: /
 
User-agent: WebReaper
Disallow: /
 
User-agent: NPBot
Disallow: /
 
User-agent: k2spider
Disallow: /
 
User-agent: grub-client
Disallow: /
 
User-agent: *

# disallow all files in these directories
Disallow: /cgi-bin/
Disallow: /closed/
Disallow: /archive/images/
Disallow: /archive/
Disallow: /intranet/
Disallow: /gallery/
Disallow: /lists/
Disallow: /newsletter/
Disallow: /wp-org/
Disallow: /wp/gallery/albums/
Disallow: /wp/gallery/cache/


# disallow all files ending with these extensions
#Disallow: /*.php$
#Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
Disallow: /*.txt$
Disallow: /*.mp3$

# disallow all files in /wp- directorys
Disallow: /wp-*/
 
# disallow all files with? in url
Disallow: /*?*

User-agent: *

# if supported delays at least 2 second.
Crawl-delay: 5