# ============================================================
# robots.txt - stop webcrawlers/spiders from 
# filling up log files with irrelevant searches
#
# Updated 16 Dec 2016
# ============================================================

User-agent: *
#Crawl-delay: 5
Disallow: /navigation/				# DOES NOT EXIST
Disallow: /local/					# internal service tools
Disallow: /staff/					# staff/ superceded by staffnet/ - added 26 Feb 2013
Disallow: /hr/						# hr/ superceded by staffnet/ - added 28 Mar 2013
Disallow: /ppg/						# ppg/ superceded by staffnet/ - added 28 Mar 2013
Disallow: /modernthought/wiki/		# broken, remove - added 02 Apr 2013
Disallow: /hoart/					# added 11 Apr 2013
Disallow: /history/					# added 11 Apr 2013
Disallow: /divinity/				# added 11 Apr 2013
Disallow: /philosophy/				# added 11 Apr 2013
Disallow: /cfss/					# added 11 Apr 2013
Disallow: /cshad/					# added 11 Apr 2013
Disallow: /religious_studies/		# added 11 Apr 2013
Disallow: /clsm/graduate-school/	# added 18 Apr 2013
Disallow: /clsm/graduateschool/		# added 18 Apr 2013
Disallow: /clsm/old-graduate/		# archive - added 18 Apr 2013
Disallow: /dit/staff/				# added 20 May 2013
Disallow: /old-clsm/					# added 5 Jun 2013, CHD
Disallow: /old-geography/				# added 5 Jun 2013, CHD
Disallow: /old-medicine/				# added 5 Jun 2013, CHD
Disallow: /old-museum/					# added 5 Jun 2013, CHD
Disallow: /old-museums/					# added 5 Jun 2013, CHD
Disallow: /old-rsd/						# added 5 Jun 2013, CHD
Disallow: /registry/courses03/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses04/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses05/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses06/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses07/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses08/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses09/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses10/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses11/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses13/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/courses13a/		# previous course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/pgcourses08/	# previous PG course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/pgcourses09/	# previous PG course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/pgcourses10/	# previous PG course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/pgcourses11/	# previous PG course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/pgcourses13/	# previous PG course catalogue - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2001/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2002/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2003/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2004/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2005/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2006/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2007/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2008/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2009/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2010/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2011/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /registry/calendar2013/	# previous course calendar - added 06 Jun 2013, MJSK
Disallow: /finance/forms/				# added 12 Jun 2013, HMP
Disallow: /alumni-giving/				# added 20 Jun 2013, AB
Disallow: /old-education/				# added 18 Jul 2013, HMP
Disallow: /virtual/						# added 19 Jul 2013, AB 
Disallow: /old-ibes/					# added 30 Jul 2013, HMP
Disallow: /old-giving/					# added 30 Jul 2013, CHD
Disallow: /simpledemo/					# added 22 Aug 2013, AB
Disallow: /simpledev/					# added 22 Aug 2013, AB
Disallow: /celtic/						# added 30 Aug 2013, AB
Disallow: /english/						# added 30 Aug 2013, AB
Disallow: /film/						# added 30 Aug 2013, AB
Disallow: /french/						# added 30 Aug 2013, AB
Disallow: /gaelic/						# added 30 Aug 2013, AB
Disallow: /german/						# added 30 Aug 2013, AB
Disallow: /spanish/						# added 30 Aug 2013, AB
Disallow: /langling/					# added 30 Aug 2013, AB
Disallow: /lwc/							# added 30 Aug 2013, AB
Disallow: /old-nhc/						# added 08 Oct 2013, HMP
Disallow: /old-cass-graduate/			# added 14 Nov 2013, CHD
Disallow: /chd-play/					# added 14 Nov 2013, CHD
Disallow: /dev-summer/					# added 16 Dec 2013, HMP
Disallow: /old-heru/					# added 18 Dec 2013, HMP
Disallow: /biologicalsci/				# added 15 Jan 2014, AB
Disallow: /ucu/people/					# added 19 Feb 2014, MJSK
Disallow: /dit/							# added 20 Feb 2014, AB
Disallow: /avu/							# added 20 Feb 2014, AB
Disallow: /tad/							# added 20 Feb 2014, AB
Disallow: /old-modernthought/			# added 28 Feb 2014, HMP
Disallow: /old-accommodation/			# added 13 Mar 2014, CHD
Disallow: /old-socsci/					# added 20 May 2014, MJSK
Disallow: /old-anthropology/			# added 20 May 2014, MJSK
Disallow: /old-sociology/				# added 20 May 2014, MJSK
Disallow: /old-pir/						# added 20 May 2014, MJSK
Disallow: /anthropology/				# added 20 May 2014, MJSK
Disallow: /sociology/					# added 20 May 2014, MJSK
Disallow: /pir/							# added 20 May 2014, MJSK
Disallow: /old-cops-staff/				# added 10 Jun 2014, CHD
Disallow: /old-geology/					# added 10 Jun 2014, CHD
Disallow: /old-geosciences/				# added 10 Jun 2014, CHD
Disallow: /old-rowett/					# added 10 Jun 2014, CHD
Disallow: /old-clinicalskills/			# added 02 Apr 2015, CHD
Disallow: /old-suttie-centre/			# added 02 Apr 2015, CHD
Disallow: /old-clinicalskills_archive/	# added 02 Apr 2015, CHD
Disallow: /study/international/inmycountry/index/	# added 20 Jun 2014, AB
Disallow: /old-music/					# added 17 Mar 2015, AB
Disallow: /chd-play/lt-test/			# added 21 Sep 2015, CHD
Disallow: /dev-manskills/				# added 21 Oct 2015, CHD
Disallow: /dev-management-skills/		# added 21 Oct 2015, CHD
Disallow: /mgtskills/					# added 27 Oct 2015, AB
Disallow: /smd/					# added 11 Feb 2016, JJM
Disallow: /sms/					# added 11 Feb 2016, JJM
Disallow: /old-smd/					# added 11 Feb 2016, JJM
Disallow: /old-sms/					# added 11 Feb 2016, JJM