User-agent: ia_archiver*
Disallow: /
User-agent: FreeWebMonitoring SiteChecker*
Disallow: /
User-agent: Xenu Link Sleuth*
Disallow: /
User-agent: Yandex*
Disallow: /
User-agent: Yanga WorldSearch Bot*
Disallow: /
User-agent: *
Disallow: /temp/
Disallow: /stats/
Disallow: /directory/
Disallow: /gallery/
Disallow: /yankee/
Disallow: /_private/
Disallow: /copley-place/store-list-by-name.htm
Disallow: /prudential-center/store-list-by-name.htm
Disallow: /newbury-street/001-100.htm
Disallow: /newbury-street/101-200.htm
Disallow: /newbury-street/201-333.htm
Disallow: /strange/witches.htm
Disallow: /investment/financial.htm
Disallow: /postcard/culture.htm
Disallow: /sites/old-north-church.htm
Disallow: /sites/uss-constitution.htm
Disallow: /mbta/brbl/
Disallow: /mbta/bery/
Disallow: /photos/walden-pond/
# White List Version
# User-agent: googlebot
# User-agent: Adsbot-Google
# User-agent: Mediapartners-Google
# User-agent: Googlebot-Image
# User-agent: Googlebot-Mobile
# User-agent: msnbot
# User-agent: msnbot-media
# User-agent: SearchSight
# User-agent: slurp
# User-agent: teoma
# User-agent: TravelNow_Template_Robot
# User-agent: WDG_SiteValidator
# Disallow:
# User-agent: *
# Disallow: /