Chris@0: #
Chris@0: # robots.txt
Chris@0: #
Chris@0: # This file is to prevent the crawling and indexing of certain parts
Chris@0: # of your site by web crawlers and spiders run by sites like Yahoo!
Chris@0: # and Google. By telling these "robots" where not to go on your site,
Chris@0: # you save bandwidth and server resources.
Chris@0: #
Chris@0: # This file will be ignored unless it is at the root of your host:
Chris@0: # Used:    http://example.com/robots.txt
Chris@0: # Ignored: http://example.com/site/robots.txt
Chris@0: #
Chris@0: # For more information about the robots.txt standard, see:
Chris@0: # http://www.robotstxt.org/robotstxt.html
Chris@0: 
Chris@0: User-agent: *
Chris@0: # CSS, JS, Images
Chris@0: Allow: /core/*.css$
Chris@0: Allow: /core/*.css?
Chris@0: Allow: /core/*.js$
Chris@0: Allow: /core/*.js?
Chris@0: Allow: /core/*.gif
Chris@0: Allow: /core/*.jpg
Chris@0: Allow: /core/*.jpeg
Chris@0: Allow: /core/*.png
Chris@0: Allow: /core/*.svg
Chris@0: Allow: /profiles/*.css$
Chris@0: Allow: /profiles/*.css?
Chris@0: Allow: /profiles/*.js$
Chris@0: Allow: /profiles/*.js?
Chris@0: Allow: /profiles/*.gif
Chris@0: Allow: /profiles/*.jpg
Chris@0: Allow: /profiles/*.jpeg
Chris@0: Allow: /profiles/*.png
Chris@0: Allow: /profiles/*.svg
Chris@0: # Directories
Chris@0: Disallow: /core/
Chris@0: Disallow: /profiles/
Chris@0: # Files
Chris@0: Disallow: /README.txt
Chris@0: Disallow: /web.config
Chris@0: # Paths (clean URLs)
Chris@0: Disallow: /admin/
Chris@0: Disallow: /comment/reply/
Chris@0: Disallow: /filter/tips/
Chris@0: Disallow: /node/add/
Chris@0: Disallow: /search/
Chris@0: Disallow: /user/register/
Chris@0: Disallow: /user/password/
Chris@0: Disallow: /user/login/
Chris@0: Disallow: /user/logout/
Chris@0: # Paths (no clean URLs)
Chris@0: Disallow: /index.php/admin/
Chris@0: Disallow: /index.php/comment/reply/
Chris@0: Disallow: /index.php/filter/tips/
Chris@0: Disallow: /index.php/node/add/
Chris@0: Disallow: /index.php/search/
Chris@0: Disallow: /index.php/user/password/
Chris@0: Disallow: /index.php/user/register/
Chris@0: Disallow: /index.php/user/login/
Chris@0: Disallow: /index.php/user/logout/