|
Chris@0
|
1 #
|
|
Chris@0
|
2 # robots.txt
|
|
Chris@0
|
3 #
|
|
Chris@0
|
4 # This file is to prevent the crawling and indexing of certain parts
|
|
Chris@0
|
5 # of your site by web crawlers and spiders run by sites like Yahoo!
|
|
Chris@0
|
6 # and Google. By telling these "robots" where not to go on your site,
|
|
Chris@0
|
7 # you save bandwidth and server resources.
|
|
Chris@0
|
8 #
|
|
Chris@0
|
9 # This file will be ignored unless it is at the root of your host:
|
|
Chris@0
|
10 # Used: http://example.com/robots.txt
|
|
Chris@0
|
11 # Ignored: http://example.com/site/robots.txt
|
|
Chris@0
|
12 #
|
|
Chris@0
|
13 # For more information about the robots.txt standard, see:
|
|
Chris@0
|
14 # http://www.robotstxt.org/robotstxt.html
|
|
Chris@0
|
15
|
|
Chris@0
|
16 User-agent: *
|
|
Chris@0
|
17 # CSS, JS, Images
|
|
Chris@0
|
18 Allow: /core/*.css$
|
|
Chris@0
|
19 Allow: /core/*.css?
|
|
Chris@0
|
20 Allow: /core/*.js$
|
|
Chris@0
|
21 Allow: /core/*.js?
|
|
Chris@0
|
22 Allow: /core/*.gif
|
|
Chris@0
|
23 Allow: /core/*.jpg
|
|
Chris@0
|
24 Allow: /core/*.jpeg
|
|
Chris@0
|
25 Allow: /core/*.png
|
|
Chris@0
|
26 Allow: /core/*.svg
|
|
Chris@0
|
27 Allow: /profiles/*.css$
|
|
Chris@0
|
28 Allow: /profiles/*.css?
|
|
Chris@0
|
29 Allow: /profiles/*.js$
|
|
Chris@0
|
30 Allow: /profiles/*.js?
|
|
Chris@0
|
31 Allow: /profiles/*.gif
|
|
Chris@0
|
32 Allow: /profiles/*.jpg
|
|
Chris@0
|
33 Allow: /profiles/*.jpeg
|
|
Chris@0
|
34 Allow: /profiles/*.png
|
|
Chris@0
|
35 Allow: /profiles/*.svg
|
|
Chris@0
|
36 # Directories
|
|
Chris@0
|
37 Disallow: /core/
|
|
Chris@0
|
38 Disallow: /profiles/
|
|
Chris@0
|
39 # Files
|
|
Chris@0
|
40 Disallow: /README.txt
|
|
Chris@0
|
41 Disallow: /web.config
|
|
Chris@0
|
42 # Paths (clean URLs)
|
|
Chris@0
|
43 Disallow: /admin/
|
|
Chris@0
|
44 Disallow: /comment/reply/
|
|
Chris@0
|
45 Disallow: /filter/tips/
|
|
Chris@0
|
46 Disallow: /node/add/
|
|
Chris@0
|
47 Disallow: /search/
|
|
Chris@0
|
48 Disallow: /user/register/
|
|
Chris@0
|
49 Disallow: /user/password/
|
|
Chris@0
|
50 Disallow: /user/login/
|
|
Chris@0
|
51 Disallow: /user/logout/
|
|
Chris@0
|
52 # Paths (no clean URLs)
|
|
Chris@0
|
53 Disallow: /index.php/admin/
|
|
Chris@0
|
54 Disallow: /index.php/comment/reply/
|
|
Chris@0
|
55 Disallow: /index.php/filter/tips/
|
|
Chris@0
|
56 Disallow: /index.php/node/add/
|
|
Chris@0
|
57 Disallow: /index.php/search/
|
|
Chris@0
|
58 Disallow: /index.php/user/password/
|
|
Chris@0
|
59 Disallow: /index.php/user/register/
|
|
Chris@0
|
60 Disallow: /index.php/user/login/
|
|
Chris@0
|
61 Disallow: /index.php/user/logout/
|
