diff options
Diffstat (limited to 'robots.txt')
| -rw-r--r-- | robots.txt | 41 |
1 files changed, 41 insertions, 0 deletions
diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..9e3dc34 --- /dev/null +++ b/robots.txt | |||
| @@ -0,0 +1,41 @@ | |||
| 1 | User-agent: Googlebot | ||
| 2 | Disallow: | ||
| 3 | User-agent: googlebot-image | ||
| 4 | Disallow: | ||
| 5 | User-agent: googlebot-mobile | ||
| 6 | Disallow: | ||
| 7 | User-agent: MSNBot | ||
| 8 | Disallow: / | ||
| 9 | User-agent: Slurp | ||
| 10 | Disallow: / | ||
| 11 | User-agent: Teoma | ||
| 12 | Disallow: / | ||
| 13 | User-agent: twiceler | ||
| 14 | Disallow: / | ||
| 15 | User-agent: Gigabot | ||
| 16 | Disallow: / | ||
| 17 | User-agent: Scrubby | ||
| 18 | Disallow: / | ||
| 19 | User-agent: Robozilla | ||
| 20 | Disallow: / | ||
| 21 | User-agent: Nutch | ||
| 22 | Disallow: / | ||
| 23 | User-agent: ia_archiver | ||
| 24 | Disallow: | ||
| 25 | User-agent: baiduspider | ||
| 26 | Disallow: / | ||
| 27 | User-agent: naverbot | ||
| 28 | Disallow: / | ||
| 29 | User-agent: yeti | ||
| 30 | Disallow: / | ||
| 31 | User-agent: yahoo-mmcrawler | ||
| 32 | Disallow: / | ||
| 33 | User-agent: psbot | ||
| 34 | Disallow: / | ||
| 35 | User-agent: asterias | ||
| 36 | Disallow: / | ||
| 37 | User-agent: yahoo-blogs/v3.9 | ||
| 38 | Disallow: / | ||
| 39 | User-agent: * | ||
| 40 | Disallow: | ||
| 41 | |||