summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'robots.txt')
-rw-r--r--robots.txt31
1 files changed, 1 insertions, 30 deletions
diff --git a/robots.txt b/robots.txt
index 9e3dc34..d2361d6 100644
--- a/robots.txt
+++ b/robots.txt
@@ -4,38 +4,9 @@ User-agent: googlebot-image
4Disallow: 4Disallow:
5User-agent: googlebot-mobile 5User-agent: googlebot-mobile
6Disallow: 6Disallow:
7User-agent: MSNBot 7User-agent: Baiduspider
8Disallow: / 8Disallow: /
9User-agent: Slurp
10Disallow: /
11User-agent: Teoma
12Disallow: /
13User-agent: twiceler
14Disallow: /
15User-agent: Gigabot
16Disallow: /
17User-agent: Scrubby
18Disallow: /
19User-agent: Robozilla
20Disallow: /
21User-agent: Nutch
22Disallow: /
23User-agent: ia_archiver
24Disallow:
25User-agent: baiduspider 9User-agent: baiduspider
26Disallow: / 10Disallow: /
27User-agent: naverbot
28Disallow: /
29User-agent: yeti
30Disallow: /
31User-agent: yahoo-mmcrawler
32Disallow: /
33User-agent: psbot
34Disallow: /
35User-agent: asterias
36Disallow: /
37User-agent: yahoo-blogs/v3.9
38Disallow: /
39User-agent: * 11User-agent: *
40Disallow: 12Disallow:
41
Powered by cgit v1.2.3 (git 2.41.0)