From 53347875073a252f20d2d417c80ec5a25f000b8b Mon Sep 17 00:00:00 2001 From: Jinwei Zhao Date: Tue, 12 Jan 2016 18:11:33 +0800 Subject: change robots.txt --- robots.txt | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/robots.txt b/robots.txt index 9e3dc34..d2361d6 100644 --- a/robots.txt +++ b/robots.txt @@ -4,38 +4,9 @@ User-agent: googlebot-image Disallow: User-agent: googlebot-mobile Disallow: -User-agent: MSNBot +User-agent: Baiduspider Disallow: / -User-agent: Slurp -Disallow: / -User-agent: Teoma -Disallow: / -User-agent: twiceler -Disallow: / -User-agent: Gigabot -Disallow: / -User-agent: Scrubby -Disallow: / -User-agent: Robozilla -Disallow: / -User-agent: Nutch -Disallow: / -User-agent: ia_archiver -Disallow: User-agent: baiduspider Disallow: / -User-agent: naverbot -Disallow: / -User-agent: yeti -Disallow: / -User-agent: yahoo-mmcrawler -Disallow: / -User-agent: psbot -Disallow: / -User-agent: asterias -Disallow: / -User-agent: yahoo-blogs/v3.9 -Disallow: / User-agent: * Disallow: - -- cgit v1.2.3