Updated robots.txt to disallow auto-generated index pages

2013-01-04 23:26:22 +00:00
parent a6a39fcb36
commit f65b4cca28
1 changed files with 11 additions and 3 deletions
--- a/i2p2www/static/robots.txt
+++ b/i2p2www/static/robots.txt
@@ -1,4 +1,12 @@
-
-# allow everything for everyone... at least for now
+# Disallow auto-generated index pages
 User-agent: *
-Disallow: 
+Disallow: /*/blog/$
+
+User-agent: *
+Disallow: /*/blog/page/
+
+User-agent: *
+Disallow: /*/meetings/$
+
+User-agent: *
+Disallow: /*/meetings/page/