[FEATURE] Add robots.txt

This will improve performance as currently bots are fetching contents
from /changes which contains JSON data of patches, diffs as files and
archives which are generated on the fly, really nothing of relevance
to search engines.

assets/etc/gerrit.config

@@ -56,6 +56,7 @@
 
 [httpd]
 	listenUrl = proxy-https://*:8080/
+	robotsFile = etc/robots.txt
 
 [sshd]
 	listenAddress = *:29418

assets/etc/robots.txt

+# Directions for web crawlers.
+# See http://www.robotstxt.org/
+
+# First deny access for several bots, then define rules for the allowed ones below
+
+# Some bots are known to be trouble, particularly those designed to copy entire sites. Please obey robots.txt.
+# (copied from wikipedia.org)
+User-agent: sitecheck.internetseer.com
+Disallow: /
+
+User-agent: Zealbot
+Disallow: /
+
+User-agent: MSIECrawler
+Disallow: /
+
+User-agent: SiteSnagger
+Disallow: /
+
+User-agent: WebStripper
+Disallow: /
+
+User-agent: WebCopier
+Disallow: /
+
+User-agent: Fetch
+Disallow: /
+
+User-agent: Offline Explorer
+Disallow: /
+
+User-agent: Teleport
+Disallow: /
+
+User-agent: TeleportPro
+Disallow: /
+
+User-agent: WebZIP
+Disallow: /
+
+User-agent: linko
+Disallow: /
+
+User-agent: HTTrack
+Disallow: /
+
+User-agent: Microsoft.URL.Control
+Disallow: /
+
+User-agent: Xenu
+Disallow: /
+
+User-agent: larbin
+Disallow: /
+
+User-agent: libwww
+Disallow: /
+
+User-agent: ZyBORG
+Disallow: /
+
+User-agent: Download Ninja
+Disallow: /
+
+# Misbehaving: requests much too fast:
+User-agent: fast
+Disallow: /
+
+
+# additional denied bots as proposed by default robots.txt of Gerrit
+User-agent: puf
+Disallow: /
+
+User-agent: Nutch
+Disallow: /
+
+
+
+# All allowed bots are ask to skip /changes as these return JSON code content only, impact performance and are of no value to search engines
+User-agent: *
+Disallow: /changes/
+
+# All bots are asked to pause at least 5 seconds between two visits
+Crawl-Delay: 5