adds robots.txt per search.gov recs

Author: jduss4

.eleventy.js

@@ -27,6 +27,9 @@ module.exports = function (config) {
   // Copy the `admin` folders to the output
   config.addPassthroughCopy('admin');
 
+  // Copy the robots.txt file to the output
+  config.addPassthroughCopy('robots.txt');
+
   // Copy USWDS init JS so we can load it in HEAD to prevent banner flashing
   config.addPassthroughCopy({'./node_modules/@uswds/uswds/dist/js/uswds-init.js': 'assets/js/uswds-init.js'});
 

robots.txt

@@ -0,0 +1,9 @@
+Sitemap: https://guides.18f.gov/sitemap.xml
+
+User-agent: usasearch
+Allow: /
+Crawl-delay: 2
+
+User-agent: *
+Allow: /
+Crawl-delay: 10