Robots.txt improvements.

- Replace file with configuration snippet.
- Allow some domains, disallow others.

conf.d/autoconfig.shore.co.il.conf

@@ -4,6 +4,7 @@ server {
     server_name autoconfig.shore.co.il;
     root        /var/www/autoconfig.shore.co.il/;
     include     snippets/www-acme-challenge.conf;
+    include     snippets/robots-allow-all.conf;
 }
 
 server {
@@ -11,5 +12,6 @@ server {
     listen      [::]:443 ssl;
     server_name autoconfig.shore.co.il;
     root        /var/www/autoconfig.shore.co.il/;
+    include     snippets/robots-allow-all.conf;
     include     snippets/ssl.conf;
 }

conf.d/git.shore.co.il.conf

@@ -4,6 +4,7 @@ server {
     listen      80;
     listen      [::]:80;
     server_name git.shore.co.il;
+    include     snippets/robots-allow-all.conf;
     include     snippets/www-acme-challenge.conf;
     include     snippets/redirect-https.conf;
 }
@@ -12,6 +13,7 @@ server {
     listen      443 ssl;
     listen      [::]:443 ssl;
     server_name git.shore.co.il;
+    include     snippets/robots-allow-all.conf;
     include     snippets/ssl.conf;
 
     location / {

conf.d/mail.shore.co.il.conf

@@ -3,6 +3,7 @@ server {
     listen      [::]:80;
     server_name imap.shore.co.il smtp.shore.co.il mta-sts.shore.co.il;
     root        /var/www/mail.shore.co.il/;
+    include     snippets/robots-disallow-all.conf;
 }
 
 server {
@@ -10,6 +11,7 @@ server {
     listen                      [::]:443 ssl;
     server_name                 mta-sts.shore.co.il;
     root                        /var/www/mail.shore.co.il/;
+    include                     snippets/robots-disallow-all.conf;
 
     # Copied from snippetes/ssl.conf.
     add_header                  Strict-Transport-Security "max-age=31536000; includeSubDomains; preload";

conf.d/nextcloud.shore.co.il.conf

@@ -4,6 +4,7 @@ server {
     listen      80;
     listen      [::]:80;
     server_name nextcloud.shore.co.il;
+    include     snippets/robots-disallow-all.conf;
     include     snippets/www-acme-challenge.conf;
     include     snippets/redirect-https.conf;
 }
@@ -12,6 +13,7 @@ server {
     listen      443 ssl;
     listen      [::]:443 ssl;
     server_name nextcloud.shore.co.il;
+    include     snippets/robots-disallow-all.conf;
     include     snippets/ssl.conf;
     include     snippets/nextcloud-well-known.conf;
 

conf.d/shore.co.il.conf

@@ -4,6 +4,7 @@ server {
     listen      80;
     listen      [::]:80;
     server_name shore.co.il;
+    include     snippets/robots-allow-all.conf;
     location = /        { return 301 https://$host/blog/; }
     location /repo/     {
         root            /var/www/www.shore.co.il/;
@@ -16,6 +17,7 @@ server {
     listen      443 ssl;
     listen      [::]:443 ssl;
     server_name shore.co.il;
+    include     snippets/robots-allow-all.conf;
     include     snippets/ssl.conf;
     include     snippets/nextcloud-well-known.conf;
     include     snippets/z-push.conf;

conf.d/www.shore.co.il.conf

@@ -6,6 +6,7 @@ server {
     listen              80 default_server;
     listen              [::]:80 default_server;
     server_name         www.shore.co.il;
+    include             snippets/robots-allow-all.conf;
     location = /        { return 301 https://$host/blog/; }
     location /repo/     {
         root            /var/www/www.shore.co.il/;
@@ -18,6 +19,7 @@ server {
     listen                  443 ssl default_server;
     listen                  [::]:443 ssl default_server;
     server_name             www.shore.co.il;
+    include                 snippets/robots-allow-all.conf;
     include                 snippets/ssl.conf;
     root                    /var/www/www.shore.co.il/;
     error_page 404          /;

snippets/robots-allow-all.conf

+location = /robots.txt {
+    add_header Content-Type text/plain;
+    return 200 "User-agent: *\nDisallow:\n";
+}

snippets/robots-disallow-all.conf

+location = /robots.txt {
+    add_header Content-Type text/plain;
+    return 200 "User-agent: *\nDisallow: *\n";
+}

www/www.shore.co.il/robots.txt

-User-agent: *
-Crawl-delay: 3600
-Host: www.shore.co.il