diff --git a/conf.d/autoconfig.shore.co.il.conf b/conf.d/autoconfig.shore.co.il.conf index 62b1be4145c4d31c09cf982fbd57f5622cd33c65..18ac9e0542ac71af1cdf02daffe2b681946a303e 100644 --- a/conf.d/autoconfig.shore.co.il.conf +++ b/conf.d/autoconfig.shore.co.il.conf @@ -4,6 +4,7 @@ server { server_name autoconfig.shore.co.il; root /var/www/autoconfig.shore.co.il/; include snippets/www-acme-challenge.conf; + include snippets/robots-allow-all.conf; } server { @@ -11,5 +12,6 @@ server { listen [::]:443 ssl; server_name autoconfig.shore.co.il; root /var/www/autoconfig.shore.co.il/; + include snippets/robots-allow-all.conf; include snippets/ssl.conf; } diff --git a/conf.d/git.shore.co.il.conf b/conf.d/git.shore.co.il.conf index 733fa63b344b0ee293055ddee13a36197cd70ec8..bf45c7028f30e17e5eebd671076d48056af26594 100644 --- a/conf.d/git.shore.co.il.conf +++ b/conf.d/git.shore.co.il.conf @@ -4,6 +4,7 @@ server { listen 80; listen [::]:80; server_name git.shore.co.il; + include snippets/robots-allow-all.conf; include snippets/www-acme-challenge.conf; include snippets/redirect-https.conf; } @@ -12,6 +13,7 @@ server { listen 443 ssl; listen [::]:443 ssl; server_name git.shore.co.il; + include snippets/robots-allow-all.conf; include snippets/ssl.conf; location / { diff --git a/conf.d/mail.shore.co.il.conf b/conf.d/mail.shore.co.il.conf index ed74238f5992233b22050a726b8690c89be55b40..4cfcbccdc26bc7a6d9b6241902187ec2b26038b8 100644 --- a/conf.d/mail.shore.co.il.conf +++ b/conf.d/mail.shore.co.il.conf @@ -3,6 +3,7 @@ server { listen [::]:80; server_name imap.shore.co.il smtp.shore.co.il mta-sts.shore.co.il; root /var/www/mail.shore.co.il/; + include snippets/robots-disallow-all.conf; } server { @@ -10,6 +11,7 @@ server { listen [::]:443 ssl; server_name mta-sts.shore.co.il; root /var/www/mail.shore.co.il/; + include snippets/robots-disallow-all.conf; # Copied from snippetes/ssl.conf. add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"; diff --git a/conf.d/nextcloud.shore.co.il.conf b/conf.d/nextcloud.shore.co.il.conf index 19b8a1ded070b7a50e901969def4654e18afc247..f6196994b85acd811d4f776eb83c79877aeffdd8 100644 --- a/conf.d/nextcloud.shore.co.il.conf +++ b/conf.d/nextcloud.shore.co.il.conf @@ -4,6 +4,7 @@ server { listen 80; listen [::]:80; server_name nextcloud.shore.co.il; + include snippets/robots-disallow-all.conf; include snippets/www-acme-challenge.conf; include snippets/redirect-https.conf; } @@ -12,6 +13,7 @@ server { listen 443 ssl; listen [::]:443 ssl; server_name nextcloud.shore.co.il; + include snippets/robots-disallow-all.conf; include snippets/ssl.conf; include snippets/nextcloud-well-known.conf; diff --git a/conf.d/shore.co.il.conf b/conf.d/shore.co.il.conf index 543f0bb289c219d40565bd199e8bc8b59cdcf7b2..e6676926bfbfdad00e528ce1369809a222395b61 100644 --- a/conf.d/shore.co.il.conf +++ b/conf.d/shore.co.il.conf @@ -4,6 +4,7 @@ server { listen 80; listen [::]:80; server_name shore.co.il; + include snippets/robots-allow-all.conf; location = / { return 301 https://$host/blog/; } location /repo/ { root /var/www/www.shore.co.il/; @@ -16,6 +17,7 @@ server { listen 443 ssl; listen [::]:443 ssl; server_name shore.co.il; + include snippets/robots-allow-all.conf; include snippets/ssl.conf; include snippets/nextcloud-well-known.conf; include snippets/z-push.conf; diff --git a/conf.d/www.shore.co.il.conf b/conf.d/www.shore.co.il.conf index a2fdeaaa51fcbba9fd159bda669a18ff8675f748..ef07233cb270f80bdb55c7fe08b7da53d6a5dac7 100644 --- a/conf.d/www.shore.co.il.conf +++ b/conf.d/www.shore.co.il.conf @@ -6,6 +6,7 @@ server { listen 80 default_server; listen [::]:80 default_server; server_name www.shore.co.il; + include snippets/robots-allow-all.conf; location = / { return 301 https://$host/blog/; } location /repo/ { root /var/www/www.shore.co.il/; @@ -18,6 +19,7 @@ server { listen 443 ssl default_server; listen [::]:443 ssl default_server; server_name www.shore.co.il; + include snippets/robots-allow-all.conf; include snippets/ssl.conf; root /var/www/www.shore.co.il/; error_page 404 /; diff --git a/snippets/robots-allow-all.conf b/snippets/robots-allow-all.conf new file mode 100644 index 0000000000000000000000000000000000000000..bff79a7517bd02812e713ad8c0c2b6eb9c7a2072 --- /dev/null +++ b/snippets/robots-allow-all.conf @@ -0,0 +1,4 @@ +location = /robots.txt { + add_header Content-Type text/plain; + return 200 "User-agent: *\nDisallow:\n"; +} diff --git a/snippets/robots-disallow-all.conf b/snippets/robots-disallow-all.conf new file mode 100644 index 0000000000000000000000000000000000000000..d5c02655ab880ac7873d74d597025e238c07d822 --- /dev/null +++ b/snippets/robots-disallow-all.conf @@ -0,0 +1,4 @@ +location = /robots.txt { + add_header Content-Type text/plain; + return 200 "User-agent: *\nDisallow: *\n"; +} diff --git a/www/www.shore.co.il/robots.txt b/www/www.shore.co.il/robots.txt deleted file mode 100644 index 5a6f1a9d3d065021b4cd1ce42c21892d9b49011b..0000000000000000000000000000000000000000 --- a/www/www.shore.co.il/robots.txt +++ /dev/null @@ -1,3 +0,0 @@ -User-agent: * -Crawl-delay: 3600 -Host: www.shore.co.il