From de40416298e3e49cfbcaa45e70327566d28bda6b Mon Sep 17 00:00:00 2001
From: Adar Nimrod <nimrod@shore.co.il>
Date: Sun, 6 Dec 2020 19:43:02 +0200
Subject: [PATCH] Robots.txt improvements.
- Replace file with configuration snippet.
- Allow some domains, disallow others.
---
conf.d/autoconfig.shore.co.il.conf | 2 ++
conf.d/git.shore.co.il.conf | 2 ++
conf.d/mail.shore.co.il.conf | 2 ++
conf.d/nextcloud.shore.co.il.conf | 2 ++
conf.d/shore.co.il.conf | 2 ++
conf.d/www.shore.co.il.conf | 2 ++
snippets/robots-allow-all.conf | 4 ++++
snippets/robots-disallow-all.conf | 4 ++++
www/www.shore.co.il/robots.txt | 3 ---
9 files changed, 20 insertions(+), 3 deletions(-)
create mode 100644 snippets/robots-allow-all.conf
create mode 100644 snippets/robots-disallow-all.conf
delete mode 100644 www/www.shore.co.il/robots.txt
diff --git a/conf.d/autoconfig.shore.co.il.conf b/conf.d/autoconfig.shore.co.il.conf
index 62b1be4..18ac9e0 100644
--- a/conf.d/autoconfig.shore.co.il.conf
+++ b/conf.d/autoconfig.shore.co.il.conf
@@ -4,6 +4,7 @@ server {
server_name autoconfig.shore.co.il;
root /var/www/autoconfig.shore.co.il/;
include snippets/www-acme-challenge.conf;
+ include snippets/robots-allow-all.conf;
}
server {
@@ -11,5 +12,6 @@ server {
listen [::]:443 ssl;
server_name autoconfig.shore.co.il;
root /var/www/autoconfig.shore.co.il/;
+ include snippets/robots-allow-all.conf;
include snippets/ssl.conf;
}
diff --git a/conf.d/git.shore.co.il.conf b/conf.d/git.shore.co.il.conf
index 733fa63..bf45c70 100644
--- a/conf.d/git.shore.co.il.conf
+++ b/conf.d/git.shore.co.il.conf
@@ -4,6 +4,7 @@ server {
listen 80;
listen [::]:80;
server_name git.shore.co.il;
+ include snippets/robots-allow-all.conf;
include snippets/www-acme-challenge.conf;
include snippets/redirect-https.conf;
}
@@ -12,6 +13,7 @@ server {
listen 443 ssl;
listen [::]:443 ssl;
server_name git.shore.co.il;
+ include snippets/robots-allow-all.conf;
include snippets/ssl.conf;
location / {
diff --git a/conf.d/mail.shore.co.il.conf b/conf.d/mail.shore.co.il.conf
index ed74238..4cfcbcc 100644
--- a/conf.d/mail.shore.co.il.conf
+++ b/conf.d/mail.shore.co.il.conf
@@ -3,6 +3,7 @@ server {
listen [::]:80;
server_name imap.shore.co.il smtp.shore.co.il mta-sts.shore.co.il;
root /var/www/mail.shore.co.il/;
+ include snippets/robots-disallow-all.conf;
}
server {
@@ -10,6 +11,7 @@ server {
listen [::]:443 ssl;
server_name mta-sts.shore.co.il;
root /var/www/mail.shore.co.il/;
+ include snippets/robots-disallow-all.conf;
# Copied from snippetes/ssl.conf.
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload";
diff --git a/conf.d/nextcloud.shore.co.il.conf b/conf.d/nextcloud.shore.co.il.conf
index 19b8a1d..f619699 100644
--- a/conf.d/nextcloud.shore.co.il.conf
+++ b/conf.d/nextcloud.shore.co.il.conf
@@ -4,6 +4,7 @@ server {
listen 80;
listen [::]:80;
server_name nextcloud.shore.co.il;
+ include snippets/robots-disallow-all.conf;
include snippets/www-acme-challenge.conf;
include snippets/redirect-https.conf;
}
@@ -12,6 +13,7 @@ server {
listen 443 ssl;
listen [::]:443 ssl;
server_name nextcloud.shore.co.il;
+ include snippets/robots-disallow-all.conf;
include snippets/ssl.conf;
include snippets/nextcloud-well-known.conf;
diff --git a/conf.d/shore.co.il.conf b/conf.d/shore.co.il.conf
index 543f0bb..e667692 100644
--- a/conf.d/shore.co.il.conf
+++ b/conf.d/shore.co.il.conf
@@ -4,6 +4,7 @@ server {
listen 80;
listen [::]:80;
server_name shore.co.il;
+ include snippets/robots-allow-all.conf;
location = / { return 301 https://$host/blog/; }
location /repo/ {
root /var/www/www.shore.co.il/;
@@ -16,6 +17,7 @@ server {
listen 443 ssl;
listen [::]:443 ssl;
server_name shore.co.il;
+ include snippets/robots-allow-all.conf;
include snippets/ssl.conf;
include snippets/nextcloud-well-known.conf;
include snippets/z-push.conf;
diff --git a/conf.d/www.shore.co.il.conf b/conf.d/www.shore.co.il.conf
index a2fdeaa..ef07233 100644
--- a/conf.d/www.shore.co.il.conf
+++ b/conf.d/www.shore.co.il.conf
@@ -6,6 +6,7 @@ server {
listen 80 default_server;
listen [::]:80 default_server;
server_name www.shore.co.il;
+ include snippets/robots-allow-all.conf;
location = / { return 301 https://$host/blog/; }
location /repo/ {
root /var/www/www.shore.co.il/;
@@ -18,6 +19,7 @@ server {
listen 443 ssl default_server;
listen [::]:443 ssl default_server;
server_name www.shore.co.il;
+ include snippets/robots-allow-all.conf;
include snippets/ssl.conf;
root /var/www/www.shore.co.il/;
error_page 404 /;
diff --git a/snippets/robots-allow-all.conf b/snippets/robots-allow-all.conf
new file mode 100644
index 0000000..bff79a7
--- /dev/null
+++ b/snippets/robots-allow-all.conf
@@ -0,0 +1,4 @@
+location = /robots.txt {
+ add_header Content-Type text/plain;
+ return 200 "User-agent: *\nDisallow:\n";
+}
diff --git a/snippets/robots-disallow-all.conf b/snippets/robots-disallow-all.conf
new file mode 100644
index 0000000..d5c0265
--- /dev/null
+++ b/snippets/robots-disallow-all.conf
@@ -0,0 +1,4 @@
+location = /robots.txt {
+ add_header Content-Type text/plain;
+ return 200 "User-agent: *\nDisallow: *\n";
+}
diff --git a/www/www.shore.co.il/robots.txt b/www/www.shore.co.il/robots.txt
deleted file mode 100644
index 5a6f1a9..0000000
--- a/www/www.shore.co.il/robots.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-User-agent: *
-Crawl-delay: 3600
-Host: www.shore.co.il
--
GitLab