Skip to content
Snippets Groups Projects
Commit a50746ca authored by nimrod's avatar nimrod
Browse files

Major refactor.

- Check base class, all checks derive from it and have a common calling
convention.
- Add retries.
- Add a DNS zone transfer check. Turns out if fails on ns1. WIP.
- A wrapper around checking web sites. Just provide a list of URLs.
parent 8ad9776a
No related branches found
No related tags found
No related merge requests found
Pipeline #1378 passed
from functools import lru_cache
import dns.resolver # pylint: disable=import-error import dns.resolver # pylint: disable=import-error
import utils import dns.query # pylint: disable=import-error
import dns.zone # pylint: disable=import-error
from utils import Check
DOMAIN = "shore.co.il" class CheckDNS(Check): # pylint: disable=abstract-method
SUBDOMAINS = [DOMAIN, f"www.{DOMAIN}"] _domain = "shore.co.il"
_subdomains = [_domain, f"www.{_domain}"]
@lru_cache(3)
def get_resolvers(): def _get_resolvers(self):
"""Return a list of resolvers for each nameserver in the DNS zone.""" """Return a list of resolvers for each nameserver in the DNS zone."""
default_resolver = dns.resolver.Resolver() default_resolver = dns.resolver.Resolver()
resolvers = [] resolvers = []
for ns in default_resolver.query( # pylint: disable=invalid-name for ns in default_resolver.query( # pylint: disable=invalid-name
DOMAIN, "NS" self._domain, "NS"
): ):
for address in default_resolver.query(ns.to_text()): for address in default_resolver.query(ns.to_text()):
resolver = dns.resolver.Resolver(configure=False) resolver = dns.resolver.Resolver(configure=False)
...@@ -19,74 +23,88 @@ def get_resolvers(): ...@@ -19,74 +23,88 @@ def get_resolvers():
resolvers.append(resolver) resolvers.append(resolver)
return resolvers return resolvers
def _cross_query(self, qname, rdtype="A"):
RESOLVERS = get_resolvers()
def cross_query(qname, rdtype="A"):
"""Return all of the answers from all nameservers.""" """Return all of the answers from all nameservers."""
answers = set() answers = set()
for resolver in RESOLVERS: for resolver in self._resolvers:
for address in resolver.query(qname, rdtype): for address in resolver.query(qname, rdtype):
answers.add(address) answers.add(address)
return answers return answers
def __init__(self):
self._resolvers = self._get_resolvers()
def validate_soa(): class CheckSOA(CheckDNS):
def _check(self):
"""Validate the SOA record.""" """Validate the SOA record."""
soas = cross_query(DOMAIN, "SOA") soas = self._cross_query(self._domain, "SOA")
if len(soas) > 1: if len(soas) > 1:
return [False, "SOA records don't match."] return False, "SOA records don't match."
try: try:
record = soas.pop() record = soas.pop()
record.mname.to_text() record.mname.to_text()
except Exception as e: # pylint: disable=broad-except,invalid-name except Exception as e: # pylint: disable=broad-except,invalid-name
print(str(e)) return False, str(e)
return [False, "SOA record is invalid."]
return [True, "SOA record validated."] return True, "SOA record validated."
def validate_mx(): class CheckMX(CheckDNS):
def _check(self):
"""Validate the MX record.""" """Validate the MX record."""
mxs = cross_query(DOMAIN, "MX") mxs = self._cross_query(self._domain, "MX")
if len(mxs) > 1: if len(mxs) > 1:
return [False, "MX records don't match."] return False, "MX records don't match."
try: try:
record = mxs.pop() record = mxs.pop()
ips = cross_query(record.exchange.to_text()) ips = self._cross_query(record.exchange.to_text())
if len(ips) > 1: if len(ips) > 1:
return [False, "MX records don't match."] return False, "MX records don't match."
except Exception as e: # pylint: disable=broad-except,invalid-name except Exception as e: # pylint: disable=broad-except,invalid-name
print(str(e)) return [False, str(e)]
return [False, "MX record is invalid."]
return [True, "MX record validated."] return True, "MX record validated."
def validate_subdomains(): class CheckSubDomains(CheckDNS):
def _check(self):
"""Validate important subdomains.""" """Validate important subdomains."""
for domain in SUBDOMAINS: for domain in self._subdomains:
try: try:
ips = cross_query(domain) ips = self._cross_query(domain)
if len(ips) > 1: if len(ips) > 1:
return [True, f"Domain {domain} records don't match."] return [True, f"Domain {domain} records don't match."]
except Exception as e: # pylint: disable=broad-except,invalid-name except Exception as e: # pylint: disable=broad-except,invalid-name
print(str(e)) return [False, str(e)]
return [False, "Failed to validate domain {d}."] return True, "Subdomains validated."
return [True, "Subdomains validated."]
class CheckTransfer(CheckDNS):
def _check(self):
"""Validate zone transfer, to check the TCP transport."""
zones = []
for resolver in self._resolvers:
try:
zone = dns.zone.from_xfr(
dns.query.xfr(resolver.nameservers[0], self._domain)
)
zones.append(zone)
except Exception as e: # pylint: disable=broad-except,invalid-name
return False, str(e)
return True, "Zone transfer validated."
def handler(event, context): # pylint: disable=unused-argument def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler.""" """Lambda event handler."""
for check in [validate_soa, validate_mx, validate_subdomains]: CheckSOA().run()
success, message = check.__call__() CheckMX().run()
print(message) CheckSubDomains().run()
if not success: # CheckTransfer().run() # AXFR is blocked on ns1, need to figure out why.
utils.publish(message)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "https://autoconfig.shore.co.il/mail/config-v1.1.xml"}, {"url": "https://autoconfig.shore.co.il/mail/config-v1.1.xml"},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://git.shore.co.il/", "codes": [301, 302]}, {"url": "http://git.shore.co.il/", "codes": [301, 302]},
{"url": "https://git.shore.co.il/", "codes": [301, 302]}, {"url": "https://git.shore.co.il/", "codes": [301, 302]},
{"url": "https://git.shore.co.il/explore/"}, {"url": "https://git.shore.co.il/explore/"},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from imaplib import IMAP4_SSL from imaplib import IMAP4_SSL
from utils import publish from utils import Check
def check_imap(): class CheckIMAP(Check):
def _check(self):
"""Check the IMAP port.""" """Check the IMAP port."""
try: try:
imap = IMAP4_SSL("imap.shore.co.il", 993) imap = IMAP4_SSL("imap.shore.co.il", 993)
...@@ -19,10 +20,7 @@ def check_imap(): ...@@ -19,10 +20,7 @@ def check_imap():
def handler(event, context): # pylint: disable=unused-argument def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler.""" """Lambda event handler."""
success, message = check_imap() CheckIMAP().run()
print(message)
if not success:
publish(message)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://kodi.shore.co.il/", "codes": [301, 302]}, {"url": "http://kodi.shore.co.il/", "codes": [301, 302]},
{"url": "https://kodi.shore.co.il/", "codes": [401]}, {"url": "https://kodi.shore.co.il/", "codes": [401]},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "https://mta-sts.shore.co.il/.well-known/mta-sts.txt"}, {"url": "https://mta-sts.shore.co.il/.well-known/mta-sts.txt"},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://myip.shore.co.il/"}, {"url": "http://myip.shore.co.il/"},
{"url": "https://myip.shore.co.il/"}, {"url": "https://myip.shore.co.il/"},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://nextcloud.shore.co.il/", "codes": [301, 302]}, {"url": "http://nextcloud.shore.co.il/", "codes": [301, 302]},
{"url": "https://nextcloud.shore.co.il/", "codes": [301, 302]}, {"url": "https://nextcloud.shore.co.il/", "codes": [301, 302]},
{"url": "https://nextcloud.shore.co.il/login"}, {"url": "https://nextcloud.shore.co.il/login"},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://notify.shore.co.il/", "codes": [301, 302]}, {"url": "http://notify.shore.co.il/", "codes": [301, 302]},
{"url": "https://notify.shore.co.il/ping"}, {"url": "https://notify.shore.co.il/ping"},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://registry.shore.co.il/", "codes": [301, 302]}, {"url": "http://registry.shore.co.il/", "codes": [301, 302]},
{"url": "https://registry.shore.co.il/"}, {"url": "https://registry.shore.co.il/"},
{"url": "https://registry.shore.co.il/v2/_catalog"}, {"url": "https://registry.shore.co.il/v2/_catalog"},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from smtplib import SMTP from smtplib import SMTP
from utils import publish from utils import Check
def check_smtp(port): class CheckSMTP(Check):
def __init__(self, port):
self._port = port
def _check(self):
"""Check the SMTP port.""" """Check the SMTP port."""
try: try:
smtp = SMTP("smtp.shore.co.il", port) smtp = SMTP("smtp.shore.co.il", self._port)
ehlo = smtp.ehlo() ehlo = smtp.ehlo()
if ehlo[0] != 250 or "LOGIN" in ehlo[1].decode().split(): if ehlo[0] != 250 or "LOGIN" in ehlo[1].decode().split():
return [False, f"First EHLO on port {port} failed."] return [False, f"First EHLO on port {self._port} failed."]
if smtp.starttls() != (220, b"TLS go ahead"): if smtp.starttls() != (220, b"TLS go ahead"):
return [False, f"STARTTLS on port {port} failed."] return [False, f"STARTTLS on port {self._port} failed."]
ehlo = smtp.ehlo() ehlo = smtp.ehlo()
if ehlo[0] != 250 or "LOGIN" not in ehlo[1].decode().split(): if ehlo[0] != 250 or "LOGIN" not in ehlo[1].decode().split():
return [False, f"Second EHLO on port {port} failed."] return [False, f"Second EHLO on port {self._port} failed."]
smtp.close() smtp.close()
except Exception as e: # pylint: disable=broad-except,invalid-name except Exception as e: # pylint: disable=broad-except,invalid-name
print(str(e)) print(str(e))
return [False, f"SMTP failure on port {port}."] return [False, f"SMTP failure on port {self._port}."]
return [True, f"SMTP on port {port} is OK."] return [True, f"SMTP on port {self._port} is OK."]
def handler(event, context): # pylint: disable=unused-argument def handler(event, context): # pylint: disable=unused-argument
...@@ -28,10 +32,7 @@ def handler(event, context): # pylint: disable=unused-argument ...@@ -28,10 +32,7 @@ def handler(event, context): # pylint: disable=unused-argument
# raise the cost. For now assume that if the submission port is working, # raise the cost. For now assume that if the submission port is working,
# the smtp port is working too (same gateway, same host, same processes). # the smtp port is working too (same gateway, same host, same processes).
for port in [587]: for port in [587]:
success, message = check_smtp(port) CheckSMTP(port).run()
print(message)
if not success:
publish(message)
if __name__ == "__main__": if __name__ == "__main__":
......
import socket import socket
from utils import publish from utils import Check
def check_ssh(host, port=22): class CheckSSH(Check):
def __init__(self, hostname, port=22):
self._hostname = hostname
self._port = port
def _check(self):
"""Check that an SSH server is available on that host and port.""" """Check that an SSH server is available on that host and port."""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try: try:
sock.connect((host, port)) sock.connect((self._hostname, self._port))
msg = sock.recv(1024) msg = sock.recv(1024)
sock.close() sock.close()
return msg.startswith(b"SSH-2.0-OpenSSH") if msg.startswith(b"SSH-2.0-OpenSSH"):
return True, f"SSH on {self._hostname}:{self._port} is OK."
return False, f"SSH on {self._hostname}:{self._port} failed."
except Exception as e: # pylint: disable=broad-except,invalid-name except Exception as e: # pylint: disable=broad-except,invalid-name
print(str(e)) return False, str(e)
return False
def handler(event, context): # pylint: disable=unused-argument def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler.""" """Lambda event handler."""
for host in ["ns1.shore.co.il", "ns4.shore.co.il"]: for host in ["ns1.shore.co.il", "ns4.shore.co.il"]:
if check_ssh(host): CheckSSH(host).run()
print(f"SSH on {host} is OK.")
else:
message = f"SSH on {host} failed."
print(message)
publish(message)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://transmission.shore.co.il/", "codes": [301, 302]}, {"url": "http://transmission.shore.co.il/", "codes": [301, 302]},
{"url": "https://transmission.shore.co.il/", "codes": [401]}, {"url": "https://transmission.shore.co.il/", "codes": [401]},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -3,35 +3,65 @@ import boto3 # pylint: disable=import-error ...@@ -3,35 +3,65 @@ import boto3 # pylint: disable=import-error
import requests # pylint: disable=import-error import requests # pylint: disable=import-error
TOPIC_ARN = os.getenv("TOPIC_ARN") class Check:
_topic_arn = os.getenv("TOPIC_ARN")
_retries = int(os.getenv("RETRIES", "2"))
def run(self):
"""Run the check function with retries. Alert if failed."""
for _ in range(self._retries):
success, message = self._check()
print(message)
if success:
break
else:
self.publish(message)
return success
def _check(self):
"""The actual check."""
raise NotImplementedError
def publish(message): def publish(self, message):
"""Publish an SNS message.""" """Publish an SNS message."""
if self._topic_arn is None:
print(f"Publish: {message}")
else:
client = boto3.client("sns") client = boto3.client("sns")
client.publish(TopicArn=TOPIC_ARN, Message=message) client.publish(TopicArn=self._topic_arn, Message=message)
class CheckURL(Check):
_method = "GET"
_valid_codes = (200,)
def check_url(url, method="GET", valid_codes=(200)): def __init__(self, url, method=None, codes=None):
"""Checks validaty of a URL. """Initialize the check object."""
self._url = url
if method is not None:
self._method = method
if codes is not None:
self._valid_codes = codes
Allows specifying the HTTP method and a list of valid codes.""" def _check(self):
"""Checks validaty of a URL."""
try: try:
response = requests.request(method, url, allow_redirects=False) response = requests.request(
return response.status_code in valid_codes self._method, self._url, allow_redirects=False
)
return (
response.status_code in self._valid_codes,
f"{self._url} is OK.",
)
except Exception as e: # pylint: disable=broad-except,invalid-name except Exception as e: # pylint: disable=broad-except,invalid-name
print(str(e)) return False, str(e)
return False
def check_urls(checks): def website_handler(urls): # pylint: disable=unused-argument
"""Check a list of URLs.""" """Return a Lambda event handler for checking web sites."""
for check in checks:
if check_url( def _handler(event, context): # pylint: disable=unused-argument
check["url"], check.get("method", "GET"), check.get("codes", [200]) for url in urls:
): CheckURL(url["url"], url.get("method"), url.get("codes")).run()
print(f"{check['url']} is OK.")
else: return _handler
message = f"Failed check for {check['url']}."
print(message)
publish(message)
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://vouch.shore.co.il/", "codes": [301, 302]}, {"url": "http://vouch.shore.co.il/", "codes": [301, 302]},
{"url": "https://vouch.shore.co.il/validate", "codes": [401]}, {"url": "https://vouch.shore.co.il/validate", "codes": [401]},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
from utils import check_urls from utils import website_handler
URLS = [
def handler(event, context): # pylint: disable=unused-argument
"""Lambda event handler."""
checks = [
{"url": "http://shore.co.il/", "codes": [301, 302]}, {"url": "http://shore.co.il/", "codes": [301, 302]},
{"url": "http://www.shore.co.il/", "codes": [301, 302]}, {"url": "http://www.shore.co.il/", "codes": [301, 302]},
{"url": "https://shore.co.il/", "codes": [301, 302]}, {"url": "https://shore.co.il/", "codes": [301, 302]},
{"url": "https://www.shore.co.il/blog"}, {"url": "https://www.shore.co.il/blog"},
] ]
check_urls(checks)
handler = website_handler(URLS)
if __name__ == "__main__": if __name__ == "__main__":
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment