run status checks each night and email the administrator with the changes from...

run status checks each night and email the administrator with the changes from the previous day's results

management/daemon.py

@@ -324,7 +324,7 @@ def system_status():
 		def print_line(self, message, monospace=False):
 			self.items[-1]["extra"].append({ "text": message, "monospace": monospace })
 	output = WebOutput()
-	run_checks(env, output, pool)
+	run_checks(False, env, output, pool)
 	return json_response(output.items)
 
 @app.route('/system/updates')

management/status_checks.py

@@ -6,7 +6,7 @@
 
 __ALL__ = ['check_certificate']
 
-import os, os.path, re, subprocess, datetime, multiprocessing.pool
+import sys, os, os.path, re, subprocess, datetime, multiprocessing.pool
 
 import dns.reversename, dns.resolver
 import dateutil.parser, dateutil.tz
@@ -17,7 +17,7 @@ from mailconfig import get_mail_domains, get_mail_aliases
 
 from utils import shell, sort_domains, load_env_vars_from_file
 
-def run_checks(env, output, pool):
+def run_checks(rounded_values, env, output, pool):
 	# run systems checks
 	output.add_heading("System")
 
@@ -33,12 +33,12 @@ def run_checks(env, output, pool):
 	# that in run_services checks.)
 	shell('check_call', ["/usr/sbin/rndc", "flush"], trap=True)
 	
-	run_system_checks(env, output)
+	run_system_checks(rounded_values, env, output)
 
 	# perform other checks asynchronously
 
 	run_network_checks(env, output)
-	run_domain_checks(env, output, pool)
+	run_domain_checks(rounded_values, env, output, pool)
 
 def get_ssh_port():
     # Returns ssh port
@@ -133,11 +133,11 @@ def check_service(i, service, env):
 
 	return (i, running, fatal, output)
 
-def run_system_checks(env, output):
+def run_system_checks(rounded_values, env, output):
 	check_ssh_password(env, output)
 	check_software_updates(env, output)
 	check_system_aliases(env, output)
-	check_free_disk_space(env, output)
+	check_free_disk_space(rounded_values, env, output)
 
 def check_ssh_password(env, output):
 	# Check that SSH login with password is disabled. The openssh-server
@@ -172,12 +172,15 @@ def check_system_aliases(env, output):
 	# admin email is automatically directed.
 	check_alias_exists("administrator@" + env['PRIMARY_HOSTNAME'], env, output)
 
-def check_free_disk_space(env, output):
+def check_free_disk_space(rounded_values, env, output):
 	# Check free disk space.
 	st = os.statvfs(env['STORAGE_ROOT'])
 	bytes_total = st.f_blocks * st.f_frsize
 	bytes_free = st.f_bavail * st.f_frsize
-	disk_msg = "The disk has %s GB space remaining." % str(round(bytes_free/1024.0/1024.0/1024.0*10.0)/10.0)
+	if not rounded_values:
+		disk_msg = "The disk has %s GB space remaining." % str(round(bytes_free/1024.0/1024.0/1024.0*10.0)/10)
+	else:
+		disk_msg = "The disk has less than %s%% space left." % str(round(bytes_free/bytes_total/10 + .5)*10)
 	if bytes_free > .3 * bytes_total:
 		output.print_ok(disk_msg)
 	elif bytes_free > .15 * bytes_total:
@@ -215,7 +218,7 @@ def run_network_checks(env, output):
 			which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s."""
 			% (env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
 
-def run_domain_checks(env, output, pool):
+def run_domain_checks(rounded_time, env, output, pool):
 	# Get the list of domains we handle mail for.
 	mail_domains = get_mail_domains(env)
 
@@ -230,17 +233,17 @@ def run_domain_checks(env, output, pool):
 
 	# Serial version:
 	#for domain in sort_domains(domains_to_check, env):
-	#	run_domain_checks_on_domain(domain, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
+	#	run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
 
 	# Parallelize the checks across a worker pool.
-	args = ((domain, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
+	args = ((domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains)
 		for domain in domains_to_check)
 	ret = pool.starmap(run_domain_checks_on_domain, args, chunksize=1)
 	ret = dict(ret) # (domain, output) => { domain: output }
 	for domain in sort_domains(ret, env):
 		ret[domain].playback(output)
 
-def run_domain_checks_on_domain(domain, env, dns_domains, dns_zonefiles, mail_domains, web_domains):
+def run_domain_checks_on_domain(domain, rounded_time, env, dns_domains, dns_zonefiles, mail_domains, web_domains):
 	output = BufferedOutput()
 
 	output.add_heading(domain)
@@ -255,7 +258,7 @@ def run_domain_checks_on_domain(domain, env, dns_domains, dns_zonefiles, mail_do
 		check_mail_domain(domain, env, output)
 
 	if domain in web_domains:
-		check_web_domain(domain, env, output)
+		check_web_domain(domain, rounded_time, env, output)
 
 	if domain in dns_domains:
 		check_dns_zone_suggestions(domain, env, output, dns_zonefiles)
@@ -472,7 +475,7 @@ def check_mail_domain(domain, env, output):
 			which may prevent recipients from receiving your mail.
 			See http://www.spamhaus.org/dbl/ and http://www.spamhaus.org/query/domain/%s.""" % (dbl, domain))
 
-def check_web_domain(domain, env, output):
+def check_web_domain(domain, rounded_time, env, output):
 	# See if the domain's A record resolves to our PUBLIC_IP. This is already checked
 	# for PRIMARY_HOSTNAME, for which it is required for mail specifically. For it and
 	# other domains, it is required to access its website.
@@ -488,7 +491,7 @@ def check_web_domain(domain, env, output):
 	# We need a SSL certificate for PRIMARY_HOSTNAME because that's where the
 	# user will log in with IMAP or webmail. Any other domain we serve a
 	# website for also needs a signed certificate.
-	check_ssl_cert(domain, env, output)
+	check_ssl_cert(domain, rounded_time, env, output)
 
 def query_dns(qname, rtype, nxdomain='[Not Set]'):
 	# Make the qname absolute by appending a period. Without this, dns.resolver.query
@@ -515,7 +518,7 @@ def query_dns(qname, rtype, nxdomain='[Not Set]'):
 	# can compare to a well known order.
 	return "; ".join(sorted(str(r).rstrip('.') for r in response))
 
-def check_ssl_cert(domain, env, output):
+def check_ssl_cert(domain, rounded_time, env, output):
 	# Check that SSL certificate is signed.
 
 	# Skip the check if the A record is not pointed here.
@@ -530,7 +533,7 @@ def check_ssl_cert(domain, env, output):
 
 	# Check that the certificate is good.
 
-	cert_status, cert_status_details = check_certificate(domain, ssl_certificate, ssl_key)
+	cert_status, cert_status_details = check_certificate(domain, ssl_certificate, ssl_key, rounded_time=rounded_time)
 
 	if cert_status == "OK":
 		# The certificate is ok. The details has expiry info.
@@ -569,7 +572,7 @@ def check_ssl_cert(domain, env, output):
 			output.print_line(cert_status_details)
 			output.print_line("")
 
-def check_certificate(domain, ssl_certificate, ssl_private_key):
+def check_certificate(domain, ssl_certificate, ssl_private_key, rounded_time=False):
 	# Use openssl verify to check the status of a certificate.
 
 	# First check that the certificate is for the right domain. The domain
@@ -680,7 +683,15 @@ def check_certificate(domain, ssl_certificate, ssl_private_key):
 		# But is it expiring soon?
 		now = datetime.datetime.now(dateutil.tz.tzlocal())
 		ndays = (cert_expiration_date-now).days
-		expiry_info = "The certificate expires in %d days on %s." % (ndays, cert_expiration_date.strftime("%x"))
+		if not rounded_time or ndays < 7:
+			expiry_info = "The certificate expires in %d days on %s." % (ndays, cert_expiration_date.strftime("%x"))
+		elif ndays <= 14:
+			expiry_info = "The certificate expires in less than two weeks, on %s." % cert_expiration_date.strftime("%x")
+		elif ndays <= 31:
+			expiry_info = "The certificate expires in less than a month, on %s." % cert_expiration_date.strftime("%x")
+		else:
+			expiry_info = "The certificate expires on %s." % cert_expiration_date.strftime("%x")
+
 		if ndays <= 31:
 			return ("The certificate is expiring soon: " + expiry_info, None)
 
@@ -721,17 +732,109 @@ def list_apt_updates(apt_update=True):
 
 	return pkgs
 
+def run_and_output_changes(env, pool, send_via_email):
+	import json
+	from difflib import SequenceMatcher
 
-class ConsoleOutput:
-	try:
-		terminal_columns = int(shell('check_output', ['stty', 'size']).split()[1])
-	except:
-		terminal_columns = 76
+	if not send_via_email:
+		out = ConsoleOutput()
+	else:
+		import io
+		out = FileOutput(io.StringIO(""), 70)
+
+	# Run status checks.
+	cur = BufferedOutput()
+	run_checks(True, env, cur, pool)
+
+	# Load previously saved status checks.
+	cache_fn = "/var/cache/mailinabox/status_checks.json"
+	if os.path.exists(cache_fn):
+		prev = json.load(open(cache_fn))
+
+		# Group the serial output into categories by the headings.
+		def group_by_heading(lines):
+			from collections import OrderedDict
+			ret = OrderedDict()
+			k = []
+			ret["No Category"] = k
+			for line_type, line_args, line_kwargs in lines:
+				if line_type == "add_heading":
+					k = []
+					ret[line_args[0]] = k
+				else:
+					k.append((line_type, line_args, line_kwargs))
+			return ret
+		prev_status = group_by_heading(prev)
+		cur_status = group_by_heading(cur.buf)
+
+		# Compare the previous to the current status checks
+		# category by category.
+		for category, cur_lines in cur_status.items():
+			if category not in prev_status:
+				out.add_heading(category + " -- Added")
+				BufferedOutput(with_lines=cur_lines).playback(out)
+			else:
+				# Actual comparison starts here...
+				prev_lines = prev_status[category]
+				def stringify(lines):
+					return [json.dumps(line) for line in lines]
+				diff = SequenceMatcher(None, stringify(prev_lines), stringify(cur_lines)).get_opcodes()
+				for op, i1, i2, j1, j2 in diff:
+					if op == "replace":
+						out.add_heading(category + " -- Previously:")
+					elif op == "delete":
+						out.add_heading(category + " -- Removed")
+					if op in ("replace", "delete"):
+						BufferedOutput(with_lines=prev_lines[i1:i2]).playback(out)
+
+					if op == "replace":
+						out.add_heading(category + " -- Currently:")
+					elif op == "insert":
+						out.add_heading(category + " -- Added")
+					if op in ("replace", "insert"):
+						BufferedOutput(with_lines=cur_lines[j1:j2]).playback(out)
+
+		for category, prev_lines in prev_status.items():
+			if category not in cur_status:
+				out.add_heading(category)
+				out.add_warning("Removed.")
+	
+	if send_via_email:
+		# If there were changes, send off an email.
+		buf = out.buf.getvalue()
+		if len(buf) > 0:
+			# create MIME message
+			from email.message import Message
+			msg = Message()
+			msg['From'] = "\"%s\" <administrator@%s>" % (env['PRIMARY_HOSTNAME'], env['PRIMARY_HOSTNAME'])
+			msg['To'] = "administrator@%s" % env['PRIMARY_HOSTNAME']
+			msg['Subject'] = "[%s] Status Checks Change Notice" % env['PRIMARY_HOSTNAME']
+			msg.set_payload(buf, "UTF-8")
+	
+			# send to administrator@
+			import smtplib
+			mailserver = smtplib.SMTP('localhost', 25)
+			mailserver.ehlo()
+			mailserver.sendmail(
+				"administrator@%s" % env['PRIMARY_HOSTNAME'], # MAIL FROM
+				"administrator@%s" % env['PRIMARY_HOSTNAME'], # RCPT TO
+				msg.as_string())
+			mailserver.quit()
+		
+	# Store the current status checks output for next time.
+	os.makedirs(os.path.dirname(cache_fn), exist_ok=True)
+	with open(cache_fn, "w") as f:
+		json.dump(cur.buf, f, indent=True)
+
+class FileOutput:
+	def __init__(self, buf, width):
+		self.buf = buf
+		self.width = width
 
 	def add_heading(self, heading):
-		print()
-		print(heading)
-		print("=" * len(heading))
+		print(file=self.buf)
+		print(heading, file=self.buf)
+		print("=" * len(heading), file=self.buf)
 
 	def print_ok(self, message):
 		self.print_block(message, first_line="✓  ")
@@ -743,28 +846,36 @@ class ConsoleOutput:
 		self.print_block(message, first_line="?  ")
 
 	def print_block(self, message, first_line="   "):
-		print(first_line, end='')
+		print(first_line, end='', file=self.buf)
 		message = re.sub("\n\s*", " ", message)
 		words = re.split("(\s+)", message)
 		linelen = 0
 		for w in words:
-			if linelen + len(w) > self.terminal_columns-1-len(first_line):
-				print()
-				print("   ", end="")
+			if linelen + len(w) > self.width-1-len(first_line):
+				print(file=self.buf)
+				print("   ", end="", file=self.buf)
 				linelen = 0
 			if linelen == 0 and w.strip() == "": continue
-			print(w, end="")
+			print(w, end="", file=self.buf)
 			linelen += len(w)
-		print()
+		print(file=self.buf)
 
 	def print_line(self, message, monospace=False):
 		for line in message.split("\n"):
 			self.print_block(line)
 
+class ConsoleOutput(FileOutput):
+	def __init__(self):
+		self.buf = sys.stdout
+		try:
+			self.width = int(shell('check_output', ['stty', 'size']).split()[1])
+		except:
+			self.width = 76
+
 class BufferedOutput:
 	# Record all of the instance method calls so we can play them back later.
-	def __init__(self):
-		self.buf = []
+	def __init__(self, with_lines=None):
+		self.buf = [] if not with_lines else with_lines
 	def __getattr__(self, attr):
 		if attr not in ("add_heading", "print_ok", "print_error", "print_warning", "print_block", "print_line"):
 			raise AttributeError
@@ -778,12 +889,17 @@ class BufferedOutput:
 
 
 if __name__ == "__main__":
-	import sys
 	from utils import load_environment
+
 	env = load_environment()
+	pool = multiprocessing.pool.Pool(processes=10)
+
 	if len(sys.argv) == 1:
-		pool = multiprocessing.pool.Pool(processes=10)
-		run_checks(env, ConsoleOutput(), pool)
+		run_checks(False, env, ConsoleOutput(), pool)
+
+	elif sys.argv[1] == "--show-changes":
+		run_and_output_changes(env, pool, sys.argv[-1] == "--smtp")
+
 	elif sys.argv[1] == "--check-primary-hostname":
 		# See if the primary hostname appears resolvable and has a signed certificate.
 		domain = env['PRIMARY_HOSTNAME']
@@ -796,5 +912,3 @@ if __name__ == "__main__":
 		if cert_status != "OK":
 			sys.exit(1)
 		sys.exit(0)
-
-

setup/management.sh

@@ -30,6 +30,17 @@ $(pwd)/management/backup.py
 EOF
 chmod +x /etc/cron.daily/mailinabox-backup
 
+# Perform daily status checks. Compare each day to the previous
+# for changes and mail the changes to the administrator.
+cat > /etc/cron.daily/mailinabox-statuschecks << EOF;
+#!/bin/bash
+# Mail-in-a-Box --- Do not edit / will be overwritten on update.
+# Run status checks.
+$(pwd)/management/status_checks.py --show-changes --smtp
+EOF
+chmod +x /etc/cron.daily/mailinabox-statuschecks
+
+
 # Start it. Remove the api key file first so that start.sh
 # can wait for it to be created to know that the management
 # server is ready.