first pass at making readable documentation by parsing the bash scripts

9d40a12f · Joshua Tauberer · c2ddabe6 · 9d40a12f · 9d40a12f · 9d40a12f
Commit 9d40a12f authored Sep 21, 2014 by Joshua Tauberer
7 changed files
--- a/setup/dns.sh
+++ b/setup/dns.sh
 #!/bin/bash
-# DNS: Configure a DNS server using nsd
+# DNS: Configure a DNS server to host our own DNS
-#######################################
+# -----------------------------------------------
 # This script installs packages, but the DNS zone files are only
 # created by the /dns/update API in the management server because
@@ -9,23 +9,23 @@
 source setup/functions.sh # load our functions
-# Install nsd, our DNS server software, and ldnsutils which helps
+# Install `nsd`, our DNS server software, and `ldnsutils` which helps
 # us sign zones for DNSSEC.
 # ...but first, we have to create the user because the 
 # current Ubuntu forgets to do so in the .deb
 # see issue #25 and https://bugs.launchpad.net/ubuntu/+source/nsd/+bug/1311886
 if id nsd > /dev/null 2>&1; then
-	true; #echo "nsd user exists... good";
+	true; #echo "nsd user exists... good"; #NODOC
 else
 	useradd nsd;
 fi
 # Okay now install the packages.
 #
-# nsd: The non-recursive nameserver that publishes our DNS records.
+# * nsd: The non-recursive nameserver that publishes our DNS records.
-# ldnsutils: Helper utilities for signing DNSSEC zones.
+# * ldnsutils: Helper utilities for signing DNSSEC zones.
-# openssh-client: Provides ssh-keyscan which we use to create SSHFP records.
+# * openssh-client: Provides ssh-keyscan which we use to create SSHFP records.
 apt_install nsd ldnsutils openssh-client
@@ -53,9 +53,10 @@ if [ ! -f "$STORAGE_ROOT/dns/dnssec/keys.conf" ]; then
 	ZSK=$(umask 077; cd $STORAGE_ROOT/dns/dnssec; ldns-keygen -a RSASHA1-NSEC3-SHA1 -b 1024 _domain_);
 	# These generate two sets of files like:
-	# K_domain_.+007+08882.ds <- DS record for adding to NSD configuration files
+	#
-	# K_domain_.+007+08882.key <- public key (goes into DS record & upstream DNS provider like your registrar)
+	# * `K_domain_.+007+08882.ds`: DS record to provide to domain name registrar
-	# K_domain_.+007+08882.private  <- private key (secret!)
+	# * `K_domain_.+007+08882.key`: public key (goes into DS record & upstream DNS provider like your registrar)
+	# * `K_domain_.+007+08882.private`: private key (secret!)
 	# The filenames are unpredictable and encode the key generation
 	# options. So we'll store the names of the files we just generated.

--- a/setup/mail-dovecot.sh
+++ b/setup/mail-dovecot.sh
 #!/bin/bash
 #
 # Dovecot (IMAP and LDA)
+# ----------------------
 #
 # Dovecot is *both* the IMAP server (the protocol that email applications
 # use to query a mailbox) as well as the local delivery agent (LDA),
@@ -17,13 +18,13 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
-# Install packages.
+# ### Install packages and basic setup
 apt_install \
 	dovecot-core dovecot-imapd dovecot-lmtpd dovecot-sqlite sqlite3 \
 	dovecot-sieve dovecot-managesieved
-# The dovecot-imapd dovecot-lmtpd packages automatically enable IMAP and LMTP protocols.
+# The dovecot-imapd and dovecot-lmtpd packages automatically enable IMAP and LMTP protocols.
 # Set the location where we'll store user mailboxes.
 tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
@@ -31,7 +32,7 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
 	mail_privileged_group=mail \
 	first_valid_uid=0
-# IMAP
+# ### IMAP
 # Require that passwords are sent over SSL only, and allow the usual IMAP authentication mechanisms.
 # The LOGIN mechanism is supposedly for Microsoft products like Outlook to do SMTP login (I guess
@@ -62,7 +63,7 @@ sed -i "s/#port = 110/port = 0/" /etc/dovecot/conf.d/10-master.conf
 tools/editconf.py /etc/dovecot/conf.d/20-imap.conf \
 	imap_idle_notify_interval="4 mins"
-# LDA (LMTP)
+# ### LDA (LMTP)
 # Enable Dovecot's LDA service with the LMTP protocol. It will listen
 # in port 10026, and Spamassassin will be configured to pass mail there.
@@ -94,12 +95,12 @@ EOF
 tools/editconf.py /etc/dovecot/conf.d/15-lda.conf \
 	postmaster_address=postmaster@$PRIMARY_HOSTNAME
-# SIEVE
+# ### Sieve
 # Enable the Dovecot sieve plugin which let's users run scripts that process
 # mail as it comes in. We'll also set a global script that moves mail marked
 # as spam by Spamassassin into the user's Spam folder.
-sudo sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins sieve/" /etc/dovecot/conf.d/20-lmtp.conf
+sed -i "s/#mail_plugins = .*/mail_plugins = \$mail_plugins sieve/" /etc/dovecot/conf.d/20-lmtp.conf
 cat > /etc/dovecot/conf.d/99-local-sieve.conf << EOF;
 plugin {

--- a/setup/mail-postfix.sh
+++ b/setup/mail-postfix.sh
 #!/bin/bash
 #
 # Postfix (SMTP)
+# --------------
 #
 # Postfix handles the transmission of email between servers
 # using the SMTP protocol. It is a Mail Transfer Agent (MTA).
@@ -29,11 +30,11 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
-# Install packages.
+# ### Install packages.
 apt_install postfix postgrey postfix-pcre ca-certificates
-# Basic Settings
+# ### Basic Settings
 # Have postfix listen on all network interfaces, set our name (the Debian default seems to be localhost),
 # and set the name of the local machine to localhost for xxx@localhost mail (but I don't think this will have any effect because
@@ -44,15 +45,16 @@ tools/editconf.py /etc/postfix/main.cf \
 	smtpd_banner="\$myhostname ESMTP Hi, I'm a Mail-in-a-Box (Ubuntu/Postfix; see https://mailinabox.email/)" \
 	mydestination=localhost
-# Outgoing Mail
+# ### Outgoing Mail
 # Enable the 'submission' port 587 smtpd server and tweak its settings.
-# a) Require the best ciphers for incoming connections per http://baldric.net/2013/12/07/tls-ciphers-in-postfix-and-dovecot/.
+#
-#    but without affecting opportunistic TLS on incoming mail, which will allow any cipher (it's better than none).
+# * Require the best ciphers for incoming connections per http://baldric.net/2013/12/07/tls-ciphers-in-postfix-and-dovecot/.
-# b) Give it a different name in syslog to distinguish it from the port 25 smtpd server.
+#   but without affecting opportunistic TLS on incoming mail, which will allow any cipher (it's better than none).
-# c) Add a new cleanup service specific to the submission service ('authclean')
+# * Give it a different name in syslog to distinguish it from the port 25 smtpd server.
-#    that filters out privacy-sensitive headers on mail being sent out by
+# * Add a new cleanup service specific to the submission service ('authclean')
-#    authenticated users.
+#   that filters out privacy-sensitive headers on mail being sent out by
+#   authenticated users.
 tools/editconf.py /etc/postfix/master.cf -s -w \
 	"submission=inet n       -       -       -       -       smtpd
 	  -o syslog_name=postfix/submission
@@ -64,7 +66,7 @@ tools/editconf.py /etc/postfix/master.cf -s -w \
 # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service.
 cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters
-# Enable TLS on incoming connections (i.e. ports 25 *and* 587) and
+# Enable TLS on these and all other connections (i.e. ports 25 *and* 587) and
 # require TLS before a user is allowed to authenticate. This also makes
 # opportunistic TLS available on *incoming* mail.
 tools/editconf.py /etc/postfix/main.cf \
@@ -74,6 +76,19 @@ tools/editconf.py /etc/postfix/main.cf \
 	smtpd_tls_key_file=$STORAGE_ROOT/ssl/ssl_private_key.pem \
 	smtpd_tls_received_header=yes
+# Prevent non-authenticated users from sending mail that requires being
+# relayed elsewhere. We don't want to be an "open relay". On outbound
+# mail, require one of:
+#
+# * permit_sasl_authenticated: Authenticated users (i.e. on port 587).
+# * permit_mynetworks: Mail that originates locally.
+# * reject_unauth_destination: No one else. (Permits mail whose destination is local and rejects other mail.)
+tools/editconf.py /etc/postfix/main.cf \
+	smtpd_relay_restrictions=permit_sasl_authenticated,permit_mynetworks,reject_unauth_destination
+# ### DANE
+#
 # When connecting to remote SMTP servers, prefer TLS and use DANE if available.
 #
 # Prefering ("opportunistic") TLS means Postfix will accept whatever SSL certificate the remote
@@ -98,38 +113,27 @@ tools/editconf.py /etc/postfix/main.cf \
 	smtp_tls_CAfile=/etc/ssl/certs/ca-certificates.crt \
 	smtp_tls_loglevel=2
-# Incoming Mail
+# ### Incoming Mail
 # Pass any incoming mail over to a local delivery agent. Spamassassin
 # will act as the LDA agent at first. It is listening on port 10025
 # with LMTP. Spamassassin will pass the mail over to Dovecot after.
 #
-# In a basic setup we would pass mail directly to Dovecot like so:
+# In a basic setup we would pass mail directly to Dovecot by setting
-# tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:unix:private/dovecot-lmtp
+# virtual_transport to `lmtp:unix:private/dovecot-lmtp`.
 #
 tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025
-# Who can send outbound mail? The purpose of this is to prevent
-# non-authenticated users from sending mail that requires being
-# relayed elsewhere. We don't want to be an "open relay".
-#
-# permit_sasl_authenticated: Authenticated users (i.e. on port 587).
-# permit_mynetworks: Mail that originates locally.
-# reject_unauth_destination: No one else. (Permits mail whose destination is local and rejects other mail.)
-tools/editconf.py /etc/postfix/main.cf \
-	smtpd_relay_restrictions=permit_sasl_authenticated,permit_mynetworks,reject_unauth_destination
 # Who can send mail to us? Some basic filters.
 #
-# reject_non_fqdn_sender: Reject not-nice-looking return paths.
+# * reject_non_fqdn_sender: Reject not-nice-looking return paths.
-# reject_unknown_sender_domain: Reject return paths with invalid domains.
+# * reject_unknown_sender_domain: Reject return paths with invalid domains.
-# reject_rhsbl_sender: Reject return paths that use blacklisted domains.
+# * reject_rhsbl_sender: Reject return paths that use blacklisted domains.
-#
+# * permit_sasl_authenticated: Authenticated users (i.e. on port 587) can skip further checks.
-# permit_sasl_authenticated: Authenticated users (i.e. on port 587) can skip further checks.
+# * permit_mynetworks: Mail that originates locally can skip further checks.
-# permit_mynetworks: Mail that originates locally can skip further checks.
+# * reject_rbl_client: Reject connections from IP addresses blacklisted in zen.spamhaus.org
-# reject_rbl_client: Reject connections from IP addresses blacklisted in zen.spamhaus.org
+# * reject_unlisted_recipient: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after.
-# reject_unlisted_recipient: Although Postfix will reject mail to unknown recipients, it's nicer to reject such mail ahead of greylisting rather than after.
+# * check_policy_service: Apply greylisting using postgrey.
-# check_policy_service: Apply greylisting using postgrey.
 #
 # Notes:
 # permit_dnswl_client can pass through mail from whitelisted IP addresses, which would be good to put before greylisting

--- a/setup/mail-users.sh
+++ b/setup/mail-users.sh
 #!/bin/bash
 #
 # User Authentication and Destination Validation
+# ----------------------------------------------
 #
 # This script configures user authentication for Dovecot
 # and Postfix (which relies on Dovecot) and destination
@@ -9,6 +10,8 @@
 source setup/functions.sh # load our functions
 source /etc/mailinabox.conf # load global vars
+# ### User and Alias Database
 # The database of mail users (i.e. authenticated users, who have mailboxes)
 # and aliases (forwarders).
@@ -21,8 +24,7 @@ if [ ! -f $db_path ]; then
 	echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL);" | sqlite3 $db_path;
 fi
-# User Authentication
+# ### User Authentication
-#####################
 # Have Dovecot query our database, and not system users, for authentication.
 sed -i "s/#*\(\!include auth-system.conf.ext\)/#\1/"  /etc/dovecot/conf.d/10-auth.conf
@@ -68,8 +70,7 @@ tools/editconf.py /etc/postfix/main.cf \
 	smtpd_sasl_path=private/auth \
 	smtpd_sasl_auth_enable=yes
-# Destination Validation
+# ### Destination Validation
-########################
 # Use a Sqlite3 database to check whether a destination email address exists,
 # and to perform any email alias rewrites in Postfix.

--- a/setup/ssl.sh
+++ b/setup/ssl.sh
 #!/bin/bash
 #
 # SSL Certificate
+# ---------------
 #
 # Create a self-signed SSL certificate if one has not yet been created.
 #
@@ -21,20 +22,22 @@ source /etc/mailinabox.conf # load global vars
 apt_install openssl
 mkdir -p $STORAGE_ROOT/ssl
+# Generate a new private key if one doesn't already exist.
+# Set the umask so the key file is not world-readable.
 if [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then
-	# Generate a new private key if one doesn't already exist.
-	# Set the umask so the key file is not world-readable.
 	(umask 077; hide_output \
 		openssl genrsa -out $STORAGE_ROOT/ssl/ssl_private_key.pem 2048)
 fi
+# Generate a certificate signing request if one doesn't already exist.
 if [ ! -f $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr ]; then
-	# Generate a certificate signing request if one doesn't already exist.
 	hide_output \
 	openssl req -new -key $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr \
 	  -sha256 -subj "/C=$CSR_COUNTRY/ST=/L=/O=/CN=$PRIMARY_HOSTNAME"
 fi
+# Generate a SSL certificate by self-signing if a SSL certificate doesn't yet exist.
 if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then
-	# Generate a SSL certificate by self-signing if a SSL certificate doesn't yet exist.
 	hide_output \
 	openssl x509 -req -days 365 \
 	  -in $STORAGE_ROOT/ssl/ssl_cert_sign_req.csr -signkey $STORAGE_ROOT/ssl/ssl_private_key.pem -out $STORAGE_ROOT/ssl/ssl_certificate.pem

--- a/setup/system.sh
+++ b/setup/system.sh
 source setup/functions.sh # load our functions
-# Base system configuration.
+# Base system configuration
+# -------------------------
+# ### Base packages
+# Update system packages:
 echo Updating system packages...
 hide_output apt-get update
@@ -8,12 +13,12 @@ hide_output apt-get -y upgrade
 # Install basic utilities.
 #
-#	haveged: Provides extra entropy to /dev/random so it doesn't stall
+# * haveged: Provides extra entropy to /dev/random so it doesn't stall
 #	         when generating random numbers for private keys (e.g. during
 #	         ldns-keygen).
-#	unattended-upgrades: Apt tool to install security updates automatically.
+# * unattended-upgrades: Apt tool to install security updates automatically.
-#   	ntp: keeps the system time correct
+# * ntp: keeps the system time correct
-#   	fail2ban: scans log files for repeated failed login attempts and blocks the remote IP at the firewall
+# * fail2ban: scans log files for repeated failed login attempts and blocks the remote IP at the firewall
 apt_install python3 python3-dev python3-pip \
 	wget curl \
@@ -28,16 +33,18 @@ APT::Periodic::Unattended-Upgrade "1";
 APT::Periodic::Verbose "1";
 EOF
-if [ -z "$DISABLE_FIREWALL" ]; then
+# ### Firewall
-	# Turn on the firewall. First allow incoming SSH, then turn on the firewall.
-	# Other ports will be opened at the point where we set up those services.
-	#
-	# Various virtualized environments like Docker and some VPSs don't provide
-	# a kernel that supports iptables. To avoid error-like output in these cases,
-	# let us disable the firewall.
+# Turn on the firewall.
+#
+# Various virtualized environments like Docker and some VPSs don't provide #NODOC
+# a kernel that supports iptables. To avoid error-like output in these cases, #NODOC
+# we skip this if the user sets DISABLE_FIREWALL=1. #NODOC
+if [ -z "$DISABLE_FIREWALL" ]; then
+	# Install `ufw` which provides a simple firewall configuration.
 	apt_install ufw
+	# Allow incoming connections to SSH.
 	ufw_allow ssh;
 	# ssh might be running on an alternate port. Use sshd -T to dump sshd's
@@ -45,34 +52,40 @@ if [ -z "$DISABLE_FIREWALL" ]; then
 	# too.
 	SSH_PORT=$(sshd -T 2>/dev/null | grep "^port " | sed "s/port //")
 	if [ ! -z "$SSH_PORT" ]; then
-		if [ "$SSH_PORT" != "22" ]; then
+	if [ "$SSH_PORT" != "22" ]; then
-			echo Opening alternate SSH port $SSH_PORT.
-			ufw_allow $SSH_PORT;
+	echo Opening alternate SSH port $SSH_PORT.
-		fi
+	ufw_allow $SSH_PORT;
+	fi
 	fi
 	ufw --force enable;
-fi
+fi #NODOC
-# Resolve DNS using bind9 locally, rather than whatever DNS server is supplied
+# ### Local DNS Service
-# by the machine's network configuration. We do this to ensure that DNS queries
+# Install a local DNS server, rather than using the DNS server provided by the
+# ISP's network configuration.
+#
+# We do this to ensure that DNS queries
 # that *we* make (i.e. looking up other external domains) perform DNSSEC checks.
 # We could use Google's Public DNS, but we don't want to create a dependency on
-# Google per our goals of decentralization. bind9, as packaged for Ubuntu, has
+# Google per our goals of decentralization. `bind9`, as packaged for Ubuntu, has
 # DNSSEC enabled by default via "dnssec-validation auto".
 #
-# So we'll be running bind9 bound to 127.0.0.1 for locally-issued DNS queries
+# So we'll be running `bind9` bound to 127.0.0.1 for locally-issued DNS queries
-# and nsd bound to the public ethernet interface for remote DNS queries asking
+# and `nsd` bound to the public ethernet interface for remote DNS queries asking
-# about our domain names. nsd is configured in dns.sh.
+# about our domain names. `nsd` is configured later.
 #
 # About the settings:
 #
-# * RESOLVCONF=yes will have bind9 take over /etc/resolv.conf to tell
+# * RESOLVCONF=yes will have `bind9` take over /etc/resolv.conf to tell
 #   local services that DNS queries are handled on localhost.
-# * Adding -4 to OPTIONS will have bind9 not listen on IPv6 addresses
+# * Adding -4 to OPTIONS will have `bind9` not listen on IPv6 addresses
 #   so that we're sure there's no conflict with nsd, our public domain
 #   name server, on IPV6.
-# * The listen-on directive in named.conf.options restricts bind9 to
+# * The listen-on directive in named.conf.options restricts `bind9` to
 #   binding to the loopback interface instead of all interfaces.
 apt_install bind9 resolvconf
 tools/editconf.py /etc/default/bind9 \
@@ -83,9 +96,11 @@ if ! grep -q "listen-on " /etc/bind/named.conf.options; then
 	sed -i "s/^}/\n\tlisten-on { 127.0.0.1; };\n}/" /etc/bind/named.conf.options
 fi
 if [ -f /etc/resolvconf/resolv.conf.d/original ]; then
-	echo "Archiving old resolv.conf (was /etc/resolvconf/resolv.conf.d/original, now /etc/resolvconf/resolv.conf.original)."
+	echo "Archiving old resolv.conf (was /etc/resolvconf/resolv.conf.d/original, now /etc/resolvconf/resolv.conf.original)." #NODOC
-	mv /etc/resolvconf/resolv.conf.d/original /etc/resolvconf/resolv.conf.original
+	mv /etc/resolvconf/resolv.conf.d/original /etc/resolvconf/resolv.conf.original #NODOC
 fi
+# Restart the DNS services.
 restart_service bind9
 restart_service resolvconf
--- a/tools/readable_bash.py
+++ b/tools/readable_bash.py
+#!/usr/bin/python3
+#
+# Generate documentation for how this machine works by
+# parsing our bash scripts!
+import cgi, re
+import markdown
+from modgrammar import *
+def generate_documentation():
+	print("""<!DOCTYPE html>
+<html>
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+        <meta name="viewport" content="width=device-width">
+        <title>Build Your Own Mail Server From Scratch</title>
+        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css">
+        <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css">
+        <style>
+		    @import url(https://fonts.googleapis.com/css?family=Iceland);
+		    @import url(https://fonts.googleapis.com/css?family=Raleway:400,700);
+			@import url(https://fonts.googleapis.com/css?family=Ubuntu:300,500);
+		   	body {
+		    		font-family: Raleway, sans-serif;
+		    		font-size: 16px;
+					color: #555;
+	    	}
+	    	h2, h3 {
+	    		margin-bottom: 1em;
+	    	}
+	    	p {
+	    		margin-bottom: 1em;
+	    	}
+	    	pre {
+	    		margin: 1em 1em 1.5em 1em;
+	    		color: black;
+	    	}
+	    	div.write-to {
+	    		margin: 1em;
+	    		border: 1px solid #999;
+	    	}
+	    	div.write-to p {
+	    		padding: .5em;
+	    		margin: 0;
+	    	}
+	    	div.write-to .filename {
+	    		background-color: #EEE;
+	    		padding: .5em;
+	    		font-weight: bold;
+	    	}
+	    	div.write-to pre {
+	    		padding: .5em;
+	    		margin: 0;
+	    	}
+        </style>
+    </head>
+    <body>
+    <div class="container">
+      <div class="row">
+        <div class="col-xs-12">
+        <h1>Build Your Own Mail Server From Scratch</h1>
+        <p>Here&rsquo;s how you can build your own mail server from scratch. This document is generated automatically from our setup script.</p>
+        <hr>
+ """)
+	parser = Source.parser()
+	for line in open("setup/start.sh"):
+		try:
+			fn = parser.parse_string(line).filename()
+		except:
+			continue
+		if fn in ("setup/preflight.sh", "setup/questions.sh", "setup/firstuser.sh", "setup/management.sh"):
+			continue
+		import sys
+		print(fn, file=sys.stderr)
+		print(BashScript.parse(fn))
+	print("""
+        <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.1/jquery.min.js"></script>
+        <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js"></script>
+    </body>
+</html>
+""")
+class HashBang(Grammar):
+	grammar = (L('#!'), REST_OF_LINE, EOL)
+	def value(self):
+		return ""
+def strip_indent(s):
+	lines = s.split("\n")
+	min_indent = min(len(re.match(r"\s*", line).group(0)) for line in lines if len(line) > 0)
+	lines = [line[min_indent:] for line in lines]
+	return "\n".join(lines)
+class Comment(Grammar):
+	grammar = ONE_OR_MORE(ZERO_OR_MORE(SPACE), L('#'), REST_OF_LINE, EOL)
+	def value(self):
+		if self.string.replace("#", "").strip() == "":
+			return "\n"
+		lines = [x[2].string for x in self[0]]
+		content = "\n".join(lines)
+		content = strip_indent(content)
+		return markdown.markdown(content, output_format="html4") + "\n\n"
+FILENAME = WORD('a-z0-9-/.')
+class Source(Grammar):
+	grammar = ((L('.') | L('source')), L(' '), FILENAME, Comment | EOL)
+	def filename(self):
+		return self[2].string.strip()
+	def value(self):
+		return BashScript.parse(self.filename())
+class CatEOF(Grammar):
+	grammar = (ZERO_OR_MORE(SPACE), L('cat > '), ANY_EXCEPT(WHITESPACE), L(" <<"), OPTIONAL(SPACE), L("EOF;"), EOL, REPEAT(ANY, greedy=False), EOL, L("EOF"), EOL)
+	def value(self):
+		return "<div class='write-to'><div class='filename'>" + self[2].string + "</div><pre>" + cgi.escape(self[7].string) + "</pre></div>\n"
+class HideOutput(Grammar):
+	grammar = (L("hide_output "), REF("BashElement"))
+	def value(self):
+		return self[1].value()
+class SuppressedLine(Grammar):
+	grammar = (OPTIONAL(SPACE), L("echo "), REST_OF_LINE, EOL)
+	def value(self):
+		if "|" in self.string  or ">" in self.string:
+			return "<pre>" + cgi.escape(self.string) + "</pre>\n"
+		return ""
+class EditConf(Grammar):
+	grammar = (
+		L('tools/editconf.py '),
+		FILENAME,
+		SPACE,
+		OPTIONAL((LIST_OF(
+			L("-w") | L("-s"),
+			sep=SPACE,
+		), SPACE)),
+		REST_OF_LINE,
+		OPTIONAL(SPACE),
+		EOL
+		)
+	def value(self):
+		conffile = self[1]
+		options = [""]
+		mode = 1
+		for c in self[4].string:
+			if mode == 1 and c in (" ", "\t") and options[-1] != "":
+				# new word
+				options.append("")
+			elif mode < 0:
+				# escaped character
+				options[-1] += c
+				mode = -mode
+			elif c == "\\":
+				# escape next character
+				mode = -mode
+			elif mode == 1 and c == '"':
+				mode = 2
+			elif mode == 2 and c == '"':
+				mode = 1
+			else:
+				options[-1] += c
+		if options[-1] == "": options.pop(-1)
+		return "<div class='write-to'><div class='filename'>" + self[1].string + "</div><pre>" + "\n".join(cgi.escape(s) for s in options) + "</pre></div>\n"
+class CaptureOutput(Grammar):
+	grammar = OPTIONAL(SPACE), WORD("A-Za-z_"), L('=$('), REST_OF_LINE, L(")"), OPTIONAL(L(';')), EOL
+	def value(self):
+		cmd = self[3].string
+		cmd = cmd.replace("; ", "\n")
+		return "<div class='write-to'><div class='filename'>$" + self[1].string + "=</div><pre>" + cgi.escape(cmd) + "</pre></div>\n"
+class SedReplace(Grammar):
+	grammar = OPTIONAL(SPACE), L('sed -i "s/'), OPTIONAL(L('^')), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/'), ONE_OR_MORE(WORD("-A-Za-z0-9 #=\\{};.*$_!()")), L('/"'), SPACE, FILENAME, EOL
+	def value(self):
+		return "<div class='write-to'><div class='filename'>" + self[8].string + "</div><p>replace</p><pre>" + cgi.escape(self[3].string.replace(".*", ". . .")) + "</pre><p>with</p><pre>" + cgi.escape(self[5].string.replace("\\n", "\n").replace("\\t", "\t")) + "</pre></div>\n"
+class AptGet(Grammar):
+	grammar = (ZERO_OR_MORE(SPACE), L("apt_install "), REST_OF_LINE, EOL)
+	def value(self):
+		return "<pre>" + self[0].string + "apt-get install -y " + cgi.escape(re.sub(r"\s+", " ", self[2].string)) + "</pre>\n"
+class UfwAllow(Grammar):
+	grammar = (ZERO_OR_MORE(SPACE), L("ufw_allow "), REST_OF_LINE, EOL)
+	def value(self):
+		return "<pre>" + self[0].string + "ufw allow " + cgi.escape(self[2].string) + "</pre>\n"
+class OtherLine(Grammar):
+	grammar = (REST_OF_LINE, EOL)
+	def value(self):
+		if self.string.strip() == "": return ""
+		return "<pre>" + cgi.escape(self.string.rstrip()) + "</pre>\n"
+class BashElement(Grammar):
+	grammar = Comment | Source | CatEOF | SuppressedLine | HideOutput | EditConf | CaptureOutput | SedReplace | AptGet | UfwAllow | OtherLine
+	def value(self):
+		return self[0].value()
+class BashScript(Grammar):
+	grammar = (OPTIONAL(HashBang), REPEAT(BashElement))
+	def value(self):
+		return [line.value() for line in self[1]]
+	@staticmethod
+	def parse(fn):
+		if fn in ("setup/functions.sh", "/etc/mailinabox.conf"): return ""
+		parser = BashScript.parser()
+		string = open(fn).read()
+		string = re.sub(r"\s*\\\n\s*", " ", string)
+		string = re.sub(".* #NODOC\n", "", string)
+		string = re.sub("\n\s*if .*|\n\s*fi|\n\s*else", "", string)
+		string = re.sub("hide_output ", "", string)
+		result = parser.parse_string(string)
+		v = "<div class='sourcefile'><a href=\"%s\">%s</a></div>\n" % ("https://github.com/mail-in-a-box/mailinabox/tree/master/" + fn, fn)
+		v += "".join(result.value())
+		v = v.replace("</pre>\n<pre>", "\n")
+		v = re.sub("<pre>([\w\W]*?)</pre>", lambda m : "<pre>" + strip_indent(m.group(1)) + "</pre>", v)
+		v = re.sub(r"\$?PRIMARY_HOSTNAME", "<b>box.yourdomain.com</b>", v)
+		v = re.sub(r"\$?STORAGE_ROOT", "<code><b>/path/to/user-data</b></code>", v)
+		v = v.replace("`pwd`",  "<code><b>/path/to/mailinabox</b></code>")
+		return v
+if __name__ == '__main__':
+	generate_documentation()