backups now use duplicity's built-in gpg symmetric encryption

Merge branch 'dhpiggott-gpg-encrypt-backups'

CHANGELOG.md

@@ -12,9 +12,9 @@ Mail:
 
 System:
 
+* Backups now use duplicity's built-in gpg symmetric AES256 encryption rather than my home-brewed encryption. Old backups will be incorporated inside the first backup after this update but then deleted from disk (i.e. your backups from the previous few days will be backed up).
 * Internationalized Domain Names (IDNs) should now work in email. If you had custom DNS or custom web settings for internationalized domains, check that they are still working.
 
-
 v0.08 (April 1, 2015)
 ---------------------
 

management/backup.py

@@ -2,11 +2,10 @@
 
 # This script performs a backup of all user data:
 # 1) System services are stopped while a copy of user data is made.
-# 2) An incremental backup is made using duplicity into the
-#    directory STORAGE_ROOT/backup/duplicity.
+# 2) An incremental encrypted backup is made using duplicity into the
+#    directory STORAGE_ROOT/backup/encrypted. The password used for
+#    encryption is stored in backup/secret_key.txt.
 # 3) The stopped services are restarted.
-# 4) The backup files are encrypted with a long password (stored in
-#    backup/secret_key.txt) to STORAGE_ROOT/backup/encrypted.
 # 5) STORAGE_ROOT/backup/after-backup is executd if it exists.
 
 import os, os.path, shutil, glob, re, datetime
@@ -14,13 +13,13 @@ import dateutil.parser, dateutil.relativedelta, dateutil.tz
 
 from utils import exclusive_process, load_environment, shell
 
-# destroy backups when the most recent increment in the chain
+# Destroy backups when the most recent increment in the chain
 # that depends on it is this many days old.
 keep_backups_for_days = 3
 
 def backup_status(env):
 	# What is the current status of backups?
-	# Loop through all of the files in STORAGE_ROOT/backup/duplicity to
+	# Loop through all of the files in STORAGE_ROOT/backup/encrypted to
 	# get a list of all of the backups taken and sum up file sizes to
 	# see how large the storage is.
 
@@ -36,10 +35,10 @@ def backup_status(env):
 		return "%d hours, %d minutes" % (rd.hours, rd.minutes)
 
 	backups = { }
-	basedir = os.path.join(env['STORAGE_ROOT'], 'backup/duplicity/')
-	encdir = os.path.join(env['STORAGE_ROOT'], 'backup/encrypted/')
-	os.makedirs(basedir, exist_ok=True) # os.listdir fails if directory does not exist
-	for fn in os.listdir(basedir):
+	backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
+	backup_dir = os.path.join(backup_root, 'encrypted')
+	os.makedirs(backup_dir, exist_ok=True) # os.listdir fails if directory does not exist
+	for fn in os.listdir(backup_dir):
 		m = re.match(r"duplicity-(full|full-signatures|(inc|new-signatures)\.(?P<incbase>\d+T\d+Z)\.to)\.(?P<date>\d+T\d+Z)\.", fn)
 		if not m: raise ValueError(fn)
 
@@ -53,15 +52,9 @@ def backup_status(env):
 				"full": m.group("incbase") is None,
 				"previous": m.group("incbase"),
 				"size": 0,
-				"encsize": 0,
 			}
 
-		backups[key]["size"] += os.path.getsize(os.path.join(basedir, fn))
-
-		# Also check encrypted size.
-		encfn = os.path.join(encdir, fn + ".enc")
-		if os.path.exists(encfn):
-			backups[key]["encsize"] += os.path.getsize(encfn)
+		backups[key]["size"] += os.path.getsize(os.path.join(backup_dir, fn))
 
 	# Ensure the rows are sorted reverse chronologically.
 	# This is relied on by should_force_full() and the next step.
@@ -78,7 +71,7 @@ def backup_status(env):
 			break
 		incremental_count += 1
 		incremental_size += bak["size"]
-	
+
 	# Predict how many more increments until the next full backup,
 	# and add to that the time we hold onto backups, to predict
 	# how long the most recent full backup+increments will be held
@@ -106,9 +99,8 @@ def backup_status(env):
 			bak["deleted_in"] = deleted_in
 
 	return {
-		"directory": basedir,
-		"encpwfile": os.path.join(env['STORAGE_ROOT'], 'backup/secret_key.txt'),
-		"encdirectory": encdir,
+		"directory": backup_dir,
+		"encpwfile": os.path.join(backup_root, 'secret_key.txt'),
 		"tz": now.tzname(),
 		"backups": backups,
 	}
@@ -137,10 +129,35 @@ def perform_backup(full_backup):
 
 	exclusive_process("backup")
 
-	# Ensure the backup directory exists.
-	backup_dir = os.path.join(env["STORAGE_ROOT"], 'backup')
-	backup_duplicity_dir = os.path.join(backup_dir, 'duplicity')
-	os.makedirs(backup_duplicity_dir, exist_ok=True)
+	backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
+	backup_cache_dir = os.path.join(backup_root, 'cache')
+	backup_dir = os.path.join(backup_root, 'encrypted')
+
+	# In an older version of this script, duplicity was called
+	# such that it did not encrypt the backups it created (in
+	# backup/duplicity), and instead openssl was called separately
+	# after each backup run, creating AES256 encrypted copies of
+	# each file created by duplicity in backup/encrypted.
+	#
+	# We detect the transition by the presence of backup/duplicity
+	# and handle it by 'dupliception': we move all the old *un*encrypted
+	# duplicity files up out of the backup/duplicity directory (as
+	# backup/ is excluded from duplicity runs) in order that it is
+	# included in the next run, and we delete backup/encrypted (which
+	# duplicity will output files directly to, post-transition).
+	old_backup_dir = os.path.join(backup_root, 'duplicity')
+	migrated_unencrypted_backup_dir = os.path.join(env["STORAGE_ROOT"], "migrated_unencrypted_backup")
+	if os.path.isdir(old_backup_dir):
+		# Move the old unencrpyted files to a new location outside of
+		# the backup root so they get included in the next (new) backup.
+		# Then we'll delete them. Also so that they do not get in the
+		# way of duplicity doing a full backup on the first run after
+		# we take care of this.
+		shutil.move(old_backup_dir, migrated_unencrypted_backup_dir)
+
+		# The backup_dir (backup/encrypted) now has a new purpose.
+		# Clear it out.
+		shutil.rmtree(backup_dir)
 
 	# On the first run, always do a full backup. Incremental
 	# will fail. Otherwise do a full backup when the size of
@@ -152,75 +169,74 @@ def perform_backup(full_backup):
 	shell('check_call', ["/usr/sbin/service", "dovecot", "stop"])
 	shell('check_call', ["/usr/sbin/service", "postfix", "stop"])
 
+	# Get the encryption passphrase. secret_key.txt is 2048 random
+	# bits base64-encoded and with line breaks every 65 characters.
+	# gpg will only take the first line of text, so sanity check that
+	# that line is long enough to be a reasonable passphrase. It
+	# only needs to be 43 base64-characters to match AES256's key
+	# length of 32 bytes.
+	with open(os.path.join(backup_root, 'secret_key.txt')) as f:
+		passphrase = f.readline().strip()
+	if len(passphrase) < 43: raise Exception("secret_key.txt's first line is too short!")
+	env_with_passphrase = { "PASSPHRASE" : passphrase }
+
 	# Update the backup mirror directory which mirrors the current
 	# STORAGE_ROOT (but excluding the backups themselves!).
 	try:
 		shell('check_call', [
 			"/usr/bin/duplicity",
 			"full" if full_backup else "incr",
-			"--no-encryption",
-			"--archive-dir", "/tmp/duplicity-archive-dir",
-			"--name", "mailinabox",
-			"--exclude", backup_dir,
-			"--volsize", "100",
-			"--verbosity", "warning",
+			"--archive-dir", backup_cache_dir,
+			"--exclude", backup_root,
+			"--volsize", "250",
+			"--gpg-options", "--cipher-algo=AES256",
 			env["STORAGE_ROOT"],
-			"file://" + backup_duplicity_dir
-			])
+			"file://" + backup_dir
+			],
+			env_with_passphrase)
 	finally:
 		# Start services again.
 		shell('check_call', ["/usr/sbin/service", "dovecot", "start"])
 		shell('check_call', ["/usr/sbin/service", "postfix", "start"])
 
+	# Once the migrated backup is included in a new backup, it can be deleted.
+	if os.path.isdir(migrated_unencrypted_backup_dir):
+		shutil.rmtree(migrated_unencrypted_backup_dir)
+
 	# Remove old backups. This deletes all backup data no longer needed
-	# from more than 31 days ago. Must do this before destroying the
-	# cache directory or else this command will re-create it.
+	# from more than 3 days ago.
 	shell('check_call', [
 		"/usr/bin/duplicity",
 		"remove-older-than",
 		"%dD" % keep_backups_for_days,
-		"--archive-dir", "/tmp/duplicity-archive-dir",
-		"--name", "mailinabox",
+		"--archive-dir", backup_cache_dir,
 		"--force",
-		"--verbosity", "warning",
-		"file://" + backup_duplicity_dir
-		])
-
-	# Remove duplicity's cache directory because it's redundant with our backup directory.
-	shutil.rmtree("/tmp/duplicity-archive-dir")
-
-	# Encrypt all of the new files.
-	backup_encrypted_dir = os.path.join(backup_dir, 'encrypted')
-	os.makedirs(backup_encrypted_dir, exist_ok=True)
-	for fn in os.listdir(backup_duplicity_dir):
-		fn2 = os.path.join(backup_encrypted_dir, fn) + ".enc"
-		if os.path.exists(fn2): continue
+		"file://" + backup_dir
+		],
+		env_with_passphrase)
+
+	# From duplicity's manual:
+	# "This should only be necessary after a duplicity session fails or is
+	# aborted prematurely."
+	# That may be unlikely here but we may as well ensure we tidy up if
+	# that does happen - it might just have been a poorly timed reboot.
+	shell('check_call', [
+		"/usr/bin/duplicity",
+		"cleanup",
+		"--archive-dir", backup_cache_dir,
+		"--force",
+		"file://" + backup_dir
+		],
+		env_with_passphrase)
 
-		# Encrypt the backup using the backup private key.
-		shell('check_call', [
-			"/usr/bin/openssl",
-			"enc",
-			"-aes-256-cbc",
-			"-a",
-			"-salt",
-			"-in", os.path.join(backup_duplicity_dir, fn),
-			"-out", fn2,
-			"-pass", "file:%s" % os.path.join(backup_dir, "secret_key.txt"),
-			])
-
-		# The backup can be decrypted with:
-		# openssl enc -d -aes-256-cbc -a -in latest.tgz.enc -out /dev/stdout -pass file:secret_key.txt | tar -z
-
-	# Remove encrypted backups that are no longer needed.
-	for fn in os.listdir(backup_encrypted_dir):
-		fn2 = os.path.join(backup_duplicity_dir, fn.replace(".enc", ""))
-		if os.path.exists(fn2): continue
-		os.unlink(os.path.join(backup_encrypted_dir, fn))
+	# Change ownership of backups to the user-data user, so that the after-bcakup
+	# script can access them.
+	shell('check_call', ["/bin/chown", "-R", env["STORAGE_USER"], backup_dir])
 
 	# Execute a post-backup script that does the copying to a remote server.
 	# Run as the STORAGE_USER user, not as root. Pass our settings in
 	# environment variables so the script has access to STORAGE_ROOT.
-	post_script = os.path.join(backup_dir, 'after-backup')
+	post_script = os.path.join(backup_root, 'after-backup')
 	if os.path.exists(post_script):
 		shell('check_call',
 			['su', env['STORAGE_USER'], '-c', post_script],

management/templates/system-backup.html

@@ -28,8 +28,6 @@
   </tbody>
 </table>
 
-<p style="margin-top: 2em"><small>The size column in the table indicates the size of the encrypted backup, but the total size on disk shown above includes storage for unencrypted intermediate files.</small></p>
-
 <script>
 function nice_size(bytes) {
   var powers = ['bytes', 'KB', 'MB', 'GB', 'TB'];
@@ -54,7 +52,7 @@ function show_system_backup() {
     "GET",
     { },
     function(r) {
-      $('#backup-location').text(r.encdirectory);
+      $('#backup-location').text(r.directory);
       $('#backup-encpassword-file').text(r.encpwfile);
 
       $('#backup-status tbody').html("");
@@ -72,7 +70,7 @@ function show_system_backup() {
         tr.append( $('<td/>').text(b.date_str + " " + r.tz) );
         tr.append( $('<td/>').text(b.date_delta + " ago") );
         tr.append( $('<td/>').text(b.full ? "full" : "increment") );
-        tr.append( $('<td style="text-align: right"/>').text( nice_size(b.encsize)) );
+        tr.append( $('<td style="text-align: right"/>').text( nice_size(b.size)) );
         if (b.deleted_in)
           tr.append( $('<td/>').text(b.deleted_in) );
         else
@@ -80,7 +78,6 @@ function show_system_backup() {
         $('#backup-status tbody').append(tr);
 
         total_disk_size += b.size;
-        total_disk_size += b.encsize;
       }
 
       $('#backup-total-size').text(nice_size(total_disk_size));