(firewall, aliases) add geoip

src/etc/rc.update_urltables

@@ -10,16 +10,15 @@ if (!isset($config['aliases']['alias'])) {
     exit;
 }
 
-// Gather list of urltable aliases
+// Gather list of urltable / geoip aliases
 $todo = array();
+$download_geoip = false;
 foreach ($config['aliases']['alias'] as $alias) {
     if (preg_match('/urltable/i', $alias['type'])) {
-        $tmp = array();
-        $tmp['type'] = $alias['type'];
-        $tmp['name'] = $alias['name'];
-        $tmp['url']  = $alias['url'];
-        $tmp['freq'] = $alias['updatefreq'];
-        $todo[] = $tmp;
+        $todo[] = $alias;
+    } elseif ($alias['type'] == 'geoip') {
+        $todo[] = $alias;
+        $download_geoip = true;
     }
 }
 
@@ -33,24 +32,49 @@ if (count($todo) > 0) {
         sleep($wait);
     }
 
+    // download geoip database
+    if ($download_geoip) {
+        // download the geoip database, first check if we haven't already done so the last day
+        if (!is_file('/usr/local/share/GeoIP/alias/NL-IPv4') || (time() - filemtime('/usr/local/share/GeoIP/alias/NL-IPv4')) > (86400 - 90)) {
+            log_error("{$argv[0]}: Download GeoIP database");
+            exec('/usr/local/opnsense/scripts/filter/download_geoip.py');
+        } else {
+            log_error("{$argv[0]}: GeoIP database doesn't need updating");
+        }
+    }
+
     log_error("{$argv[0]}: Starting URL table alias updates");
 
     $filter_reload = false;
-    foreach ($todo as $t) {
-        $r = process_alias_urltable($t['name'], $t['url'], $t['freq']);
-        if ($r == 1) {
-            $result = "";
-            // TODO: Change it when pf supports tables with ports
-            if ($t['type'] == "urltable") {
-                exec("/sbin/pfctl -t " . escapeshellarg($t['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($t['name']) . ".txt 2>&1", $result);
+    foreach ($todo as $alias) {
+        if (preg_match('/urltable/i', $alias['type'])) {
+            $r = process_alias_urltable($alias['name'], $alias['url'], $alias['updatefreq']);
+            if ($r == 1) {
+                if ($alias['type'] == "urltable") {
+                    exec("/sbin/pfctl -t " . escapeshellarg($alias['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($alias['name']) . ".txt 2>&1", $result);
+                    log_error("{$argv[0]}: Updated {$alias['name']} content from {$alias['url']}: ". $result[count($result)-1]);
+                } else {
+                    $filter_reload = true;
+                }
+            } elseif ($r == -1) {
+                log_error("{$argv[0]}: {$alias['name']} does not need updating.");
             } else {
-                $filter_reload = true;
+                log_error("{$argv[0]}: ERROR: could not update {$alias['name']} content from {$alias['url']}");
             }
-            log_error("{$argv[0]}: Updated {$t['name']} content from {$t['url']}: {$result[0]}");
-        } elseif ($r == -1) {
-            log_error("{$argv[0]}: {$t['name']} does not need updating.");
-        } else {
-            log_error("{$argv[0]}: ERROR: could not update {$t['name']} content from {$t['url']}");
+        } elseif ($alias['type'] == 'geoip') {
+            // concat geoip countries and load into pf table
+            $alias_content = "";
+            foreach (explode(' ', $alias['address']) as $country_code) {
+                if (strlen($country_code) == 2 && in_array($alias['proto'], array('IPv4', 'IPv6'))) {
+                    $filename = "/usr/local/share/GeoIP/alias/".$country_code."-".$alias['proto'];
+                    if (is_file($filename)) {
+                        $alias_content .= file_get_contents($filename);
+                    }
+                }
+            }
+            file_put_contents('/var/db/aliastables/'.basename($alias['name']).'.txt', $alias_content);
+            exec("/sbin/pfctl -t " . escapeshellarg($alias['name']) . " -T replace -f /var/db/aliastables/" . escapeshellarg($alias['name']) . ".txt 2>&1", $result);
+            log_error("{$argv[0]}: Updated {$alias['name']} content from geoip database: ". $result[count($result)-1]);
         }
     }
 

src/opnsense/scripts/filter/download_geoip.py

+#!/usr/local/bin/python2.7
+
+"""
+    Copyright (c) 2016 Ad Schellevis
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+    download maxmind GeoLite2 Free database into easy to use alias files [<COUNTRY>-<PROTO>] located
+    in /usr/local/share/GeoIP/alias
+"""
+import tempfile
+import subprocess
+import os
+import sys
+import ujson
+import requests
+import zipfile
+
+# define geoip download location
+url = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip'
+
+if __name__ == '__main__':
+    address_count = 0
+    file_count = 0
+    # flush data from remote url to temp file and unpack from there
+    with tempfile.NamedTemporaryFile() as tmp_stream:
+        r = requests.get(url)
+        if r.status_code == 200:
+            tmp_stream.write(r.content)
+            tmp_stream.seek(0)
+            with zipfile.ZipFile(tmp_stream, mode='r', compression=zipfile.ZIP_DEFLATED) as zf:
+                # fetch zip file contents
+                file_handles = dict()
+                for item in zf.infolist():
+                    if item.file_size > 0:
+                        file_handles[os.path.basename(item.filename)] = item
+                # only process geo ip data when archive contains country definitions
+                if 'GeoLite2-Country-Locations-en.csv' in file_handles:
+                    country_codes = dict()
+                    # parse geoname_id to country code map
+                    for line in zf.open(file_handles['GeoLite2-Country-Locations-en.csv']).read().split('\n'):
+                        parts = line.split(',')
+                        if len(parts) > 4 and len(parts[4]) >= 1 and len(parts[4]) <= 3:
+                            country_codes[parts[0]] = parts[4]
+                    # process all details into files per country / protocol
+                    for proto in ['IPv4', 'IPv6']:
+                        if 'GeoLite2-Country-Blocks-%s.csv' % proto in file_handles:
+                            output_handles = dict()
+                            for line in zf.open(file_handles['GeoLite2-Country-Blocks-%s.csv' % proto]).read().split('\n'):
+                                parts = line.split(',')
+                                if len(parts) > 3 and parts[1] in country_codes:
+                                    country_code = country_codes[parts[1]]
+                                    if country_code not in output_handles:
+                                        if not os.path.exists('/usr/local/share/GeoIP/alias'):
+                                            os.makedirs('/usr/local/share/GeoIP/alias')
+                                        output_handles[country_code] = open(
+                                            '/usr/local/share/GeoIP/alias/%s-%s'%(country_code,proto), 'w'
+                                        )
+                                        file_count += 1
+                                    output_handles[country_code].write("%s\n" % parts[0])
+                                    address_count += 1
+                            for country_code in output_handles:
+                                output_handles[country_code].close()
+
+            # output files and lines processed
+            print ('%d files written, with a total number of %d lines' % (file_count, address_count))