(captiveportal, new) work in progress script base

src/opnsense/scripts/OPNsense/CaptivePortal/captiveportal.sh

 #!/bin/sh
 
-# setup chroot environment for lighttpd
-mkdir -p /var/captiveportal
+CPWORKDIR="/var/captiveportal"
+CPDEFAULTTEMPLATE="/usr/local/opnsense/scripts/OPNsense/CaptivePortal/htdocs_default"
+
+# extract all zones from captive portal configuration
+CPZONES=`cat /usr/local/etc/captiveportal.conf | grep "\[zone_" | sed 's/\[zone_//' | sed 's/\]//'`
+
+# initialize captiveportal work directory
+if [ ! -d $CPWORKDIR ]; then
+  mkdir -p $CPWORKDIR
+fi
+
+# startup API dispatcher, forwards captive portal api request to shared OPNsense API
+if [ -f /var/run/lighttpd-api-dispatcher.pid ]; then
+  echo "Stopping API dispatcher"
+  /bin/pkill -TERM -F /var/run/lighttpd-api-dispatcher.pid
+fi
+echo "Starting API dispatcher"
+/usr/local/sbin/lighttpd -f /var/etc/lighttpd-api-dispatcher.conf
+
+# startup / bootstrap zones
+for zoneid in $CPZONES
+do
+    # bootstrap captiveportal jail
+    zonedirname="zone$zoneid"
+    echo "Install : zone $zoneid"
+    if [ -d $CPWORKDIR/$zonedirname/tmp ]; then
+        # remove temp (flush)
+        rm -rf $CPWORKDIR/$zonedirname/tmp
+    fi
+    mkdir $CPWORKDIR/$zonedirname/tmp
+    chmod 770 $CPWORKDIR/$zonedirname/tmp
+    chown www:www $CPWORKDIR/$zonedirname/tmp
+
+    # sync default template
+    /usr/local/bin/rsync -a $CPDEFAULTTEMPLATE/* $CPWORKDIR/$zonedirname/htdocs/
+
+    # todo, overlay custom user layout if available
+
+    # stop running instance
+    zonepid="/var/run/lighttpd-cp-zone-$zoneid.pid"
+    if [ -f $zonepid ]; then
+        echo "Stop : zone $zoneid"
+        /bin/pkill -TERM -F /var/run/lighttpd-cp-zone-$zoneid.pid
+        rm /var/run/lighttpd-cp-zone-$zoneid.pid
+    fi
+
+    # start new instance
+    echo "Start : zone $zoneid"
+    /usr/local/sbin/lighttpd -f /var/etc/lighttpd-cp-zone-$zoneid.conf
+done
+
+
+# cleanup removed zones
+for installed_zoneid in `ls $CPWORKDIR |  sed 's/zone//g'`
+do
+    if [ -d $CPWORKDIR/zone$installed_zoneid ]; then
+        for zoneid in $CPZONES
+        do
+            is_installed=0
+            if [ "$zoneid" -eq "$installed_zoneid" ]; then
+                is_installed=1
+            fi
+            if [ "$is_installed" -eq 0 ]; then
+                echo "Uninstall : zone $installed_zoneid"
+                # todo, insert rm
+            fi
+        done
+    fi
+done

src/opnsense/scripts/OPNsense/CaptivePortal/update_stats.py → src/opnsense/scripts/OPNsense/CaptivePortal/cp-background-process.py

@@ -30,34 +30,84 @@
 """
 import sys
 import ujson
+import time
+import syslog
+import traceback
+from lib import Config
 from lib.db import DB
 from lib.arp import ARP
 from lib.ipfw import IPFW
+from lib.daemonize import Daemonize
 
-db = DB()
-cur = db._connection.cursor();
-
-# update accounting
-ipfw = IPFW()
-#print ipfw.list_accounting_info()
-db.update_accounting_info(ipfw.list_accounting_info())
-
-# tmp = """
-# create table session_info (
-#       zoneid int
-# ,     sessionid varchar
-# ,     prev_packets_in integer
-# ,     prev_bytes_in   integer
-# ,     prev_packets_out integer
-# ,     prev_bytes_out   integer
-# ,     packets_in integer default (0)
-# ,     packets_out integer default (0)
-# ,     bytes_in integer default (0)
-# ,     bytes_out integer default (0)
-# ,     last_accessed integer
-# ,     primary key (zoneid, sessionid)
-# );
-#  """
-
-# cur.execute("drop table session_info");
-# cur.execute(tmp);
+def main():
+    syslog.syslog(syslog.LOG_ERR, 'starting captiveportal background process')
+    # handle to ipfw, arp and the config
+    ipfw = IPFW()
+    arp = ARP()
+    cnf = Config()
+    while True:
+        try:
+            # construct objects
+            db = DB()
+
+            # update accounting info
+            db.update_accounting_info(ipfw.list_accounting_info())
+
+            # process sessions per zone
+            arp.reload()
+            cpzones = cnf.get_zones()
+            for zoneid in cpzones:
+                registered_addresses = ipfw.list_table(zoneid)
+                registered_add_accounting = ipfw.list_accounting_info()
+                expected_clients = db.list_clients(zoneid)
+                # handle connected clients, timeouts, address changes, etc.
+                for db_client in expected_clients:
+                    # convert ip address to net, tables are registered as  nets
+                    if db_client['ipAddress'].strip().find('/') == -1:
+                        cpnet = '%s/32' % db_client['ipAddress'].strip()
+                    else:
+                        cpnet = db_client['ipAddress'].strip()
+
+                    # there are different reasons why a session should be removed, check for all reasons and
+                    # use the same method for the actual removal
+                    drop_session = False
+
+                    # todo, static ip and addresses shouldn't be affected by the timeout rules below.
+                    # check if hardtimeout is set and overrun for this session
+                    if 'hardtimeout' in cpzones[zoneid] and str(cpzones[zoneid]['hardtimeout']).isdigit():
+                        if int(cpzones[zoneid]['hardtimeout']) > 0:
+                            if time.time() - float(db_client['startTime']) / 60 > int(cpzones[zoneid]['hardtimeout']):
+                                drop_session = True
+                                drop_session = False
+
+                    # check session, if it should be active, validate its properties
+                    if not drop_session:
+                        # registered client, but not active according to ipfw (after reboot)
+                        if cpnet not in registered_addresses:
+                            ipfw.add_to_table(zoneid, cpnet)
+
+                        # is accounting rule still available? need to reapply after reload / reboot
+                        if cpnet not in registered_add_accounting and db_client['ipAddress'] not in registered_add_accounting:
+                            ipfw.add_accounting(cpnet)
+                    else:
+                        # remove session
+                        db.del_client(zoneid, db_client['sessionId'])
+                        ipfw.delete_from_table(zoneid, cpnet)
+                        ipfw.del_accounting(cpnet)
+
+            # cleanup, destruct
+            del db
+
+            # sleep
+            time.sleep(5)
+        except KeyboardInterrupt:
+            break
+        except:
+            syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
+
+# startup
+if len(sys.argv) > 1 and sys.argv[1].strip().lower() == 'run':
+    main()
+else:
+    daemon = Daemonize(app=__file__.split('/')[-1].split('.py')[0], pid='/var/run/captiveportal.db.pid', action=main)
+    daemon.start()

src/opnsense/scripts/OPNsense/CaptivePortal/lib/__init__.py

+"""
+    Copyright (c) 2015 Ad Schellevis
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+"""
+import os.path
+import stat
+from ConfigParser import ConfigParser
+
+class Config(object):
+    """ handle to captive portal config (/usr/local/etc/captiveportal.conf)
+    """
+    _cnf_filename = "/usr/local/etc/captiveportal.conf"
+
+    def __init__(self):
+        """ consctruct new config object
+        """
+        self.last_updated = 0
+        self._conf_handle = None
+        self._update()
+
+    def _update(self):
+        """ check if config is changed and (re)load
+        """
+        mod_time = os.stat(self._cnf_filename)[stat.ST_MTIME]
+        if os.path.exists(self._cnf_filename) and self.last_updated != mod_time:
+            self._conf_handle = ConfigParser()
+            self._conf_handle.read(self._cnf_filename)
+            self.last_updated = mod_time
+
+
+    def get_zones(self):
+        """ return list of configured zones
+            :return: dictionary index by zoneid, containing dictionaries with zone properties
+        """
+        result = dict()
+        self._update()
+        if self._conf_handle != None:
+            for section in self._conf_handle.sections():
+                if section.find('zone_') == 0:
+                    result[section.split('_')[1]] = dict()
+                    for item in self._conf_handle.items(section):
+                        result[section.split('_')[1]][item[0]] = item[1]
+        return result

src/opnsense/scripts/OPNsense/CaptivePortal/lib/arp.py

@@ -36,6 +36,11 @@ class ARP(object):
         self._arp_table = dict()
         self._fetch_arp_table()
 
+    def reload(self):
+        """ reload / parse arp table
+        """
+        self._fetch_arp_table()
+
     def _fetch_arp_table(self):
         """ parse system arp table and store result in this object
         :return: None

src/opnsense/scripts/OPNsense/CaptivePortal/lib/daemonize.py

+#!/usr/local/bin/python2.7
+
+# original source from https://github.com/thesharp/daemonize
+
+import fcntl
+import os
+import pwd
+import grp
+import sys
+import signal
+import resource
+import logging
+import atexit
+from logging import handlers
+
+
+class Daemonize(object):
+    """ Daemonize object
+    Object constructor expects three arguments:
+    - app: contains the application name which will be sent to syslog.
+    - pid: path to the pidfile.
+    - action: your custom function which will be executed after daemonization.
+    - keep_fds: optional list of fds which should not be closed.
+    - auto_close_fds: optional parameter to not close opened fds.
+    - privileged_action: action that will be executed before drop privileges if user or
+                         group parameter is provided.
+                         If you want to transfer anything from privileged_action to action, such as
+                         opened privileged file descriptor, you should return it from
+                         privileged_action function and catch it inside action function.
+    - user: drop privileges to this user if provided.
+    - group: drop privileges to this group if provided.
+    - verbose: send debug messages to logger if provided.
+    - logger: use this logger object instead of creating new one, if provided.
+    """
+    def __init__(self, app, pid, action, keep_fds=None, auto_close_fds=True, privileged_action=None, user=None, group=None, verbose=False, logger=None):
+        self.app = app
+        self.pid = pid
+        self.action = action
+        self.keep_fds = keep_fds or []
+        self.privileged_action = privileged_action or (lambda: ())
+        self.user = user
+        self.group = group
+        self.logger = logger
+        self.verbose = verbose
+        self.auto_close_fds = auto_close_fds
+
+    def sigterm(self, signum, frame):
+        """ sigterm method
+        These actions will be done after SIGTERM.
+        """
+        self.logger.warn("Caught signal %s. Stopping daemon." % signum)
+        os.remove(self.pid)
+        sys.exit(0)
+
+    def exit(self):
+        """ exit method
+        Cleanup pid file at exit.
+        """
+        self.logger.warn("Stopping daemon.")
+        os.remove(self.pid)
+        sys.exit(0)
+
+    def start(self):
+        """ start method
+        Main daemonization process.
+        """
+        # If pidfile already exists, we should read pid from there; to overwrite it, if locking
+        # will fail, because locking attempt somehow purges the file contents.
+        if os.path.isfile(self.pid):
+            with open(self.pid, "r") as old_pidfile:
+                old_pid = old_pidfile.read()
+        # Create a lockfile so that only one instance of this daemon is running at any time.
+        try:
+            lockfile = open(self.pid, "w")
+        except IOError:
+            print("Unable to create the pidfile.")
+            sys.exit(1)
+        try:
+            # Try to get an exclusive lock on the file. This will fail if another process has the file
+            # locked.
+            fcntl.flock(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except IOError:
+            print("Unable to lock on the pidfile.")
+            # We need to overwrite the pidfile if we got here.
+            with open(self.pid, "w") as pidfile:
+                pidfile.write(old_pid)
+            sys.exit(1)
+
+        # Fork, creating a new process for the child.
+        process_id = os.fork()
+        if process_id < 0:
+            # Fork error. Exit badly.
+            sys.exit(1)
+        elif process_id != 0:
+            # This is the parent process. Exit.
+            sys.exit(0)
+        # This is the child process. Continue.
+
+        # Stop listening for signals that the parent process receives.
+        # This is done by getting a new process id.
+        # setpgrp() is an alternative to setsid().
+        # setsid puts the process in a new parent group and detaches its controlling terminal.
+        process_id = os.setsid()
+        if process_id == -1:
+            # Uh oh, there was a problem.
+            sys.exit(1)
+
+        # Add lockfile to self.keep_fds.
+        self.keep_fds.append(lockfile.fileno())
+
+        # Close all file descriptors, except the ones mentioned in self.keep_fds.
+        devnull = "/dev/null"
+        if hasattr(os, "devnull"):
+            # Python has set os.devnull on this system, use it instead as it might be different
+            # than /dev/null.
+            devnull = os.devnull
+
+        if self.auto_close_fds:
+            for fd in range(3, resource.getrlimit(resource.RLIMIT_NOFILE)[0]):
+                if fd not in self.keep_fds:
+                    try:
+                        os.close(fd)
+                    except OSError:
+                        pass
+
+        devnull_fd = os.open(devnull, os.O_RDWR)
+        os.dup2(devnull_fd, 0)
+        os.dup2(devnull_fd, 1)
+        os.dup2(devnull_fd, 2)
+
+        if self.logger is None:
+            # Initialize logging.
+            self.logger = logging.getLogger(self.app)
+            self.logger.setLevel(logging.DEBUG)
+            # Display log messages only on defined handlers.
+            self.logger.propagate = False
+
+            # Initialize syslog.
+            # It will correctly work on OS X, Linux and FreeBSD.
+            if sys.platform == "darwin":
+                syslog_address = "/var/run/syslog"
+            else:
+                syslog_address = "/dev/log"
+
+            # We will continue with syslog initialization only if actually have such capabilities
+            # on the machine we are running this.
+            if os.path.isfile(syslog_address):
+                syslog = handlers.SysLogHandler(syslog_address)
+                if self.verbose:
+                    syslog.setLevel(logging.DEBUG)
+                else:
+                    syslog.setLevel(logging.INFO)
+                # Try to mimic to normal syslog messages.
+                formatter = logging.Formatter("%(asctime)s %(name)s: %(message)s",
+                                              "%b %e %H:%M:%S")
+                syslog.setFormatter(formatter)
+
+                self.logger.addHandler(syslog)
+
+        # Set umask to default to safe file permissions when running as a root daemon. 027 is an
+        # octal number which we are typing as 0o27 for Python3 compatibility.
+        os.umask(0o27)
+
+        # Change to a known directory. If this isn't done, starting a daemon in a subdirectory that
+        # needs to be deleted results in "directory busy" errors.
+        os.chdir("/")
+
+        # Execute privileged action
+        privileged_action_result = self.privileged_action()
+        if not privileged_action_result:
+            privileged_action_result = []
+
+        # Change gid
+        if self.group:
+            try:
+                gid = grp.getgrnam(self.group).gr_gid
+            except KeyError:
+                self.logger.error("Group {0} not found".format(self.group))
+                sys.exit(1)
+            try:
+                os.setgid(gid)
+            except OSError:
+                self.logger.error("Unable to change gid.")
+                sys.exit(1)
+
+        # Change uid
+        if self.user:
+            try:
+                uid = pwd.getpwnam(self.user).pw_uid
+            except KeyError:
+                self.logger.error("User {0} not found.".format(self.user))
+                sys.exit(1)
+            try:
+                os.setuid(uid)
+            except OSError:
+                self.logger.error("Unable to change uid.")
+                sys.exit(1)
+
+        try:
+            lockfile.write("%s" % (os.getpid()))
+            lockfile.flush()
+        except IOError:
+            self.logger.error("Unable to write pid to the pidfile.")
+            print("Unable to write pid to the pidfile.")
+            sys.exit(1)
+
+        # Set custom action on SIGTERM.
+        signal.signal(signal.SIGTERM, self.sigterm)
+        atexit.register(self.exit)
+
+        self.logger.warn("Starting daemon.")
+
+        self.action(*privileged_action_result)

src/opnsense/scripts/OPNsense/CaptivePortal/lib/db.py

@@ -31,7 +31,7 @@ import sqlite3
 
 
 class DB(object):
-    database_filename = '/tmp/captiveportal.sqlite'
+    database_filename = '/var/captiveportal/captiveportal.sqlite'
 
     def __init__(self):
         """ construct new database connection
@@ -40,6 +40,11 @@ class DB(object):
         self._connection = sqlite3.connect(self.database_filename)
         self.create()
 
+    def __del__(self):
+        """ destruct, close database handle
+        """
+        self._connection.close()
+
     def create(self, force_recreate=False):
         """ create/initialize new database
         :param force_recreate: if database already exists, remove old one first

src/opnsense/scripts/OPNsense/CaptivePortal/lib/ipfw.py

@@ -42,12 +42,13 @@ class IPFW(object):
         devnull = open(os.devnull, 'w')
         result = list()
         with tempfile.NamedTemporaryFile() as output_stream:
-            subprocess.check_call(['/sbin/ipfw', 'table', table_number, 'list'],
+            subprocess.check_call(['/sbin/ipfw', 'table', str(table_number), 'list'],
                                   stdout=output_stream,
                                   stderr=devnull)
             output_stream.seek(0)
             for line in output_stream.read().split('\n'):
-                result.append(line.split(' ')[0])
+                if line.split(' ')[0].strip() != "":
+                    result.append(line.split(' ')[0])
             return result
 
     def ip_or_net_in_table(self, table_number, address):