(captiveportal, new) work in progress script base

src/opnsense/scripts/OPNsense/CaptivePortal/disconnect.py

+#!/usr/local/bin/python2.7
+
+"""
+    Copyright (c) 2015 Deciso B.V. - Ad Schellevis
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+    disconnect client
+"""
+import sys
+import ujson
+from lib.db import DB
+from lib.arp import ARP
+from lib.ipfw import IPFW
+
+# parse input parameters
+parameters = {'sessionid': None, 'zoneid': None, 'output_type':'plain'}
+current_param = None
+for param in sys.argv[1:]:
+    if len(param) > 1 and param[0] == '/':
+        current_param = param[1:].lower()
+    elif current_param is not None:
+        if current_param in parameters:
+            parameters[current_param] = param.strip()
+        current_param = None
+
+# disconnect client
+response = {'terminateCause': 'UNKNOWN'}
+if parameters['sessionid'] is not None and parameters['zoneid']  is not None:
+    cp_db = DB()
+    # remove client
+    client_session_info = cp_db.del_client(parameters['zoneid'], parameters['sessionid'])
+    if client_session_info is not None:
+        cpIPFW = IPFW()
+        cpIPFW.delete_from_table(parameters['zoneid'], client_session_info['ip_address'])
+        client_session_info['terminateCause'] = 'User-Request'
+        response = client_session_info
+
+# output result as plain text or json
+if parameters['output_type'] != 'json':
+    for item in response:
+        print '%20s %s' % (item, response[item])
+else:
+    print(ujson.dumps(response))

src/opnsense/scripts/OPNsense/CaptivePortal/lib/db.py

@@ -75,18 +75,14 @@ class DB(object):
         response['sessionId'] = base64.b64encode(os.urandom(16))  # generate a new random session id
 
         cur = self._connection.cursor()
-        # update cp_clients in case there's already a user logged-in at this ip address.
-        # places an implicit lock on this client.
+        # set cp_client as deleted in case there's already a user logged-in at this ip address.
         cur.execute("""update cp_clients
-                       set    created = :startTime
-                       ,      username = :userName
-                       ,      mac_address = :macAddress
+                       set    deleted = 1
                        where  zoneid = :zoneid
                        and    ip_address = :ipAddress
                     """, response)
 
-        # normal operation, new user at this ip, add to host
-        if cur.rowcount == 0:
+        # add new session
         cur.execute("""insert into cp_clients(zoneid, sessionid, username,  ip_address, mac_address, created)
                        values (:zoneid, :sessionId, :userName, :ipAddress, :macAddress, :startTime)
                     """, response)
@@ -94,8 +90,36 @@ class DB(object):
         self._connection.commit()
         return response
 
+    def del_client(self, zoneid, sessionid):
+        """ mark (administrative) client for removal
+        :param zoneid: zone id
+        :param sessionid: session id
+        :return: client info before removal or None if client not found
+        """
+        cur = self._connection.cursor()
+        cur.execute(""" select  *
+                        from    cp_clients
+                        where   sessionid = :sessionid
+                        and     zoneid = :zoneid
+                        and     deleted = 0
+                    """, {'zoneid': zoneid, 'sessionid': sessionid})
+        data = cur.fetchall()
+        if len(data) > 0:
+            session_info = dict()
+            for fields in cur.description:
+                session_info[fields[0]] = data[0][len(session_info)]
+            # remove client
+            cur.execute("update cp_clients set deleted = 1 where sessionid = :sessionid and zoneid = :zoneid",
+                        {'zoneid': zoneid, 'sessionid': sessionid})
+            self._connection.commit()
+
+            return session_info
+        else:
+            return None
+
+
     def list_clients(self, zoneid):
-        """ return list of (administrative) connected clients
+        """ return list of (administrative) connected clients and usage statistics
         :param zoneid: zone id
         :return: list of clients
         """
@@ -103,14 +127,21 @@ class DB(object):
         fieldnames = list()
         cur = self._connection.cursor()
         # rename fields for API
-        cur.execute(""" select  zoneid
-                        ,       sessionid   sessionId
-                        ,       username    userName
-                        ,       created     startTime
-                        ,       ip_address  ipAddress
-                        ,       mac_address macAddress
-                        from    cp_clients
-                        where   zoneid = :zoneid
+        cur.execute(""" select  cc.zoneid
+                        ,       cc.sessionid   sessionId
+                        ,       cc.username    userName
+                        ,       cc.created     startTime
+                        ,       cc.ip_address  ipAddress
+                        ,       cc.mac_address macAddress
+                        ,       case when si.packets_in is null then 0 else si.packets_in end packets_in
+                        ,       case when si.packets_out is null then 0 else si.packets_out end packets_out
+                        ,       case when si.bytes_in is null then 0 else si.bytes_in end bytes_in
+                        ,       case when si.bytes_out is null then 0 else si.bytes_out end bytes_out
+                        ,       case when si.last_accessed is null then 0 else si.last_accessed end last_accessed
+                        from    cp_clients cc
+                        left join session_info si on si.zoneid = cc.zoneid and si.sessionid = cc.sessionid
+                        where   cc.zoneid = :zoneid
+                        and     cc.deleted = 0
                         """, {'zoneid': zoneid})
         while True:
             # fetch field names
@@ -128,3 +159,81 @@ class DB(object):
 
                 result.append(record)
         return result
+
+    def update_accounting_info(self, details):
+        """ update internal accounting database with given ipfw info (not per zone)
+        :param details: ipfw accounting details
+        """
+        if type(details) == dict:
+            # query registered data
+            sql = """ select    cc.ip_address, cc.zoneid, cc.sessionid
+                      ,         si.rowid si_rowid, si.prev_packets_in, si.prev_bytes_in
+                      ,         si.prev_packets_out, si.prev_bytes_out, si.last_accessed
+                      from      cp_clients cc
+                      left join session_info si on si.zoneid = cc.zoneid and si.sessionid = cc.sessionid
+                      order by  cc.ip_address, cc.deleted
+                  """
+            cur = self._connection.cursor()
+            cur2 = self._connection.cursor()
+            cur.execute(sql)
+            prev_record = {'ip_address': None}
+            for row in cur.fetchall():
+                # map fieldnumbers to names
+                record = {}
+                for fieldId in range(len(row)):
+                    record[cur.description[fieldId][0]] = row[fieldId]
+                # search unique hosts from dataset, both disabled and enabled.
+                if prev_record['ip_address'] != record['ip_address'] and record['ip_address'] in details:
+                    if record['si_rowid'] is None:
+                        # new session, add info object
+                        sql_new = """ insert into session_info(zoneid, sessionid, prev_packets_in, prev_bytes_in,
+                                                               prev_packets_out, prev_bytes_out,
+                                                               packets_in, packets_out, bytes_in, bytes_out,
+                                                               last_accessed)
+                                      values (:zoneid, :sessionid, :packets_in, :bytes_in, :packets_out, :bytes_out,
+                                              :packets_in, :packets_out, :bytes_in, :bytes_out, :last_accessed)
+                        """
+                        record['packets_in'] = details[record['ip_address']]['in_pkts']
+                        record['bytes_in'] = details[record['ip_address']]['in_bytes']
+                        record['packets_out'] = details[record['ip_address']]['out_pkts']
+                        record['bytes_out'] = details[record['ip_address']]['out_bytes']
+                        record['last_accessed'] = details[record['ip_address']]['last_accessed']
+                        cur2.execute(sql_new, record)
+                    else:
+                        # update session
+                        sql_update = """ update session_info
+                                         set    last_accessed = :last_accessed
+                                         ,      prev_packets_in = :prev_packets_in
+                                         ,      prev_packets_out = :prev_packets_out
+                                         ,      prev_bytes_in = :prev_bytes_in
+                                         ,      prev_bytes_out = :prev_bytes_out
+                                         ,      packets_in = packets_in + :packets_in
+                                         ,      packets_out = packets_out + :packets_out
+                                         ,      bytes_in = bytes_in + :bytes_in
+                                         ,      bytes_out = bytes_out + :bytes_out
+                                         where  rowid = :si_rowid
+                        """
+                        # add usage to session
+                        record['last_accessed'] = details[record['ip_address']]['last_accessed']
+                        if record['prev_packets_in'] <= details[record['ip_address']]['in_pkts'] and \
+                            record['prev_packets_out'] <= details[record['ip_address']]['out_pkts']:
+                            # ipfw data is still valid, add difference to use
+                            record['packets_in']  = (details[record['ip_address']]['in_pkts'] - record['prev_packets_in'])
+                            record['packets_out'] = (details[record['ip_address']]['out_pkts'] - record['prev_packets_out'])
+                            record['bytes_in']  = (details[record['ip_address']]['in_bytes'] - record['prev_bytes_in'])
+                            record['bytes_out'] = (details[record['ip_address']]['out_bytes'] - record['prev_bytes_out'])
+                        else:
+                            # the data has been reset (reloading rules), add current packet count
+                            record['packets_in'] = details[record['ip_address']]['in_pkts']
+                            record['packets_out'] = details[record['ip_address']]['out_pkts']
+                            record['bytes_in'] = details[record['ip_address']]['in_bytes']
+                            record['bytes_out'] = details[record['ip_address']]['out_bytes']
+
+                        record['prev_packets_in'] = details[record['ip_address']]['in_pkts']
+                        record['prev_packets_out'] = details[record['ip_address']]['out_pkts']
+                        record['prev_bytes_in'] = details[record['ip_address']]['in_bytes']
+                        record['prev_bytes_out'] = details[record['ip_address']]['out_bytes']
+                        cur2.execute(sql_update, record)
+
+                prev_record = record
+            self._connection.commit()

src/opnsense/scripts/OPNsense/CaptivePortal/lib/ipfw.py

@@ -103,8 +103,8 @@ class IPFW(object):
                 parts = line.split()
                 if len(parts) > 5:
                     if 30001 <= int(parts[0]) <= 50000 and parts[4] == 'count':
-                        in_pkts = int(parts[1])
-                        out_pkts = int(parts[2])
+                        line_pkts = int(parts[1])
+                        line_bytes = int(parts[2])
                         last_accessed = int(parts[3])
                         if parts[7] != 'any':
                             ip_address = parts[7]
@@ -113,15 +113,23 @@ class IPFW(object):
 
                         if ip_address not in result:
                             result[ip_address] = {'rule': int(parts[0]),
-                                                  'last_accessed': last_accessed,
-                                                  'in_pkts': in_pkts,
-                                                  'out_pkts': out_pkts
+                                                  'last_accessed': 0,
+                                                  'in_pkts' : 0,
+                                                  'in_bytes' : 0,
+                                                  'out_pkts' : 0,
+                                                  'out_bytes' : 0
                                                   }
-                        else:
-                            result[ip_address]['in_pkts'] += in_pkts
-                            result[ip_address]['out_pkts'] += out_pkts
                         result[ip_address]['last_accessed'] = max(result[ip_address]['last_accessed'],
                                                                   last_accessed)
+                        if parts[7] != 'any':
+                            # count input
+                            result[ip_address]['in_pkts'] = line_pkts
+                            result[ip_address]['in_bytes'] = line_bytes
+                        else:
+                            # count output
+                            result[ip_address]['out_pkts'] = line_pkts
+                            result[ip_address]['out_bytes'] = line_bytes
+
             return result
 
     def add_accounting(self, address):

src/opnsense/scripts/OPNsense/CaptivePortal/listClients.py

@@ -51,13 +51,15 @@ else:
 
 # output result as plain text or json
 if parameters['output_type'] != 'json':
-    heading = {'sessionid': 'sessionid',
-               'username': 'username',
-               'ip_address': 'ip_address',
-               'mac_address': 'mac_address'
+    heading = {'sessionId': 'sessionid',
+               'userName': 'username',
+               'ipAddress': 'ip_address',
+               'macAddress': 'mac_address',
+               'total_bytes': 'total_bytes'
                }
-    print '%(sessionid)-30s %(username)-20s %(ip_address)-20s %(mac_address)-20s' % heading
+    print '%(sessionId)-30s %(userName)-20s %(ipAddress)-20s %(macAddress)-20s %(total_bytes)-20s' % heading
     for item in response:
-        print '%(sessionid)-30s %(username)-20s %(ip_address)-20s %(mac_address)-20s' % item
+        item['total_bytes'] = (item['bytes_out'] + item['bytes_in'])
+        print '%(sessionId)-30s %(userName)-20s %(ipAddress)-20s %(macAddress)-20s %(total_bytes)-20s' % item
 else:
     print(ujson.dumps(response))

src/opnsense/scripts/OPNsense/CaptivePortal/sql/init.sql

@@ -10,6 +10,7 @@ create table cp_clients (
 ,     ip_address varchar
 ,     mac_address varchar
 ,     created number
+,     deleted integer default (0)
 ,     primary key (zoneid, sessionid)
 );
 
@@ -20,6 +21,14 @@ create index cp_clients_zone ON cp_clients (zoneid);
 create table session_info (
       zoneid int
 ,     sessionid varchar
+,     prev_packets_in integer
+,     prev_bytes_in   integer
+,     prev_packets_out integer
+,     prev_bytes_out   integer
+,     packets_in integer default (0)
+,     packets_out integer default (0)
+,     bytes_in integer default (0)
+,     bytes_out integer default (0)
+,     last_accessed integer
 ,     primary key (zoneid, sessionid)
 );
-

src/opnsense/scripts/OPNsense/CaptivePortal/update_stats.py

+#!/usr/local/bin/python2.7
+
+"""
+    Copyright (c) 2015 Deciso B.V. - Ad Schellevis
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+    update captive portal statistics
+"""
+import sys
+import ujson
+from lib.db import DB
+from lib.arp import ARP
+from lib.ipfw import IPFW
+
+db = DB()
+cur = db._connection.cursor();
+
+# update accounting
+ipfw = IPFW()
+#print ipfw.list_accounting_info()
+db.update_accounting_info(ipfw.list_accounting_info())
+
+# tmp = """
+# create table session_info (
+#       zoneid int
+# ,     sessionid varchar
+# ,     prev_packets_in integer
+# ,     prev_bytes_in   integer
+# ,     prev_packets_out integer
+# ,     prev_bytes_out   integer
+# ,     packets_in integer default (0)
+# ,     packets_out integer default (0)
+# ,     bytes_in integer default (0)
+# ,     bytes_out integer default (0)
+# ,     last_accessed integer
+# ,     primary key (zoneid, sessionid)
+# );
+#  """
+
+# cur.execute("drop table session_info");
+# cur.execute(tmp);