(netflow) add missing configd tempates, add flowctl stats api/ui components

src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/NetflowController.php

@@ -125,4 +125,20 @@ class NetflowController extends ApiControllerBase
             return array("status" => "inactive");
         }
     }
+
+    /**
+     * Retrieve netflow cache statistics
+     * @return array cache statistics per netgraph node
+     */
+    public function cache_statsAction()
+    {
+        $backend = new Backend();
+        $response = $backend->configdRun("netflow cache stats json");
+        $stats = json_decode($response, true);
+        if ($stats != null) {
+            return $stats;
+        } else {
+            return array();
+        }
+    }
 }

src/opnsense/mvc/app/views/OPNsense/Diagnostics/netflow.volt

@@ -45,15 +45,73 @@ POSSIBILITY OF SUCH DAMAGE.
             });
         });
 
+        $("#act_refresh_cache_stats").click(function(){
+            ajaxGet(url='/api/diagnostics/netflow/cache_stats',sendData={}, callback=function(data, status) {
+                var html = []
+                // convert to plain Array
+                var data_arr = $.makeArray(data)[0];
+                // sort by flow
+                Object.keys(data_arr).sort().forEach(function (index) {
+                    value = data_arr[index];
+                    var fields = ["if", "DstIPaddresses", "SrcIPaddresses", "Pkts"];
+                    tr_str = '<tr>';
+                    tr_str += '<td>'+index+'</td>';
+                    for (var i = 0; i < fields.length; i++) {
+                        if (value[fields[i]] != null) {
+                            tr_str += '<td>' + value[fields[i]] + '</td>';
+                        } else {
+                            tr_str += '<td></td>';
+                        }
+                    }
+                    tr_str += '</tr>';
+                    html.push(tr_str);
+                });
+                $("#cache_stats > tbody").html(html.join(''));
+            });
+        });
+
+        // refresh cache stats on tab open
+        $('a[data-toggle="tab"]').on('shown.bs.tab', function (e) {
+            if (e.target.id == 'cache_tab'){
+                $("#act_refresh_cache_stats").click();
+            }
+        });
     });
 </script>
 
 <ul class="nav nav-tabs" data-tabs="tabs" id="maintabs">
-    <li class="active"><a data-toggle="tab" href="#capture">{{ lang._('Capture') }}</a></li>
+    <li class="active"><a data-toggle="tab" id="capture_tab" href="#capture">{{ lang._('Capture') }}</a></li>
+    <li><a data-toggle="tab" id="cache_tab" href="#cache">{{ lang._('Cache') }}</a></li>
 </ul>
 <div class="tab-content content-box tab-content">
     <div id="capture" class="tab-pane fade in active">
         <!-- tab page capture -->
         {{ partial("layout_partials/base_form",['fields':captureForm,'id':'frm_CaptureSettings', 'apply_btn_id':'btn_save_capture'])}}
     </div>
+    <div id="cache" class="tab-pane fade in">
+        <!-- tab page netfow cache -->
+        <table class="table table-striped" id="cache_stats">
+          <thead>
+            <tr>
+              <th>{{ lang._('Flow') }}</th>
+              <th>{{ lang._('Interface') }}</th>
+              <th>{{ lang._('Destinations') }}</th>
+              <th>{{ lang._('Sources') }}</th>
+              <th>{{ lang._('Pkts') }}</th>
+            </tr>
+          </thead>
+          <tbody>
+          </tbody>
+          <tfoot>
+            <tr>
+              <td colspan="5">
+                <button id="act_refresh_cache_stats" type="button" class="btn btn-default">
+                  <span>{{ lang._('Refresh') }}</span>
+                  <span class="fa fa-refresh"></span>
+                </button>
+              </td>
+            </tr>
+          </tfoot>
+        </table>
+    </div>
 </div>

src/opnsense/scripts/netflow/flowctl_stats.py

+#!/usr/local/bin/python2.7
+
+"""
+    Copyright (c) 2016 Ad Schellevis
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+    returns the aggregated output of flowctl (netflow)
+"""
+import tempfile
+import subprocess
+import os
+import sys
+import ujson
+
+if __name__ == '__main__':
+    result = dict()
+    netflow_nodes=list()
+    with tempfile.NamedTemporaryFile() as output_stream:
+        subprocess.call(['/usr/sbin/ngctl', 'list'], stdout=output_stream, stderr=open(os.devnull, 'wb'))
+        output_stream.seek(0)
+        for line in output_stream.read().split('\n'):
+            if line.find('netflow_') > -1:
+                netflow_nodes.append(line.split()[1])
+
+    for netflow_node in netflow_nodes:
+        node_stats={'SrcIPaddress': list(), 'DstIPaddress': list(), 'Pkts': 0}
+        with tempfile.NamedTemporaryFile() as output_stream:
+            subprocess.call(['/usr/sbin/flowctl','%s:'%netflow_node, 'show'],
+                             stdout=output_stream, stderr=open(os.devnull, 'wb'))
+            output_stream.seek(0)
+            for line in output_stream.read().split('\n'):
+                fields=line.split()
+                if (len(fields) >= 8 and fields[0] != 'SrcIf'):
+                    node_stats['Pkts'] += int(fields[7])
+                    if fields[1] not in node_stats['SrcIPaddress']:
+                        node_stats['SrcIPaddress'].append(fields[1])
+                    if fields[3] not in node_stats['DstIPaddress']:
+                        node_stats['DstIPaddress'].append(fields[3])
+        result[netflow_node]={'Pkts': node_stats['Pkts'],
+                              'if': netflow_node[8:],
+                              'SrcIPaddresses': len(node_stats['SrcIPaddress']),
+                              'DstIPaddresses': len(node_stats['DstIPaddress'])}
+
+    # handle command line argument (type selection)
+    if len(sys.argv) > 1 and 'json' in sys.argv:
+        print(ujson.dumps(result))
+    else:
+        print ('[contents of netflow cache]')
+        for netflow_node in result:
+            print ('node : %s' % netflow_node)
+            print ('  #source addresses        : %d' % result[netflow_node]['SrcIPaddresses'])
+            print ('  #destination addresses   : %d' % result[netflow_node]['DstIPaddresses'])
+            print ('  #packets                 : %d' % result[netflow_node]['Pkts'])

src/opnsense/service/conf/actions.d/actions_netflow.conf

@@ -22,3 +22,8 @@ parameters:
 type:script_output
 message:get netflow status
 
+[cache.stats]
+command:/usr/local/opnsense/scripts/netflow/flowctl_stats.py
+parameters:%s
+type:script_output
+message:retrieve flow cache statistics

src/opnsense/service/templates/OPNsense/Netflow/+MANIFEST

+name: opnsense-netflow
+version: 1.0
+origin: opnsense/netflow
+comment: Netflow configuration
+desc: netflow configuration templates
+maintainer: ad at opnsense.org
+www: https://opnsense.org
+prefix: /

src/opnsense/service/templates/OPNsense/Netflow/+TARGETS

+netflow.conf:/usr/local/etc/netflow.conf
+rc.conf.d:/etc/rc.conf.d/netflow

src/opnsense/service/templates/OPNsense/Netflow/netflow.conf

+#
+# Automatic generated configuration for netflow.
+# Do not edit this file manually.
+#
+{% from 'OPNsense/Macros/interface.macro' import physical_interface %}
+{%
+  if helpers.exists('OPNsense.Netflow.capture.interfaces')
+  and
+  OPNsense.Netflow.capture.interfaces.strip()
+  and
+  OPNsense.Netflow.capture.targets.strip()
+  %}
+netflow_interfaces="{% for interface in OPNsense.Netflow.capture.interfaces.split(',')
+%}{{
+    physical_interface(interface)
+}} {%
+  endfor%}"
+netflow_version="{%if OPNsense.Netflow.capture.version == 'v9' %}9{% else %}5{%endif%}"
+netflow_int_destination="127.0.0.1:2055"
+netflow_destinations="{{OPNsense.Netflow.capture.targets.replace(',', ' ')}}"
+{% endif %}

src/opnsense/service/templates/OPNsense/Netflow/rc.conf.d

+#
+# Automatic generated configuration for netflow.
+# Do not edit this file manually.
+#
+{%
+  if helpers.exists('OPNsense.Netflow.capture.interfaces')
+  and
+  OPNsense.Netflow.capture.interfaces.strip()
+  and
+  OPNsense.Netflow.capture.targets.strip()%}
+netflow_enable="YES"
+{% else %}
+netflow_enable="NO"
+{% endif %}