(network insight) add (raw) data export

src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/NetworkinsightController.php

@@ -169,6 +169,24 @@ class NetworkinsightController extends ApiControllerBase
         return array();
     }
 
+    /**
+     * get metadata from backend aggregation process
+     * @return array timeseries
+     */
+    public function getMetadataAction()
+    {
+        if ($this->request->isGet()) {
+            $backend = new Backend();
+            $configd_cmd = "netflow aggregate metadata json";
+            $response = $backend->configdRun($configd_cmd);
+            $metadata = json_decode($response, true);
+            if ($metadata != null) {
+                return $metadata;
+            }
+        }
+        return array();
+    }
+
     /**
      * return interface map (device / name)
      * @return array interfaces
@@ -220,4 +238,34 @@ class NetworkinsightController extends ApiControllerBase
         }
         return $result;
     }
+
+    /**
+     * request timeserie data to use for reporting
+     * @param string $provider provider class name
+     * @param string $from_date from timestamp
+     * @param string $to_date to timestamp
+     * @param string $resolution resolution in seconds
+     * @return string csv output
+     */
+     public function exportAction(
+        $provider = null,
+        $from_date = null,
+        $to_date = null,
+        $resolution = null
+    ) {
+        $this->response->setContentType('application/CSV', 'UTF-8');
+        $this->response->setHeader(
+                'Content-Disposition:',
+                "Attachment; filename=\"" . $provider . ".csv\""
+            );
+        if ($this->request->isGet()) {
+            $backend = new Backend();
+            $configd_cmd = "netflow aggregate export {$provider} {$from_date} {$to_date} {$resolution}" ;
+            $response = $backend->configdRun($configd_cmd);
+            return $response;
+        } else {
+            return "";
+        }
+
+    }
 }

src/opnsense/mvc/app/views/OPNsense/Diagnostics/networkinsight.volt

@@ -75,7 +75,27 @@ POSSIBILITY OF SUCH DAMAGE.
                   // fetch service names
                   ajaxGet('/api/diagnostics/networkinsight/getServices',{}, function(services, status) {
                       service_names = services;
+                      // return promise, no need to wait for getMetadata
                       dfObj.resolve();
+                      // fetch aggregators
+                      ajaxGet('/api/diagnostics/networkinsight/getMetadata',{}, function(metadata, status) {
+                        Object.keys(metadata['aggregators']).forEach(function (agg_name) {
+                          var res = metadata['aggregators'][agg_name]['resolutions'].join(',');
+                          $("#export_collection").append($("<option data-resolutions='"+res+"'/>").val(agg_name).text(agg_name));
+                        });
+                        $("#export_collection").change(function(){
+                            //alert($(this).find('option:selected').data('resolutions'));
+                            $("#export_resolution").html("");
+                            var resolutions = String($(this).find('option:selected').data('resolutions'));
+                            resolutions.split(',').map(function(item) {
+                                $("#export_resolution").append($("<option/>").val(item).text(item));
+                                console.log(item);
+                            });
+                            $("#export_resolution").selectpicker('refresh');
+                        });
+                        $("#export_collection").change();
+                        $("#export_collection").selectpicker('refresh');
+                      });
                   });
               });
           });
@@ -244,7 +264,7 @@ POSSIBILITY OF SUCH DAMAGE.
                     chart_data.push({'label': label + proto, 'value': item.total});
                 });
 
-                  d3.select("#chart_top_ports svg")
+                var diag = d3.select("#chart_top_ports svg")
                     .datum(chart_data)
                     .transition().duration(350)
                     .call(chart);
@@ -408,6 +428,21 @@ POSSIBILITY OF SUCH DAMAGE.
         });
       }
 
+      /**
+       * export detailed data (generate download link and click)
+       */
+      function export_flow_data()
+      {
+          var time_url = $("#export_date_from").val() + '/' +  $("#export_date_to").val();
+          var url = '/api/diagnostics/networkinsight/export/'+$("#export_collection").val()+'/'+time_url+'/'+$("#export_resolution").val();
+          var link = document.createElement("a");
+          $(link).click(function(e) {
+              e.preventDefault();
+              window.location.href = url;
+          });
+          $(link).click();
+      }
+
       // hide heading
       $(".page-content-head").addClass("hidden");
 
@@ -472,17 +507,26 @@ POSSIBILITY OF SUCH DAMAGE.
               to_date_ts = parseInt((date_end - (24*60*60*1000 * i)) / 1000);
               tmp_date = new Date(from_date_ts*1000);
               tmp = tmp_date.getDate() + '/' + (tmp_date.getMonth()+1) + '/' + tmp_date.getFullYear();
+              if (i < 62) {
                   $("#date_detail_from").append($("<option/>").val(from_date_ts).text(tmp));
                   $("#date_detail_to").append($("<option/>").val(to_date_ts).text(tmp));
               }
+              $("#export_date_from").append($("<option/>").val(from_date_ts).text(tmp));
+              $("#export_date_to").append($("<option/>").val(to_date_ts).text(tmp));
+
+          }
 
           $("#date_detail_from").selectpicker('refresh');
           $("#date_detail_to").selectpicker('refresh');
           $("#date_detail_from").change(function(){
               // change to date on change from date.
+              if ($("#date_detail_to").prop('selectedIndex') > $("#date_detail_from").prop('selectedIndex')) {
                   $("#date_detail_to").prop('selectedIndex', $("#date_detail_from").prop('selectedIndex'));
                   $("#date_detail_to").selectpicker('refresh');
+              }
           });
+          $("#export_date_from").selectpicker('refresh');
+          $("#export_date_to").selectpicker('refresh');
 
           chart_interface_totals();
           chart_top_dst_port_usage();
@@ -491,6 +535,7 @@ POSSIBILITY OF SUCH DAMAGE.
 
 
       $("#refresh_details").click(grid_details);
+      $("#export_btn").click(export_flow_data);
 
     });
 </script>
@@ -498,6 +543,7 @@ POSSIBILITY OF SUCH DAMAGE.
 <ul class="nav nav-tabs" data-tabs="tabs" id="maintabs">
     <li class="active"><a data-toggle="tab" id="totals_tab" href="#totals">{{ lang._('Totals') }}</a></li>
     <li><a data-toggle="tab" id="details_tab" href="#details">{{ lang._('Details') }}</a></li>
+    <li><a data-toggle="tab" id="export_tab" href="#export">{{ lang._('Export') }}</a></li>
 </ul>
 <div class="tab-content content-box tab-content" style="padding: 10px;">
     <div id="totals" class="tab-pane fade in active">
@@ -605,4 +651,49 @@ POSSIBILITY OF SUCH DAMAGE.
         </tfoot>
       </table>
     </div>
+    <div id="export" class="tab-pane fade in">
+      <br/>
+      <table class="table table-condensed table-striped">
+        <thead>
+          <tr>
+            <th>{{ lang._('Attribute') }}</th>
+            <th>{{ lang._('Value') }}</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr>
+            <td>{{ lang._('Collection') }}</td>
+            <td>
+              <select class="selectpicker" id="export_collection">
+              </select>
+            </td>
+          </tr>
+          <tr>
+            <td>{{ lang._('Resolution (seconds)') }}</td>
+            <td>
+              <select class="selectpicker" id="export_resolution">
+              </select>
+            </td>
+          </tr>
+          <tr>
+            <td>{{ lang._('From date') }}</td>
+            <td>
+              <select class="selectpicker" id="export_date_from"  data-live-search="true" data-size="10"></select>
+            </td>
+          </tr>
+          <tr>
+            <td>{{ lang._('To date') }}</td>
+            <td>
+              <select class="selectpicker" id="export_date_to"  data-live-search="true" data-size="10"></select>
+            </td>
+          </tr>
+          <tr>
+            <td></td>
+            <td>
+              <button id="export_btn" class="btn btn-default btn-xs"><i class="fa fa-cloud-download"></i> {{ lang._('Export')}}</button>
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
 </div>

src/opnsense/scripts/netflow/export_details.py

+#!/usr/local/bin/python2.7
+
+"""
+    Copyright (c) 2016 Ad Schellevis
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+    fetch detailed data from provider for specified timeserie
+"""
+import time
+import datetime
+import os
+import sys
+import ujson
+sys.path.insert(0, "/usr/local/opnsense/site-python")
+from lib.parse import parse_flow
+from lib.aggregate import BaseFlowAggregator
+import lib.aggregates
+import params
+
+
+app_params = {'start_time': '0',
+              'end_time': '1461251783',
+              'resolution': '300',
+              'provider': 'FlowSourceAddrTotals'
+              }
+params.update_params(app_params)
+
+# handle input parameters
+valid_params = False
+if app_params['start_time'].isdigit():
+    start_time = int(app_params['start_time'])
+    if app_params['end_time'].isdigit():
+        end_time = int(app_params['end_time'])
+        if app_params['resolution'].isdigit():
+            resolution = int(app_params['resolution'])
+            valid_params = True
+
+if valid_params:
+    for agg_class in lib.aggregates.get_aggregators():
+        if app_params['provider'] == agg_class.__name__:
+            if resolution in agg_class.resolutions():
+                # found provider and resolution, start spooling data
+                obj = agg_class(resolution)
+                rownum=0
+                column_names = dict()
+                for record in obj.get_data(start_time, end_time):
+                    if rownum == 0:
+                        column_names = record.keys()
+                        # dump heading
+                        print (','.join(column_names))
+                    line = list()
+                    for item in column_names:
+                        if not record[item]:
+                            line.append("")
+                        if type(record[item]) == datetime.datetime:
+                            line.append('%s+00:00'%record[item].strftime('%Y/%m/%d %H:%M:%S'))
+                        elif type(record[item]) == float:
+                            line.append('%.4f' % record[item])
+                        elif type(record[item]) == int:
+                            line.append('%d' % record[item])
+                        else:
+                            line.append(record[item])
+                    print (','.join(line))
+                    rownum += 1
+else:
+    print ('missing parameters :')
+    tmp = list()
+    for key in app_params:
+        tmp.append('/%s %s' % (key, app_params[key]))
+    print ('  %s %s'%(sys.argv[0], ' '.join(tmp)))
+    print ('')
+    print ('  resolution : sample rate in seconds')
+    print ('  start_time : start time (seconds since epoch)')
+    print ('  end_time : end timestamp (seconds since epoch)')
+    print ('  provider : data provider classname')

src/opnsense/scripts/netflow/lib/aggregate.py

@@ -31,6 +31,33 @@ import os
 import datetime
 import sqlite3
 
+def convert_timestamp(val):
+    """ convert timestamps from string (internal sqlite type) or seconds since epoch
+    """
+    if val.find('-') > -1:
+        # formatted date/time
+        if val.find(" ") > -1:
+            datepart, timepart = val.split(" ")
+        else:
+            datepart = val
+            timepart = "0:0:0,0"
+        year, month, day = map(int, datepart.split("-"))
+        timepart_full = timepart.split(".")
+        hours, minutes, seconds = map(int, timepart_full[0].split(":"))
+        if len(timepart_full) == 2:
+            microseconds = int('{:0<6.6}'.format(timepart_full[1].decode()))
+        else:
+            microseconds = 0
+
+        val = datetime.datetime(year, month, day, hours, minutes, seconds, microseconds)
+    else:
+        # timestamp stored as seconds since epoch, convert to utc
+        val = datetime.datetime.utcfromtimestamp(float(val))
+
+    return val
+
+sqlite3.register_converter('timestamp', convert_timestamp)
+
 class AggMetadata(object):
     """ store some metadata needed to keep track of parse progress
     """
@@ -384,3 +411,32 @@ class BaseFlowAggregator(object):
             cur.close()
 
         return result
+
+    def get_data(self, start_time, end_time):
+        """ get detail data
+        :param start_time: start timestamp
+        :param end_time: end timestamp
+        :return: iterator
+        """
+        query_params = {}
+        query_params['start_time'] = self._parse_timestamp((int(start_time/self.resolution))*self.resolution)
+        query_params['end_time'] = self._parse_timestamp(end_time)
+        sql_select = 'select mtime start_time, '
+        sql_select += '%s, octets, packets, last_seen as "last_seen [timestamp]"  \n' % ','.join(self.agg_fields)
+        sql_select += 'from timeserie \n'
+        sql_select += 'where mtime >= :start_time and mtime < :end_time\n'
+        cur = self._db_connection.cursor()
+        cur.execute(sql_select, query_params)
+
+        # fetch all data, to a max of [max_hits] rows.
+        field_names = (map(lambda x:x[0], cur.description))
+        while True:
+            record = cur.fetchone()
+            if record is None:
+                break
+            else:
+                result_record=dict()
+                for field_indx in range(len(field_names)):
+                    if len(record) > field_indx:
+                        result_record[field_names[field_indx]] = record[field_indx]
+                yield result_record

src/opnsense/service/conf/actions.d/actions_netflow.conf

@@ -85,3 +85,9 @@ command:/usr/local/opnsense/scripts/netflow/get_top_usage.py
 parameters:/provider %s /start_time %s /end_time %s /key_fields %s /value_field %s /filter %s /max_hits %s
 type:script_output
 message:request netflow data aggregator top usage for %s
+
+[aggregate.export]
+command:/usr/local/opnsense/scripts/netflow/export_details.py
+parameters:/provider %s /start_time %s /end_time %s /resolution %s
+type:script_output
+message:export netflow data aggregator details for %s