Skip to content

O
OpnSense
Commit 3f9c8f32 authored by Ad Schellevis's avatar Ad Schellevis
Browse files
Options

(network insight) add aggregation for source addresses and destination ports + add ui template

parent 0b198f7e
Hide whitespace changes
Inline Side-by-side
Showing with 411 additions and 100 deletions
src/opnsense/mvc/app/views/OPNsense/Diagnostics/networkinsight.volt
View file @ 3f9c8f32
......@@ -53,52 +53,126 @@ POSSIBILITY OF SUCH DAMAGE.
});
// collect all chars for resize update
var pageCharts = [];
var pageCharts = {};
// human readable interface names
// form metadata definitions
var interface_names = [];
var service_names = [];
var protocol_names = [];
function interface_totals(target, from_date, to_date, resolution, direction) {
var dfObj = new $.Deferred();
if (direction != 'in' && direction != 'out') {
fetch_params = from_date + '/' + to_date + '/' + resolution + '/if' ;
} else {
fetch_params = from_date + '/' + to_date + '/' + resolution + '/if,direction' ;
}
ajaxGet('/api/diagnostics/networkinsight/timeserie/FlowInterfaceTotals/octets_ps/' + fetch_params,{},function(data,status){
nv.addGraph(function() {
var chart = nv.models.stackedAreaChart()
.x(function(d) { return d[0] })
.y(function(d) { return d[1] })
.useInteractiveGuideline(true)
.interactive(true)
.showControls(true)
.clipEdge(true)
;
if (resolution < 60) {
chart.xAxis.tickSize(8).tickFormat(function(d) {
return d3.time.format('%b %e %H:%M:%S')(new Date(d));
});
} else if (resolution < 3600) {
chart.xAxis.tickSize(16).tickFormat(function(d) {
return d3.time.format('%b %e %H:%M')(new Date(d));
});
} else if (resolution < 86400) {
chart.xAxis.tickSize(16).tickFormat(function(d) {
return d3.time.format('%b %e %H h')(new Date(d));
/**
* load shared metadata (interfaces, protocols, )
*/
function get_metadata()
{
var dfObj = new $.Deferred();
// fetch interface names
ajaxGet('/api/diagnostics/networkinsight/getInterfaces',{},function(intf_names,status){
interface_names = intf_names;
// fetch protocol names
ajaxGet('/api/diagnostics/networkinsight/getProtocols',{}, function(protocols, status) {
protocol_names = protocols;
// fetch service names
ajaxGet('/api/diagnostics/networkinsight/getServices',{}, function(services, status) {
service_names = services;
dfObj.resolve();
});
} else {
chart.xAxis.tickFormat(function(d) {
return d3.time.format('%b %e')(new Date(d));
});
}
chart.yAxis.tickFormat(d3.format(',.2s'));
});
});
return dfObj;
}
/**
* delete all graps accounted in pageCharts
*/
function delete_all_charts()
{
var svg = d3.select("svg");
svg.selectAll("*").remove();
pageCharts = {};
}
/**
* get selected time
*/
function get_time_select()
{
// current time stamp
var timestamp_now = Math.round((new Date()).getTime() / 1000);
var duration = 0;
var resolution = 0;
switch ($("#total_time_select").val()) {
case "2h":
duration = 60*60*2;
resolution = 30;
break;
case "8h":
duration = 60*60*8;
resolution = 300;
break;
case "1w":
duration = 60*60*24*7;
resolution = 3600;
break;
case "1m":
duration = 60*60*24*31;
resolution = 86400;
break;
case "1y":
duration = 60*60*24*365;
resolution = 86400;
break;
}
// always round from timestamp to nearest hour
from_timestamp = Math.floor((timestamp_now -duration) / 3600 ) * 3600;
return {resolution: resolution, from: from_timestamp, to: timestamp_now};
}
chart_data = [];
data.map(function(item){
if (direction != undefined) {
/**
* draw interface totals
*/
function chart_interface_totals() {
var selected_time = get_time_select();
fetch_params = selected_time.from + '/' + selected_time.to + '/' + selected_time.resolution + '/if,direction' ;
ajaxGet('/api/diagnostics/networkinsight/timeserie/FlowInterfaceTotals/bps/' + fetch_params,{},function(data,status){
$.each(['chart_intf_in', 'chart_intf_out'], function(idx, target) {
var direction = '';
if (target == 'chart_intf_in') {
direction = 'in';
} else {
direction = 'out';
}
nv.addGraph(function() {
var chart = nv.models.stackedAreaChart()
.x(function(d) { return d[0] })
.y(function(d) { return d[1] })
.useInteractiveGuideline(true)
.interactive(true)
.showControls(true)
.clipEdge(true)
;
if (selected_time.resolution < 60) {
chart.xAxis.tickSize(8).tickFormat(function(d) {
return d3.time.format('%b %e %H:%M:%S')(new Date(d));
});
} else if (selected_time.resolution < 3600) {
chart.xAxis.tickSize(8).tickFormat(function(d) {
return d3.time.format('%b %e %H:%M')(new Date(d));
});
} else if (selected_time.resolution < 86400) {
chart.xAxis.tickSize(8).tickFormat(function(d) {
return d3.time.format('%b %e %H h')(new Date(d));
});
} else {
chart.xAxis.tickFormat(function(d) {
return d3.time.format('%b %e')(new Date(d));
});
}
chart.yAxis.tickFormat(d3.format(',.2s'));
chart_data = [];
data.map(function(item){
item_dir = item.key.split(',').pop();
item_intf = item.key.split(',')[0];
if (item_intf != '0' && item_intf != 'lo0' ) {
......@@ -111,75 +185,146 @@ POSSIBILITY OF SUCH DAMAGE.
chart_data.push(item);
}
}
} else {
if (item.key != '0' && item.key != 'lo0' ) {
chart_data.push(item);
}
}
});
chart_data.sort(function(a, b) {
return a.key > b.key;
});
d3.select("#" + target + " svg").datum(chart_data).call(chart);
pageCharts[target] = chart;
return chart;
});
});
});
}
/**
* graph top usage per destination port for selected time period
*/
function chart_top_dst_port_usage()
{
// remove existing chart
if (pageCharts["chart_top_ports"] != undefined) {
var svg = d3.select("#chart_top_ports > svg");
svg.selectAll("*").remove();
}
var selected_time = get_time_select();
var time_url = selected_time.from + '/' + selected_time.to;
ajaxGet('/api/diagnostics/networkinsight/top/FlowDstPortTotals/'+time_url+'/dst_port,protocol/octets/10/',
{'filter_field': 'if', 'filter_value': $('#interface_select').val()}, function(data, status){
if (status == 'success'){
nv.addGraph(function() {
var chart = nv.models.pieChart()
.x(function(d) { return d.label })
.y(function(d) { return d.value })
.showLabels(true)
.labelThreshold(.05)
.labelType("percent")
.donut(true)
.donutRatio(0.35)
.legendPosition("right")
.valueFormat(d3.format(',.2s'));
;
chart_data.sort(function(a, b) {
return a.key > b.key;
chart_data = [];
data.map(function(item){
var label = "(other)";
var proto = "";
if (item.protocol != "") {
if (item.protocol in protocol_names) {
proto = ' (' + protocol_names[item.protocol] + ')';
}
if (item.dst_port in service_names) {
label = service_names[item.dst_port];
} else {
label = item.dst_port
}
}
chart_data.push({'label': label + proto, 'value': item.total});
});
d3.select("#chart_top_ports svg")
.datum(chart_data)
.transition().duration(350)
.call(chart);
pageCharts["chart_top_ports"] = chart;
return chart;
});
}
});
}
d3.select("#" + target + " svg")
.datum(chart_data)
.call(chart);
/**
* graph top usage per destination port for selected time period
*/
function chart_top_src_addr_usage()
{
// remove existing chart
if (pageCharts["chart_top_sources"] != undefined) {
var svg = d3.select("#chart_top_sources > svg");
svg.selectAll("*").remove();
}
var selected_time = get_time_select();
var time_url = selected_time.from + '/' + selected_time.to;
ajaxGet('/api/diagnostics/networkinsight/top/FlowSourceAddrTotals/'+time_url+'/src_addr/octets/10/',
{'filter_field': 'if', 'filter_value': $('#interface_select').val()}, function(data, status){
if (status == 'success'){
nv.addGraph(function() {
var chart = nv.models.pieChart()
.x(function(d) { return d.label })
.y(function(d) { return d.value })
.showLabels(true)
.labelThreshold(.05)
.labelType("percent")
.donut(true)
.donutRatio(0.35)
.legendPosition("right")
.valueFormat(d3.format(',.2s'));
pageCharts.push(chart);
return chart;
}, function(){
// wait some time before resolving promise, easier for some browsers to handle the amount of data
setTimeout(function() {
dfObj.resolve();
}, 1000);
});
chart_data = [];
data.map(function(item){
var label = "(other)";
if (item.src_addr != "") {
label = item.src_addr;
}
chart_data.push({'label': label, 'value': item.total});
});
d3.select("#chart_top_sources svg")
.datum(chart_data)
.transition().duration(350)
.call(chart);
pageCharts["chart_top_sources"] = chart;
return chart;
});
}
});
return dfObj;
}
// hide heading
$(".page-content-head").addClass("hidden");
// event page resize
$(window).on('resize-end', function() {
pageCharts.forEach(function(chart) {
chart.update();
$.each(pageCharts, function(idx, chart) {
chart.update();
});
});
// change time select
// event change time select
$("#total_time_select").change(function(){
// current time stamp
var timestamp_now = Math.round((new Date()).getTime() / 1000);
var duration = 0;
var resolution = 0;
switch ($(this).val()) {
case "2h":
duration = 60*60*2;
resolution = 30;
break;
case "8h":
duration = 60*60*8;
resolution = 300;
break;
case "1w":
duration = 60*60*24*7;
resolution = 3600;
break;
case "1y":
duration = 60*60*24*365;
resolution = 86400;
break;
}
if (resolution != 0) {
// remove all charts
var svg = d3.select("svg");
svg.selectAll("*").remove();
pageCharts = [];
// fetch interface names
ajaxGet('/api/diagnostics/networkinsight/getInterfaces',{},function(intf_names,status){
interface_names = intf_names;
interface_totals('chart_intf_in', timestamp_now - duration, timestamp_now, resolution, 'in');
interface_totals('chart_intf_out', timestamp_now - duration, timestamp_now, resolution, 'out');
});
}
delete_all_charts();
chart_interface_totals();
chart_top_dst_port_usage();
chart_top_src_addr_usage();
});
// event change interface selection
$('#interface_select').change(function(){
chart_top_dst_port_usage();
chart_top_src_addr_usage();
});
$('a[data-toggle="tab"]').on('shown.bs.tab', function (e) {
......@@ -191,7 +336,16 @@ POSSIBILITY OF SUCH DAMAGE.
});
// trigger initial tab load
$("#total_time_select").change();
get_metadata().done(function(){
// known interfaces
for (var key in interface_names) {
$('#interface_select').append($("<option></option>").attr("value",key).text(interface_names[key]));
}
$('#interface_select').selectpicker('refresh');
chart_interface_totals();
chart_top_dst_port_usage();
chart_top_src_addr_usage();
});
});
</script>
......@@ -207,6 +361,7 @@ POSSIBILITY OF SUCH DAMAGE.
<option value="2h">{{ lang._('Last 2 hours, 30 second average') }}</option>
<option value="8h">{{ lang._('Last 8 hours, 5 minute average') }}</option>
<option value="1w">{{ lang._('Last week, 1 hour average') }}</option>
<option value="1m">{{ lang._('Last month, 24 hour average') }}</option>
<option value="1y">{{ lang._('Last year, 24 hour average') }}</option>
</select>
</div>
......@@ -214,7 +369,7 @@ POSSIBILITY OF SUCH DAMAGE.
<br/>
<div class="panel panel-primary">
<div class="panel-heading panel-heading-sm">
{{ lang._('Interface totals') }}
{{ lang._('Interface totals (bits/sec)') }}
</div>
<div class="panel-body">
<div id="chart_intf_in">
......@@ -229,9 +384,25 @@ POSSIBILITY OF SUCH DAMAGE.
</div>
<div class="panel panel-primary">
<div class="panel-heading panel-heading-sm">
{{ lang._('') }}
{{ lang._('Top usage ports / sources (bytes)') }}
</div>
<div class="panel-body">
<div class="row">
<div class="col-xs-12">
<select class="selectpicker" id="interface_select">
</select>
</div>
<div class="col-xs-12 col-sm-6">
<div id="chart_top_ports">
<svg style="height:300px;"></svg>
</div>
</div>
<div class="col-xs-12 col-sm-6">
<div id="chart_top_sources">
<svg style="height:300px;"></svg>
</div>
</div>
</div>
</div>
</div>
</div>
......
src/opnsense/scripts/netflow/lib/aggregates/ports.py 0 → 100644
View file @ 3f9c8f32
"""
Copyright (c) 2016 Ad Schellevis
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------------
data aggregator type
"""
from lib.aggregate import BaseFlowAggregator
class FlowDstPortTotals(BaseFlowAggregator):
""" collect interface totals
"""
target_filename = '/var/netflow/dst_port_%06d.sqlite'
agg_fields = ['if', 'protocol', 'dst_port']
@classmethod
def resolutions(cls):
"""
:return: list of sample resolutions
"""
return [30, 300, 3600, 86400]
@classmethod
def history_per_resolution(cls):
"""
:return: dict sample resolution / expire time (seconds)
"""
# only save daily totals for a longer period of time, we probably only want to answer questions like
# "top usage over the last 30 seconds, 5 minutes, etc.."
return {30: 300,
300: 3600,
3600: 86400,
86400: cls.seconds_per_day(365)
}
def __init__(self, resolution):
"""
:param resolution: sample resultion (seconds)
:return: None
"""
super(FlowDstPortTotals, self).__init__(resolution)
def add(self, flow):
# most likely service (destination) port
flow['dst_port'] = min(flow['dst_port'], flow['src_port'])
flow['if'] = flow['if_in']
super(FlowDstPortTotals, self).add(flow)
flow['if'] = flow['if_out']
super(FlowDstPortTotals, self).add(flow)
src/opnsense/scripts/netflow/lib/aggregates/source.py 0 → 100644
View file @ 3f9c8f32
"""
Copyright (c) 2016 Ad Schellevis
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------------
data aggregator type
"""
from lib.aggregate import BaseFlowAggregator
class FlowSourceAddrTotals(BaseFlowAggregator):
""" collect interface totals
"""
target_filename = '/var/netflow/src_addr_%06d.sqlite'
agg_fields = ['if', 'src_addr']
@classmethod
def resolutions(cls):
"""
:return: list of sample resolutions
"""
return [30, 300, 3600, 86400]
@classmethod
def history_per_resolution(cls):
"""
:return: dict sample resolution / expire time (seconds)
"""
# only save daily totals for a longer period of time, we probably only want to answer questions like
# "top usage over the last 30 seconds, 5 minutes, etc.."
return {30: 300,
300: 3600,
3600: 86400,
86400: cls.seconds_per_day(365)
}
def __init__(self, resolution):
"""
:param resolution: sample resultion (seconds)
:return: None
"""
super(FlowSourceAddrTotals, self).__init__(resolution)
def add(self, flow):
# most likely service (destination) port
flow['if'] = flow['if_in']
super(FlowSourceAddrTotals, self).add(flow)
flow['src_addr'] = flow['dst_addr']
flow['if'] = flow['if_out']
super(FlowSourceAddrTotals, self).add(flow)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Technounit-GROUP 2023