(SystemHealth) work in progress

src/opnsense/mvc/app/controllers/OPNsense/SystemHealth/Api/ServiceController.php → src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/SystemHealthController.php

@@ -28,7 +28,7 @@
  */
 
 
-namespace OPNsense\SystemHealth\Api;
+namespace OPNsense\Diagnostics\Api;
 
 use \OPNsense\Base\ApiControllerBase;
 use \OPNsense\Core\Backend;
@@ -36,141 +36,9 @@ use \OPNsense\Core\Backend;
  * Class ServiceController
  * @package OPNsense\SystemHealth
  */
-class ServiceController extends ApiControllerBase
+class SystemHealthController extends ApiControllerBase
 {
 
-    /**
-     * retrieve Available RRD data
-     * @return array
-     */
-    public function getRRDlistAction()
-    {
-        # Suurce of data: filelisting of /var/db/rrd/*.rrd
-        $result = array();
-
-        $output = [
-            "system" => ["processor","states", "mbuf"],
-            "traffic" => ["lan", "wan", "ipsec"],
-            "packets" => ["wan", "lan", "ipsec"],
-            "quality" => ["GW_WAN"]
-
-        ];
-        $result["result"] = "ok";
-        $result["data"] = $output;
-
-        // Category => Items
-        return $result;
-    }
-
-
-    private function getRRDdetails($rrd = "")
-    {
-        # Source of data: xml fields of corresponding .xml metadata
-        $result = array();
-        if ($rrd == "system-processor") {
-            $backend = new Backend();
-            $response = $backend->configdpRun("systemhealth query details"); #, array(1, 0, "filepos/".$id, $fileid));
-            // $response = $backend->configdRun("configd actions");
-            $xml=simplexml_load_string($response);
-            $json = json_encode($xml);
-            $output= json_decode($json, true);
-//            $output = [
-//                "title" => "System Information - Utilization and Processes",
-//                "x-axis_label" => "[U]tilization ,  [#]Number",
-//                "field_units" => [
-//                    "user" => "[U]",
-//                    "nice" => "[U]",
-//                    "system" => "[U]",
-//                    "interrupt" => "[#]",
-//                    "processes" => "[#]"
-//                ]
-//            ];
-            $result["result"] = "ok";
-        } else {
-            $result["result"] = "not found";
-            $output=["title"=>"","x-axis_label"=>"","field_units"=>[]]; // always return a valid (empty) data set
-        }
-
-        $result["data"] = $output;
-
-        return $result;
-    }
-    /**
-     * retrieve SystemHealth Data (previously called RRD Graphs)
-     * @param string $rrd
-     * @param int $from
-     * @param int $to
-     * @param int $max_values
-     * @param bool $inverse
-     * @param int $detail
-     * @return array
-     */
-    public function getSystemHealthAction(
-        $rrd = "",
-        $from = 0,
-        $to = 0,
-        $max_values = 120,
-        $inverse = false,
-        $detail = -1
-    ) {
-        /**
-         * $rrd = rrd filename without extension
-         * $from = from timestamp (0=min)
-         * $to = to timestamp (0=max)
-         * $max_values = limit datapoint as close as possible to this number (or twice if detail (zoom) + overview )
-         * $inverse = Inverse every odd row (multiply by -1)
-         * $detail = limits processing of dataSets to max given (-1 = all ; 1 = 0,1 ; 2 = 0,1,2 ; etc)
-         */
-
-        $rrd_details=$this->getRRDdetails($rrd)["data"];
-
-        $rrd = $rrd . ".xml"; // Test data
-        $xml = $this->getXMLdata($rrd);
-
-
-        $data_sets_full = $this->getDataSetInfo($xml); // get dataSet information to include in answer
-
-        if ($inverse == 'true') {
-            $inverse = true;
-        } else {
-            $inverse = false;
-        }
-
-        if ((int)$detail >= 0) {
-            for ($count = count($xml->rra); $count > $detail; $count--) {
-                unset($xml->rra[$count]);
-            }
-        }
-
-        // determine available dataSets within range and how to handle them
-        $selected_archives = $this->getSelection($this->getDataSetInfo($xml), $from, $to, $max_values);
-        // get condensed dataSets and translate them to d3 usable data
-        $result = $this->translateD3(
-            $this->getCondensedArchive($xml, $selected_archives),
-            $inverse,
-            $rrd_details["field_units"]
-        );
-
-        return ["sets" => $data_sets_full,
-            "d3" => $result,
-            "title"=>$rrd_details["title"],
-            "x-axis_label"=>$rrd_details["x-axis_label"]
-        ]; // return details and d3 data
-    }
-
-    /**
-     * Return XML data dump for given rrd
-     * @param $rrd
-     * @return \SimpleXMLElement
-     */
-    private function getXMLdata($rrd)
-    {
-        # Source: rrdtool dump filename.rrd
-        $xml = simplexml_load_file(__DIR__ . '/../../../../../../../../../../opnsense_gui/test/conf/' . $rrd);
-
-        return $xml;
-    }
-
     /**
      * Return full archive information
      * @param array $xml
@@ -364,7 +232,7 @@ class ServiceController extends ApiControllerBase
      * @param boolean $applyInverse
      * @return array
      */
-    private function translateD3($data = array(), $applyInverse = false, $field_units)
+    private function translateD3($data, $applyInverse, $field_units)
     {
         $d3_data = array();
         $from_timestamp = 0;
@@ -448,7 +316,13 @@ class ServiceController extends ApiControllerBase
         ];
     }
 
-    private function getCondensedArchive($xml = array(), $selection = array())
+    /**
+     * retrieve rrd data
+     * @param array $xml
+     * @param array $selection
+     * @return array
+     */
+    private function getCondensedArchive($xml, $selection)
     {
         $key_counter = 0;
         $info = $this->getDataSetInfo($xml);
@@ -584,4 +458,138 @@ class ServiceController extends ApiControllerBase
     {
         return $a[0] - $b[0];
     }
+
+    /**
+     * retrieve descriptive details of rrd
+     * @param string $rrd rrd filename
+     * @return array result status and data
+     */
+    private function getRRDdetails($rrd)
+    {
+        # Source of data: xml fields of corresponding .xml metadata
+        $result = array();
+        $backend = new Backend();
+        $response = $backend->configdpRun("systemhealth list");
+        $output= json_decode($response, true);
+        if (is_array($output) && array_key_exists($rrd, $output)) {
+            $result["result"] = "ok";
+            $result["data"] = $output[$rrd];
+        } else {
+            // always return a valid (empty) data set
+            $result["result"] = "not found";
+            $result["data"] = ["title"=>"","y-axis_label"=>"","field_units"=>[]];
+        }
+        return $result;
+    }
+
+
+    /**
+     * retrieve Available RRD data
+     * @return array
+     */
+    public function getRRDlistAction()
+    {
+        # Suurce of data: filelisting of /var/db/rrd/*.rrd
+        $result = array();
+        $backend = new Backend();
+        $response = $backend->configdpRun("systemhealth list");
+        $healthList = json_decode($response, true);
+
+        $result['data'] = array();
+        if (is_array($healthList)) {
+            foreach ($healthList as $healthItem => $details) {
+                if (!array_key_exists($details['topic'], $result['data'])) {
+                    $result['data'][$details['topic']] = array();
+                }
+                $result['data'][$details['topic']][] = $details['itemName'];
+            }
+        }
+        ksort($result['data']);
+        $output = [
+            "system" => ["processor","states", "mbuf"],
+            "traffic" => ["lan", "wan", "ipsec"],
+            "packets" => ["wan", "lan", "ipsec"],
+            "quality" => ["GW_WAN"]
+
+        ];
+        $result["result"] = "ok";
+        $result["data2"] = $output;
+
+        // Category => Items
+        return $result;
+    }
+
+    /**
+     * retrieve SystemHealth Data (previously called RRD Graphs)
+     * @param string $rrd
+     * @param int $from
+     * @param int $to
+     * @param int $max_values
+     * @param bool $inverse
+     * @param int $detail
+     * @return array
+     */
+    public function getSystemHealthAction(
+        $rrd = "",
+        $from = 0,
+        $to = 0,
+        $max_values = 120,
+        $inverse = false,
+        $detail = -1
+    ) {
+        /**
+         * $rrd = rrd filename without extension
+         * $from = from timestamp (0=min)
+         * $to = to timestamp (0=max)
+         * $max_values = limit datapoint as close as possible to this number (or twice if detail (zoom) + overview )
+         * $inverse = Inverse every odd row (multiply by -1)
+         * $detail = limits processing of dataSets to max given (-1 = all ; 1 = 0,1 ; 2 = 0,1,2 ; etc)
+         */
+
+        $rrd_details=$this->getRRDdetails($rrd)["data"];
+
+        $backend = new Backend();
+        $response = $backend->configdpRun("systemhealth fetch ", array($rrd));
+        $xml = simplexml_load_string($response);
+
+        if ($xml !== false) {
+            // we only use the average databases in any RRD, remove the rest to avoid strange behaviour.
+            for ($count = count($xml->rra) -1; $count >= 0; $count--) {
+                if ((string)$xml->rra[$count]->cf != "AVERAGE") {
+                    unset($xml->rra[$count]);
+                }
+
+            }
+            $data_sets_full = $this->getDataSetInfo($xml); // get dataSet information to include in answer
+
+            if ($inverse == 'true') {
+                $inverse = true;
+            } else {
+                $inverse = false;
+            }
+
+            if ((int)$detail >= 0) {
+                for ($count = count($xml->rra) - 1; $count > $detail; $count--) {
+                    unset($xml->rra[$count]);
+                }
+            }
+
+            // determine available dataSets within range and how to handle them
+            $selected_archives = $this->getSelection($this->getDataSetInfo($xml), $from, $to, $max_values);
+            // get condensed dataSets and translate them to d3 usable data
+            $result = $this->translateD3(
+                $this->getCondensedArchive($xml, $selected_archives),
+                $inverse,
+                $rrd_details["field_units"]
+            );
+
+            return ["sets" => $data_sets_full,
+                "d3" => $result,
+                "title"=>$rrd_details["title"],
+                "y-axis_label"=>$rrd_details["y-axis_label"]
+            ]; // return details and d3 data
+        } else {
+            return ["sets" => [], "d3" => [], "title" => "error", "y-axis_label" => ""];
+        }
+    }
 }

src/opnsense/mvc/app/controllers/OPNsense/SystemHealth/IndexController.php → src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/SystemHealthController.php

@@ -26,18 +26,18 @@
  *    POSSIBILITY OF SUCH DAMAGE.
  *
  */
-namespace OPNsense\SystemHealth;
+namespace OPNsense\Diagnostics;
 
 /**
  * Class IndexController
  * @package OPNsense\Proxy
  */
-class IndexController extends \OPNsense\Base\IndexController
+class SystemHealthController extends \OPNsense\Base\IndexController
 {
     public function indexAction()
     {
         $this->view->title = "System Health";
-        $this->view->pick('OPNsense/SystemHealth/index');
+        $this->view->pick('OPNsense/Diagnostics/systemhealth');
     }
 }
 

src/opnsense/mvc/app/controllers/OPNsense/SystemHealth/Api/SettingsController.php

-<?php
-/**
- *    Copyright (C) 2015  Deciso B.V. - J. Schellevis
- *
- *    All rights reserved.
- *
- *    Redistribution and use in source and binary forms, with or without
- *    modification, are permitted provided that the following conditions are met:
- *
- *    1. Redistributions of source code must retain the above copyright notice,
- *       this list of conditions and the following disclaimer.
- *
- *    2. Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *
- *    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
- *    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
- *    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- *    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
- *    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- *    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- *    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- *    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- *    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- *    POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-
-namespace OPNsense\SystemHealth\Api;
-
-use \OPNsense\Base\ApiControllerBase;
-
-/**
- * Class SettingsController
- * @package OPNsense\SystemHealth
- */
-class SettingsController extends ApiControllerBase
-{
-
-}
-
-?>

src/opnsense/mvc/app/views/OPNsense/SystemHealth/index.volt → src/opnsense/mvc/app/views/OPNsense/Diagnostics/systemhealth.volt

@@ -172,7 +172,7 @@
     }
 
     function getRRDlist() {
-        ajaxGet(url = "/api/systemhealth/service/getRRDlist/", sendData = {}, callback = function (data, status) {
+        ajaxGet(url = "/api/diagnostics/systemhealth/getRRDlist/", sendData = {}, callback = function (data, status) {
             if (status == "success") {
                 var category;
                 var tabs="";
@@ -293,7 +293,7 @@
         // info bar - show loading info bar while refreshing data
         $('#loading').show();
         // API call to request data
-        ajaxGet(url = "/api/systemhealth/service/getSystemHealth/" + rrd_name + "/" + String(from) + "/" + String(to) + "/" + String(maxitems) + "/" + String(inverse) + "/" + String(detail), sendData = {}, callback = function (data, status) {
+        ajaxGet(url = "/api/diagnostics/systemhealth/getSystemHealth/" + rrd_name + "/" + String(from) + "/" + String(to) + "/" + String(maxitems) + "/" + String(inverse) + "/" + String(detail), sendData = {}, callback = function (data, status) {
             if (status == "success") {
                 var stepsize = data["d3"]["stepSize"];
                 var scale = "{{ lang._('seconds') }}";
@@ -473,7 +473,7 @@
                         .tickFormat(function (d) {
                             return d3.time.format(dtformat)(new Date(d))
                         });
-                chart.yAxis.axisLabel(data["x-axis_label"]);
+                chart.yAxis.axisLabel(data["y-axis_label"]);
                 chart.useInteractiveGuideline(true);
                 chart.interactive(true);
 

src/opnsense/scripts/systemhealth/definitions/interface-packets.xml

+<?xml version="1.0"?>
+<systemhealth>
+    <file-match>.*-packets$</file-match>
+    <title>System Information - Packets </title>
+    <y-axis_label>Packets/Second</y-axis_label>
+</systemhealth>
\ No newline at end of file

src/opnsense/scripts/systemhealth/metadata/system-processor.xml → src/opnsense/scripts/systemhealth/definitions/system-processor.xml

 <?xml version="1.0"?>
-<details>
+<systemhealth>
+    <file-match>system-processor</file-match>
     <title> System Information - Utilization and Processes</title>
-    <x-axis_label>[U]tilization ,  [#]Number</x-axis_label>
+    <y-axis_label>[U]tilization ,  [#]Number</y-axis_label>
     <field_units>
         <user>[U]</user>
         <nice>[U]</nice>
@@ -9,4 +10,4 @@
         <interrupt>[#]</interrupt>
         <processes>[#]</processes>
     </field_units>
-</details>
+</systemhealth>
\ No newline at end of file

src/opnsense/scripts/systemhealth/queryDetails.py → src/opnsense/scripts/systemhealth/fetchData.py

 #!/usr/local/bin/python2.7
 
 """
-    Copyright (c) 2015 Deciso B.V. - J. Schellevis
+    Copyright (c) 2015 Deciso B.V. - Ad Schellevis
     All rights reserved.
 
     Redistribution and use in source and binary forms, with or without
@@ -26,10 +26,29 @@
     POSSIBILITY OF SUCH DAMAGE.
 
     --------------------------------------------------------------------------------------
+    fetch xmldata from rrd tool, but only if filename is valid (with or without .rrd extension)
 
 """
+import sys
+import glob
+import tempfile
+import subprocess
+import os.path
+
+rrd_reports_dir = '/var/db/rrd'
+if len(sys.argv) > 1:
+    filename = sys.argv[1]
+    # suffix rrd if not already in request
+    if filename.split('.')[-1] != 'rrd':
+        filename += '.rrd'
+
+    # scan rrd directory for requested file
+    for rrdFilename in glob.glob('%s/*.rrd' % rrd_reports_dir):
+        if os.path.basename(rrdFilename) == filename:
+            with tempfile.NamedTemporaryFile() as output_stream:
+                subprocess.check_call(['/usr/local/bin/rrdtool', 'dump', rrdFilename],
+                                                      stdout=output_stream, stderr=subprocess.STDOUT)
+                output_stream.seek(0)
+                print (output_stream.read())
+            break
 
-file = open('/Users/josschellevis/Development/opnsense/scripts/systemhealth/metadata/system-processor.xml', 'r')
-file_contents = file.read()
-print (file_contents)
-file.close()

src/opnsense/scripts/systemhealth/listReports.py

+#!/usr/local/bin/python2.7
+
+"""
+    Copyright (c) 2015 Deciso B.V. - Ad Schellevis
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+    return a list of all available rrd files including additional definition data
+
+"""
+import re
+import glob
+import xml.etree.ElementTree
+import os.path
+import ujson
+
+rrd_definition_dir = '%s/definitions' % os.path.dirname(os.path.abspath(__file__))
+rrd_reports_dir = '/var/db/rrd'
+
+# query all rrd files available and initialize with empty definition
+result = {}
+for filename in glob.glob('%s/*.rrd' % rrd_reports_dir):
+    rrdFilename = os.path.basename(filename).split('.')[0]
+    # determine topic and item name, fixes naming issues
+    if rrdFilename.split('-')[0] in ('system') and rrdFilename.find('-') > -1:
+        topic = rrdFilename.split('-')[0]
+        itemName = '-'.join(rrdFilename.split('-')[1:])
+    else:
+        topic = rrdFilename.split('-')[-1]
+        itemName = '-'.join(rrdFilename.split('-')[:-1])
+    result[rrdFilename] = {'title': '', 'y-axis_label': '', 'field_units': {}, 'topic': topic, 'itemName': itemName}
+
+# scan all definition files
+for filename in glob.glob('%s/*.xml' % rrd_definition_dir):
+    rrdFilename = os.path.basename(filename).split('.')[0]
+    try:
+        ruleXML=xml.etree.ElementTree.fromstring(open(filename).read())
+        rrdDef = {}
+        if ruleXML.tag == 'systemhealth':
+            # only parse systemhealth items
+            for child in ruleXML:
+                if len(list(child)) == 0:
+                    # single items
+                    rrdDef[child.tag] = child.text
+                else:
+                    # named list items
+                    rrdDef[child.tag] = {}
+                    for subchild in child:
+                        rrdDef[child.tag][subchild.tag] = subchild.text
+        else:
+            rrdDef['__msg'] = 'xml metadata not valid'
+    except xml.etree.ElementTree.ParseError:
+        # no valid xml, always return a valid (empty) data set
+        rrdDef = {'__msg': 'xml metadata load error'}
+
+    # link definition data to rrd file, use property file-match in xml to determine it's targets
+    if 'file-match' in rrdDef:
+        # remove file-match property from actual result
+        file_match = rrdDef['file-match']
+        del rrdDef['file-match']
+        for rrdFilename in result:
+            if re.search(file_match, rrdFilename) is not None:
+                for fieldname in rrdDef:
+                    result[rrdFilename][fieldname] = rrdDef[fieldname]
+
+print(ujson.dumps(result))
+