(ids) add support for inline configuration settings (subscription based url's...

(ids) add support for inline configuration settings (subscription based url's for example), add basic auth support.

Example supported format:

<?xml version="1.0"?>
<ruleset>
    <location url="https://www.snort.org/rules/snortrules-snapshot-2990.tar.gz?oinkcode=%%snort.oinkcode%%" prefix="Snort"/>
    <files>
        <file description="blacklist" url="inline::rules/blacklist.rules">snort.blacklist.rules</file>
    </files>
    <properties>
        <property name="snort.oinkcode" default=""/>
    </properties>
</ruleset>

---
Registers the setting "snort.oinkcode" which is used to construct the download url.
This commit doesn't include definitions for new content, in case someone wants to create a definition file, it should be easy now :)

src/opnsense/mvc/app/controllers/OPNsense/IDS/Api/SettingsController.php

@@ -266,6 +266,76 @@ class SettingsController extends ApiMutableModelControllerBase
         return $result;
     }
 
+    /**
+     * list ruleset properties
+     * @return array
+     */
+    public function getRulesetpropertiesAction()
+    {
+        $result = array('properties' => array());
+        $backend = new Backend();
+        $response = $backend->configdRun("ids list installablerulesets");
+        $data = json_decode($response, true);
+        if ($data != null && isset($data["properties"])) {
+            foreach ($data['properties'] as $key => $settings) {
+                $result['properties'][$key] = !empty($settings['default']) ? $settings['default'] : "";
+                foreach ($this->getModel()->fileTags->tag->__items as $tag) {
+                    if ((string)$tag->property == $key) {
+                        $result['properties'][(string)$tag->property] = (string)$tag->value;
+                    }
+                }
+            }
+        }
+        return $result;
+    }
+
+    /**
+     * update ruleset properties
+     * @return array
+     */
+    public function setRulesetpropertiesAction()
+    {
+        $result = array("result" => "failed");
+        if ($this->request->isPost() && $this->request->hasPost("properties")) {
+            // only update properties available in "ids list installablerulesets"
+            $backend = new Backend();
+            $response = $backend->configdRun("ids list installablerulesets");
+            $data = json_decode($response, true);
+            if ($data != null && isset($data["properties"])) {
+                $setProperties = $this->request->getPost("properties");
+                foreach ($setProperties as $key => $value) {
+                    if (isset($data['properties'][$key])) {
+                        if (!isset($result['fields'])) {
+                            $result['fields'] = array(); // return updated fields
+                        }
+                        $result['fields'][] = $key;
+                        $resultTag = null;
+                        foreach ($this->getModel()->fileTags->tag->__items as $tag) {
+                            if ((string)$tag->property == $key) {
+                                $resultTag = $tag;
+                                break;
+                            }
+                        }
+                        if ($resultTag == null) {
+                            $resultTag = $this->getModel()->fileTags->tag->Add();
+                        }
+                        $resultTag->property = (string)$key;
+                        $resultTag->value = (string)$value;
+                    }
+                }
+                $validations = $this->getModel()->validate();
+                if (count($validations)) {
+                    $result['validations'] = $validations;
+                } else {
+                    $this->getModel()->serializeToConfig();
+                    Config::getInstance()->save();
+                    $result["result"] = "saved";
+                }
+            }
+        }
+        return $result;
+    }
+
     /**
      * list all installable rules including current status
      * @return array|mixed list of items when $id is null otherwise the selected item is returned

src/opnsense/mvc/app/models/OPNsense/IDS/IDS.xml

@@ -84,6 +84,18 @@
                 </enabled>
             </file>
         </files>
+        <fileTags>
+            <tag type="ArrayField">
+              <property type="TextField">
+                  <Required>Y</Required>
+                  <mask>/^([\t\n\v\f\r\- 0-9a-zA-Z.,_\x{00A0}-\x{FFFF}]){1,255}$/u</mask>
+              </property>
+              <value type="TextField">
+                  <Required>N</Required>
+                  <mask>/^([\t\n\v\f\r\- 0-9a-zA-Z.,_\x{00A0}-\x{FFFF}]){1,255}$/u</mask>
+              </value>
+            </tag>
+        </fileTags>
         <general>
             <enabled type="BooleanField">
                 <default>0</default>

src/opnsense/mvc/app/views/OPNsense/IDS/index.volt

@@ -153,6 +153,7 @@ POSSIBILITY OF SUCH DAMAGE.
                     if (status == "success" || data['status'].toLowerCase().trim() == "ok") {
                         result_status = true;
                     }
+                    $('#scheduled_updates').show();
                     callback_funct(result_status);
                 });
             });
@@ -207,6 +208,65 @@ POSSIBILITY OF SUCH DAMAGE.
         $('a[data-toggle="tab"]').on('shown.bs.tab', function (e) {
             if (e.target.id == 'settings_tab'){
                 loadGeneralSettings();
+            } else if (e.target.id == 'download_settings_tab') {
+                /**
+                 * grid for installable rule files
+                 */
+                $('#grid-rule-files').bootgrid('destroy'); // always destroy previous grid, so data is always fresh
+                $("#grid-rule-files").UIBootgrid({
+                    search:'/api/ids/settings/listRulesets',
+                    get:'/api/ids/settings/getRuleset/',
+                    set:'/api/ids/settings/setRuleset/',
+                    toggle:'/api/ids/settings/toggleRuleset/',
+                    options:{
+                        navigation:0,
+                        formatters:{
+                            rowtoggle: function (column, row) {
+                                var toggle = " <button type=\"button\" class=\"btn btn-xs btn-default command-edit\" data-row-id=\"" + row.filename + "\"><span class=\"fa fa-pencil\"></span></button> ";
+                                if (parseInt(row[column.id], 2) == 1) {
+                                    toggle += "<span style=\"cursor: pointer;\" class=\"fa fa-check-square-o command-toggle\" data-value=\"1\" data-row-id=\"" + row.filename + "\"></span>";
+                                } else {
+                                    toggle += "<span style=\"cursor: pointer;\" class=\"fa fa-square-o command-toggle\" data-value=\"0\" data-row-id=\"" + row.filename + "\"></span>";
+                                }
+                                return toggle;
+                            }
+                        },
+                        converters: {
+                            // show "not installed" for rules without timestamp (not on disc)
+                            rulets: {
+                                from: function (value) {
+                                    return value;
+                                },
+                                to: function (value) {
+                                    if ( value == null ) {
+                                        return "{{ lang._('not installed') }}";
+                                    } else {
+                                        return value;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                });
+                // display file settings (if available)
+                ajaxGet(url="/api/ids/settings/getRulesetproperties", sendData={}, callback=function(data, status) {
+                    if (status == "success") {
+                        var rows = [];
+                        // generate rows with field references
+                        $.each(data['properties'], function(key, value) {
+                            rows.push('<tr><td>'+key+'</td><td><input class="rulesetprop" data-id="'+key+'" type="text"></td></tr>');
+                        });
+                        $("#grid-rule-files-settings > tbody").html(rows.join(''));
+                        // update with data
+                        $(".rulesetprop").each(function(){
+                            $(this).val(data['properties'][$(this).data('id')]);
+                        });
+                        if (rows.length > 0) {
+                            $("#grid-rule-files-settings").parent().parent().show();
+                            $("#updateSettings").show();
+                        }
+                    }
+                });
             } else if (e.target.id == 'rule_tab'){
                 //
                 // activate rule tab page
@@ -275,46 +335,6 @@ POSSIBILITY OF SUCH DAMAGE.
                         toggle:'/api/ids/settings/toggleUserRule/'
                     }
                 );
-            } else if (e.target.id == 'download_settings_tab') {
-                /**
-                 * grid for installable rule files
-                 */
-                $('#grid-rule-files').bootgrid('destroy'); // always destroy previous grid, so data is always fresh
-                $("#grid-rule-files").UIBootgrid({
-                    search:'/api/ids/settings/listRulesets',
-                    get:'/api/ids/settings/getRuleset/',
-                    set:'/api/ids/settings/setRuleset/',
-                    toggle:'/api/ids/settings/toggleRuleset/',
-                    options:{
-                        navigation:0,
-                        formatters:{
-                            rowtoggle: function (column, row) {
-                                var toggle = " <button type=\"button\" class=\"btn btn-xs btn-default command-edit\" data-row-id=\"" + row.filename + "\"><span class=\"fa fa-pencil\"></span></button> ";
-                                if (parseInt(row[column.id], 2) == 1) {
-                                    toggle += "<span style=\"cursor: pointer;\" class=\"fa fa-check-square-o command-toggle\" data-value=\"1\" data-row-id=\"" + row.filename + "\"></span>";
-                                } else {
-                                    toggle += "<span style=\"cursor: pointer;\" class=\"fa fa-square-o command-toggle\" data-value=\"0\" data-row-id=\"" + row.filename + "\"></span>";
-                                }
-                                return toggle;
-                            }
-                        },
-                        converters: {
-                            // show "not installed" for rules without timestamp (not on disc)
-                            rulets: {
-                                from: function (value) {
-                                    return value;
-                                },
-                                to: function (value) {
-                                    if ( value == null ) {
-                                        return "{{ lang._('not installed') }}";
-                                    } else {
-                                        return value;
-                                    }
-                                }
-                            }
-                        }
-                    }
-                });
             }
         })
 
@@ -344,6 +364,16 @@ POSSIBILITY OF SUCH DAMAGE.
                 }
             });
         });
+        $("#updateSettings").click(function(){
+            $("#updateSettings_progress").addClass("fa fa-spinner fa-pulse");
+            var settings = {};
+            $(".rulesetprop").each(function(){
+                settings[$(this).data('id')] = $(this).val();
+            });
+            ajaxCall(url="/api/ids/settings/setRulesetproperties", sendData={'properties': settings}, callback=function(data,status) {
+                $("#updateSettings_progress").removeClass("fa fa-spinner fa-pulse");
+            });
+        });
 
         /**
          * update (userdefined) rules
@@ -365,15 +395,7 @@ POSSIBILITY OF SUCH DAMAGE.
                 // when done, disable progress animation and reload grid.
                 $('#grid-rule-files').bootgrid('reload');
                 updateStatus();
-                if ($('#scheduled_updates').is(':hidden') ){
-                    // save and reconfigure on initial download (tries to create a cron job)
-                    actionReconfigure(function(status){
-                        loadGeneralSettings();
-                        $("#updateRulesAct_progress").removeClass("fa fa-spinner fa-pulse");
-                    });
-                } else {
                 $("#updateRulesAct_progress").removeClass("fa fa-spinner fa-pulse");
-                }
             });
         });
 
@@ -586,20 +608,24 @@ POSSIBILITY OF SUCH DAMAGE.
                   </div>
                 </td>
             </tr>
-            <tr>
+            <tr style="display:none">
                 <td><div class="control-label">
                     <i class="fa fa-info-circle text-muted"></i>
                     <b>{{ lang._('Settings') }}</b>
                     </div>
                 </td>
                 <td>
-
+                  <table id="grid-rule-files-settings" class="table-condensed table-hover">
+                    <tbody>
+                    </tbody>
+                  </table>
                 </td>
             </tr>
           </tbody>
       </table>
       <div class="col-md-12">
           <hr/>
+          <button class="btn btn-primary" style="display:none" id="updateSettings" type="button"><b>{{ lang._('Save') }}</b><i id="updateSettings_progress" class=""></i></button>
           <button class="btn btn-primary" id="updateRulesAct" type="button"><b>{{ lang._('Download & Update Rules') }}</b><i id="updateRulesAct_progress" class=""></i></button>
           <br/>
           <i>{{ lang._('Please use "Download & Update Rules" to fetch your initial ruleset, automatic updating can be scheduled after the first download.') }} </i>

src/opnsense/scripts/suricata/lib/downloader.py

@@ -110,7 +110,7 @@ class Downloader(object):
         else:
             return src.read()
 
-    def download(self, proto, url, url_filename, filename, input_filter):
+    def download(self, proto, url, url_filename, filename, input_filter, auth = None):
         """ download ruleset file
             :param proto: protocol (http,https)
             :param url: download url
@@ -121,7 +121,13 @@ class Downloader(object):
             frm_url = url.replace('//', '/').replace(':/', '://')
             # stream to temp file
             if frm_url not in self._download_cache:
-                req = requests.get(url=frm_url, stream=True, verify=False)
+                req_opts = dict()
+                req_opts['url'] = frm_url
+                req_opts['stream'] = True
+                if auth is not None:
+                    req_opts['auth'] = auth
+                req = requests.get(**req_opts)
+
                 if req.status_code == 200:
                     src = tempfile.NamedTemporaryFile('wb+', 10240)
                     while True:

src/opnsense/scripts/suricata/rule-updater.py

@@ -82,5 +82,9 @@ if __name__ == '__main__':
                         pass
                 else:
                     input_filter = enabled_rulefiles[rule['filename']]['filter']
+                    if ('username' in rule['source'] and 'password' in rule['source']):
+                        auth = (rule['source']['username'], rule['source']['password'])
+                    else:
+                        auth = None
                     dl.download(proto=download_proto, url=rule['url'], url_filename=rule['url_filename'],
-                                filename=rule['filename'], input_filter=input_filter)
+                                filename=rule['filename'], input_filter=input_filter, auth=auth)

src/opnsense/service/templates/OPNsense/IDS/rule-updater.config

@@ -2,6 +2,14 @@
  create configuration for OPNsense suricata rule file downloader
 #}
 # autogenerated, do not edit.
+[__properties__]
+{% if helpers.exists('OPNsense.IDS.fileTags.tag') %}
+{%      for tag in helpers.toList('OPNsense.IDS.fileTags.tag') %}
+{{tag.property}}={{tag.value}}
+
+{%      endfor %}
+{% endif %}
+
 {% if helpers.exists('OPNsense.IDS.files.file') %}
 {%      for file in helpers.toList('OPNsense.IDS.files.file') %}
 [{{file.filename|default('-')}}]