(ids) add script to merge user defined presets and installed rules

src/opnsense/scripts/suricata/installRules.py

+#!/usr/local/bin/python2.7
+"""
+    Copyright (c) 2015 Ad Schellevis
+
+    part of OPNsense (https://www.opnsense.org/)
+
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+
+    --------------------------------------------------------------------------------------
+"""
+import os.path
+import rulecache
+from ConfigParser import ConfigParser
+
+RuleCache = rulecache.RuleCache()
+
+rule_config_fn = ('%s../rules.config'%RuleCache.rule_source_dir)
+rule_target_dir = ('%s../opnsense.rules'%RuleCache.rule_source_dir)
+
+# parse OPNsense rule config
+rule_updates = {}
+if os.path.exists(rule_config_fn):
+    cnf = ConfigParser()
+    cnf.read(rule_config_fn)
+    for section in cnf.sections():
+        if section[0:5] == 'rule_':
+            sid = section[5:]
+            rule_updates[sid] = {}
+            for rule_item in cnf.items(section):
+                rule_updates[sid][rule_item[0]] = rule_item[1]
+
+# create target rule directory if not existing
+if not os.path.exists(rule_target_dir):
+    os.mkdir(rule_target_dir, 0o755)
+
+# install ruleset
+for filename in RuleCache.listLocal():
+    output_data = []
+    for rule_info_record in RuleCache.listRules(filename=filename):
+        # default behavior, do not touch rule, only copy to output
+        rule = rule_info_record['rule']
+        # change rule if in rule rule updates
+        if rule_info_record['metadata'] is not None and 'sid' in rule_info_record['metadata'] \
+                and rule_info_record['metadata']['sid'] in rule_updates:
+            # search last comment marker
+            for i in range(len(rule_info_record['rule'])):
+                if rule[i] != '#':
+                    break
+
+            # generate altered rule
+            if 'enabled' in rule_updates[rule_info_record['metadata']['sid']]:
+                if (rule_updates[rule_info_record['metadata']['sid']]['enabled']) == '0':
+                    rule = ('#%s'%rule[i:])
+                else:
+                    rule = rule[i:]
+
+        output_data.append(rule)
+
+    # write data to file
+    open('%s/%s'%(rule_target_dir, filename.split('/')[-1]), 'wb').write('\n'.join(output_data))

src/opnsense/scripts/suricata/rulecache.py

@@ -52,6 +52,54 @@ class RuleCache(object):
 
         return all_rule_files
 
+    def listRules(self, filename):
+        """ generator function to list rule file content including metadata
+        :param filename:
+        :return:
+        """
+        data = open(filename)
+        for rule in data.read().split('\n'):
+            rule_info_record = {'rule':rule, 'metadata':None}
+            if rule.find('msg:') != -1:
+                # define basic record
+                record = {'enabled':True, 'source':filename.split('/')[-1]}
+                if rule.strip()[0] =='#':
+                    record['enabled'] = False
+
+                rule_metadata = rule[rule.find('msg:'):-1]
+                for field in rule_metadata.split(';'):
+                    fieldName = field[0:field.find(':')].strip()
+                    fieldContent = field[field.find(':')+1:].strip()
+                    if fieldName in self._rule_fields:
+                        if fieldContent[0] == '"':
+                            content = fieldContent[1:-1]
+                        else:
+                            content = fieldContent
+
+                        if fieldName in record:
+                            # if same field repeats, put items in list
+                            if type(record[fieldName]) != list:
+                                record[fieldName] = [record[fieldName]]
+                            record[fieldName].append(content)
+                        else:
+                            record[fieldName] = content
+
+                for rule_field in self._rule_fields:
+                    if rule_field not in record:
+                        if rule_field in self._rule_defaults:
+                            record[rule_field] = self._rule_defaults[rule_field]
+                        else:
+                            record[rule_field] = None
+
+                # perform type conversions
+                for fieldname in record:
+                    if type(record[fieldname]) == list:
+                        record[fieldname] = '\n'.join(record[fieldname])
+
+                rule_info_record['metadata'] = record
+
+            yield rule_info_record
+
     def isChanged(self):
         """ check if rules on disk are probably different from rules in cache
         :return: boolean
@@ -97,45 +145,9 @@ class RuleCache(object):
             if file_mtime > last_mtime:
                 last_mtime = file_mtime
             rules = []
-            data = open(filename)
-            for rule in data.read().split('\n'):
-                if rule.find('msg:') != -1:
-                    # define basic record
-                    record = {'enabled':True, 'source':filename.split('/')[-1]}
-                    if rule.strip()[0] =='#':
-                        record['enabled'] = False
-
-                    rule_metadata = rule[rule.find('msg:'):-1]
-                    for field in rule_metadata.split(';'):
-                        fieldName = field[0:field.find(':')].strip()
-                        fieldContent = field[field.find(':')+1:].strip()
-                        if fieldName in self._rule_fields:
-                            if fieldContent[0] == '"':
-                                content = fieldContent[1:-1]
-                            else:
-                                content = fieldContent
-
-                            if fieldName in record:
-                                # if same field repeats, put items in list
-                                if type(record[fieldName]) != list:
-                                    record[fieldName] = [record[fieldName]]
-                                record[fieldName].append(content)
-                            else:
-                                record[fieldName] = content
-
-                    for rule_field in self._rule_fields:
-                        if rule_field not in record:
-                            if rule_field in self._rule_defaults:
-                                record[rule_field] = self._rule_defaults[rule_field]
-                            else:
-                                record[rule_field] = None
-
-                    # perform type conversions
-                    for fieldname in record:
-                        if type(record[fieldname]) == list:
-                            record[fieldname] = '\n'.join(record[fieldname])
-
-                    rules.append(record)
+            for rule_info_record in self.listRules(filename=filename):
+                if rule_info_record['metadata'] is not None:
+                    rules.append(rule_info_record['metadata'])
 
             cur.executemany('insert into rules(%(fieldnames)s) '
                             'values (%(fieldvalues)s)'%{'fieldnames':(','.join(self._rule_fields)),
@@ -227,3 +239,4 @@ class RuleCache(object):
             result.append(record[0])
 
         return sorted(result)
+