(ids) add input filters to file download for easier IPS support, add prefix to rules xml definition

(cherry picked from commit 75c3fda1)

src/opnsense/scripts/suricata/lib/downloader.py

@@ -36,14 +36,46 @@ class Downloader(object):
     def __init__(self, target_dir):
         self._target_dir = target_dir
 
-    def download(self, proto, url):
+    def filter(self, in_data, filter_type):
+        """ apply input filter to downloaded data
+            :param in_data: raw input data (ruleset)
+            :param filter_type: filter type to use on input data
+            :return: ruleset data
+        """
+        if filter_type == "drop":
+            return self.filter_drop(in_data)
+        else:
+            return in_data
+
+    def filter_drop(self, in_data):
+        """ change all alert rules to block
+            :param in_data: raw input data (ruleset)
+            :return: new ruleset
+        """
+        output = list()
+        for line in in_data.split('\n'):
+            if len(line) > 10:
+                if line[0:5] == 'alert':
+                    line = 'drop %s' % line[5:]
+                elif line[0:6] == '#alert':
+                    line = '#drop %s' % line[5:]
+            output.append(line)
+        return '\n'.join(output)
+
+    def download(self, proto, url, input_filter):
+        """ download ruleset file
+            :param proto: protocol (http,https)
+            :param url: download url
+            :param input_filter: filter to use on received data before save
+        """
         if proto in ('http', 'https'):
             frm_url = url.replace('//', '/').replace(':/', '://')
             req = requests.get(url=frm_url)
             if req.status_code == 200:
                 target_filename = ('%s/%s' % (self._target_dir, frm_url.split('/')[-1])).replace('//', '/')
                 try:
-                    open(target_filename, 'wb').write(req.text)
+                    save_data = self.filter(req.text, input_filter)
+                    open(target_filename, 'wb').write(save_data)
                 except IOError:
                     syslog.syslog(syslog.LOG_ERR, 'cannot write to %s' % target_filename)
                     return None

src/opnsense/scripts/suricata/lib/metadata.py

@@ -61,9 +61,15 @@ class Metadata(object):
                         metadata_record = dict()
                         metadata_record['source'] = src_location.attrib
                         metadata_record['filename'] = rule_filename.text.strip()
+                        if 'prefix' in src_location.attrib:
+                            description_prefix = "%s/" % src_location.attrib['prefix']
+                        else:
+                            description_prefix = ""
                         if 'description' in rule_filename.attrib:
-                            metadata_record['description'] = rule_filename.attrib['description']
+                            metadata_record['description'] = '%s%s' % (description_prefix,
+                                                                       rule_filename.attrib['description'])
                         else:
-                            metadata_record['description'] = rule_filename.text
+                            metadata_record['description'] = '%s%s' % (description_prefix,
+                                                                       rule_filename.text)
 
                         yield metadata_record

src/opnsense/scripts/suricata/rule-updater.py

@@ -48,14 +48,20 @@ except IOError:
 
 if __name__ == '__main__':
     # load list of configured rules from generated config
-    enabled_rulefiles = []
+    enabled_rulefiles = dict()
     updater_conf = '/usr/local/etc/suricata/rule-updater.config'
     if os.path.exists(updater_conf):
         cnf = ConfigParser()
         cnf.read(updater_conf)
         for section in cnf.sections():
             if cnf.has_option(section, 'enabled') and cnf.getint(section, 'enabled') == 1:
-                enabled_rulefiles.append(section.strip())
+                enabled_rulefiles[section.strip()] = {}
+                # input filter
+                if cnf.has_option(section, 'filter'):
+                    enabled_rulefiles[section.strip()]['filter'] = cnf.get(section, 'filter').strip()
+                else:
+                    enabled_rulefiles[section.strip()]['filter'] = ""
+
 
     # download / remove rules
     md = metadata.Metadata()
@@ -71,5 +77,6 @@ if __name__ == '__main__':
                     except OSError:
                         pass
                 else:
+                    input_filter = enabled_rulefiles[rule['filename']]['filter']
                     url = ('%s/%s' % (rule['source']['url'], rule['filename']))
-                    dl.download(proto=download_proto, url=url)
+                    dl.download(proto=download_proto, url=url, input_filter=input_filter)

src/opnsense/service/templates/OPNsense/IDS/rule-updater.config

@@ -6,6 +6,7 @@
 {%      for file in helpers.toList('OPNsense.IDS.files.file') %}
 [{{file.filename|default('-')}}]
 enabled={{ file.enabled|default('0') }}
+filter={{ file.filter|default('') }}
 
 {%      endfor %}
 {% endif %}