(proxy) add optional filter for remote blacklist

(cherry picked from commit 7df30b7d)

src/opnsense/scripts/proxy/fetchACLs.py

@@ -126,24 +126,38 @@ if os.path.exists(acl_config_fn):
     cnf = ConfigParser()
     cnf.read(acl_config_fn)
     for section in cnf.sections():
-        # check if tag enabled exists in section
-        if cnf.has_option(section,'enabled'):
-            # if enabled fetch file
         target_filename = acl_target_dir+'/'+section
         if cnf.has_option(section,'url'):
+            # collect filters to apply
+            acl_filters = list()
+            if cnf.has_option(section,'filter'):
+                for acl_filter in cnf.get(section,'filter').strip().split(','):
+                    if len(acl_filter.strip()) > 0:
+                        acl_filters.append(acl_filter)
+
             # define targets
             targets = {'domain': {'filename': target_filename, 'handle' : None},
                        'url': {'filename': '%s.url'%target_filename, 'handle': None}}
 
-                # download file
-                if cnf.get(section,'enabled') == '1':
             # only generate files if enabled, otherwise dump empty files
+            if cnf.has_option(section,'enabled') and cnf.get(section,'enabled') == '1':
                 download_url = cnf.get(section,'url')
                 acl = ACLDownload(download_url, acl_max_timeout)
                 all_filenames = list()
                 for filename, filetype, line in acl.download():
                     if filename not in all_filenames:
                         all_filenames.append(filename)
+
+                    if len(acl_filters) > 0:
+                        acl_found = False
+                        for acl_filter in acl_filters:
+                            if filename.find(acl_filter) > -1:
+                                acl_found = True
+                                break
+                        if not acl_found:
+                            # skip this acl entry
+                            continue
+
                     if filetype in targets and targets[filetype]['handle'] is None:
                         targets[filetype]['handle'] = open(targets[filetype]['filename'], 'wb')
                     if filetype in targets:
@@ -157,11 +171,12 @@ if os.path.exists(acl_config_fn):
                             if index_key not in index_data:
                                 index_data[index_key] = index_key
                     idx_out.write(json.dumps(index_data))
+
             # cleanup
             for filetype in targets:
                 if targets[filetype]['handle'] is not None:
                     targets[filetype]['handle'].close()
-                    elif cnf.get(section,'enabled') != '1':
+                elif cnf.has_option(section,'enabled') and cnf.get(section,'enabled') != '1':
                     if os.path.isfile(targets[filetype]['filename']):
                         # disabled, remove previous data
                         os.remove(targets[filetype]['filename'])

src/opnsense/service/templates/OPNsense/Proxy/externalACLs.conf

@@ -6,5 +6,6 @@
 [{{blacklist.filename}}]
 url:{{blacklist.url}}
 enabled:{{blacklist.enabled}}
+filter:{{blacklist.filter|default('')}}
 {%   endfor %}
 {% endif %}