From 0c05cd6fce7a6671b4d279d6963fb398cf72e05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Mon, 25 Jan 2021 13:53:41 +0100 Subject: [PATCH 01/22] Regex evaluation for POC message text --- config/config.template.ini | 3 +++ includes/decoders/poc.py | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/config/config.template.ini b/config/config.template.ini index e20a7eb..8f22956 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -127,6 +127,9 @@ geo_enable = 0 geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat +# Analyze message and associate data to named fields +schemaRegex = ^(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?)$ + [multicastAlarm] # Configure multicastAlarm if your POCSAG network uses an optimized transmission scheme for alarms with more than one RIC (often found in Swissphone networks). diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index bf906fd..1a0bc45 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -157,7 +157,20 @@ def decode(freq, decoded): # check for double alarm if doubleFilter.checkID("POC", poc_id+poc_sub, poc_text): - data = {"ric":poc_id, "function":poc_sub, "msg":poc_text, "bitrate":bitrate, "description":poc_id, "has_geo":has_geo} + data = {"ric":poc_id, "function":poc_sub, "msg":poc_text, "bitrate":bitrate, "description":poc_id, "has_geo":has_geo, "has_schema_fields":False} + + # if a schema is defined, analyze and associate + if globalVars.config.has_option("POC", "schemaRegex"): + logging.info("schemaRegex found") + m = re.match(globalVars.config.get("POC", "schemaRegex"), poc_text) + if m: + logging.info("POC Schema match") + # enrich data structure by regex groups + data.update(m.groupdict()) + data["has_schema_fields"] = True + else: + logging.info("No POC Schema match") + if has_geo == True: data["lon"] = lon data["lat"] = lat @@ -177,7 +190,7 @@ def decode(freq, decoded): logging.debug(" - multicastAlarm without msg") from includes import multicastAlarm multicastAlarm.newEntrymultiList(data) - + # multicastAlarm processing if enabled and alarm message has been received elif globalVars.config.getint("multicastAlarm", "multicastAlarm") and data["msg"] != "" and data["ric"] in globalVars.config.get("multicastAlarm", "multicastAlarm_ric"): logging.debug(" - multicastAlarm with message") From fed8c4638262cb6cc09c898995e22165f1213c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Mon, 25 Jan 2021 17:53:51 +0100 Subject: [PATCH 02/22] Location configuration based on POC message elements --- boswatch.py | 11 +++++ config/config.template.ini | 17 ++++++++ includes/decoders/poc.py | 4 ++ includes/locationCoordinates.py | 77 +++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 includes/locationCoordinates.py diff --git a/boswatch.py b/boswatch.py index 8d316b6..76d8f99 100755 --- a/boswatch.py +++ b/boswatch.py @@ -287,6 +287,17 @@ try: logging.error("cannot load description lists") logging.debug("cannot load description lists", exc_info=True) + # + # Load location RegEx + # + try: + from includes import locationCoordinates + locationCoordinates.loadFilters() + except: + # It's an error, but we could work without that stuff... + logging.error("cannot load location regex") + logging.debug("cannot load location regex", exc_info=True) + # # Start rtl_fm # diff --git a/config/config.template.ini b/config/config.template.ini index e20a7eb..c097993 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -127,6 +127,23 @@ geo_enable = 0 geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat +[LocationCoordinates] +# Regex Coordinate replacement (only for POC) +# All fields in data structure can be used, also dynamically added fields that have been evaluated in "schemaPOCMsg". +# Multiple search criteria can be given, then all of them must be hit (AND-condition). +# Coordinates must be the last field, consisting of latitude and longitude (order is important), split by comma. +# First match has priority; search will not proceed as soon as one hit is found. +# Important: Semicolon and comma must not be part of a field or regex, as they are used internally for splitting up the config value correctly. +#LocationName = field1;regex1;...;lat, lon + +# Examples: +# msg starting with "BOSWatch-Test" +Location1 = msg;^BOSWatch-Test;49.344394413024084, 8.167496841047555 +# Objekt containing "VS Wachtenburg" +Location2 = Objekt;VS Wachtenburg;49.437673, 8.173793 +# Ort starting with "B9 ", Ortsteil starting with "16 AK " +B9_16 = Ort;^B9 .*$;Ortsteil;^16 AK .*$;49.428685, 8.408548 + [multicastAlarm] # Configure multicastAlarm if your POCSAG network uses an optimized transmission scheme for alarms with more than one RIC (often found in Swissphone networks). diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index bf906fd..d51e7d0 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -15,6 +15,7 @@ import re # Regex for validation from includes import globalVars # Global variables from includes import doubleFilter # double alarm filter +from includes import locationCoordinates ## # @@ -161,6 +162,9 @@ def decode(freq, decoded): if has_geo == True: data["lon"] = lon data["lat"] = lat + else: + locationCoordinates.findCoordinates(data) + # Add function as character a-d to dataset data["functionChar"] = data["function"].replace("1", "a").replace("2", "b").replace("3", "c").replace("4", "d") data["ricFuncChar"] = data["ric"] + data["functionChar"] diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py new file mode 100644 index 0000000..d9c6e52 --- /dev/null +++ b/includes/locationCoordinates.py @@ -0,0 +1,77 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +""" +Functions for the location RegEX + +@author: Marco Schotthöfer + +@requires: Configuration has to be set in the config.ini +""" + +import logging # Global logger +import re #Regex for Filter Check + +from includes import globalVars # Global variables + +# local variables +filterList = [] + + +def loadFilters(): + try: + logging.debug("Loading Location Coordinates") + + for key,val in globalVars.config.items("LocationCoordinates"): + logging.debug(" - %s = %s", key, val) + filterData = val.split(";") + + # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items + if len(filterData) < 3 and len(filterData) % 2 == 0: + logging.debug("Invalid argument count; skipping") + else: + # first store all regular expressions in list + filterItem = [] + i = 0 + + while i < len(filterData) - 2: + filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) + + # step to next field + i += 2 + # then transfer to filterList; include coordinates + filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) + except: + logging.error("cannot read config file") + logging.debug("cannot read config file", exc_info=True) + return + +def findCoordinates(data): + try: + logging.debug("Find coordinates") + for i in filterList: + logging.debug("Filter: " + str(i)) + regexMatch = True + for k in i["filterItem"]: + logging.debug("Pattern : " + str(k)) + if k["field"] not in data.keys(): + logging.debug("field " + k["field"] + " not in data structure, hence no match") + regexMatch = False + break + else: + if (k["field"] and not re.search(k["pattern"], data.get(k["field"]))): + logging.debug("No match") + regexMatch = False + if regexMatch: + coordinatesString = i["coordinates"] + logging.debug("Coordinate String: " + coordinatesString) + coordinatesList = coordinatesString.replace(" ", "").split(",") + if len(coordinatesList) == 2: + data["lat"] = coordinatesList[0] + data["lon"] = coordinatesList[1] + data["has_geo"] = True + logging.info("Coordinates found!") + break + except: + logging.error("cannot read config file") + logging.debug("cannot read config file", exc_info=True) From 6b2b34b3bbdc34d0576dd4858900498d778903f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 09:55:05 +0100 Subject: [PATCH 03/22] Comments --- includes/locationCoordinates.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py index d9c6e52..ef95ecd 100644 --- a/includes/locationCoordinates.py +++ b/includes/locationCoordinates.py @@ -20,7 +20,7 @@ filterList = [] def loadFilters(): try: - logging.debug("Loading Location Coordinates") + logging.debug("Loading location coordinates") for key,val in globalVars.config.items("LocationCoordinates"): logging.debug(" - %s = %s", key, val) @@ -55,16 +55,17 @@ def findCoordinates(data): for k in i["filterItem"]: logging.debug("Pattern : " + str(k)) if k["field"] not in data.keys(): - logging.debug("field " + k["field"] + " not in data structure, hence no match") + logging.debug("Field " + k["field"] + " not in data structure, hence no match") regexMatch = False break else: - if (k["field"] and not re.search(k["pattern"], data.get(k["field"]))): + if not re.search(k["pattern"], data.get(k["field"])): logging.debug("No match") regexMatch = False + break if regexMatch: coordinatesString = i["coordinates"] - logging.debug("Coordinate String: " + coordinatesString) + logging.debug("Coordinates string: " + coordinatesString) coordinatesList = coordinatesString.replace(" ", "").split(",") if len(coordinatesList) == 2: data["lat"] = coordinatesList[0] From 3f9bb7ceb2b6a016c18e3420add16cb449849d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 11:22:40 +0100 Subject: [PATCH 04/22] Added changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3107002..75759ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added +- GPS-coordinates configuration based on POC message elements ##### Changed ##### Deprecated ##### Removed From c0b5e896929447b3ace8fa9905b6ca3cc99ec498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 11:24:49 +0100 Subject: [PATCH 05/22] Added changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3107002..a53f25d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added +- Extending POC data-structure by Regex named groups matching. ##### Changed ##### Deprecated ##### Removed From 7e4a271a3c67f6bc309b93f10fd88dc5387ec603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 11:50:25 +0100 Subject: [PATCH 06/22] Add pull request link to changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a53f25d..dbd9df5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added -- Extending POC data-structure by Regex named groups matching. +- Extending POC data-structure by Regex named groups matching. [#508](https://github.com/Schrolli91/BOSWatch/pull/508) ##### Changed ##### Deprecated ##### Removed From ec2e5bd99b2f9a9ba90cbe340531e30df05483f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 11:54:29 +0100 Subject: [PATCH 07/22] Add pull request link to changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75759ef..8e950cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added -- GPS-coordinates configuration based on POC message elements +- GPS-coordinates configuration based on POC message elements. [#509](https://github.com/Schrolli91/BOSWatch/pull/509) ##### Changed ##### Deprecated ##### Removed From 13d76ceab998ed66cc5739896826168d9663447e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 12:00:35 +0100 Subject: [PATCH 08/22] Corrected pull request link in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e950cb..dae948b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added -- GPS-coordinates configuration based on POC message elements. [#509](https://github.com/Schrolli91/BOSWatch/pull/509) +- GPS-coordinates configuration based on POC message elements. [#510](https://github.com/Schrolli91/BOSWatch/pull/510) ##### Changed ##### Deprecated ##### Removed From 88bfb4fb76969e404f8b0b78e4cb0ea3c98a4484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 12:25:42 +0100 Subject: [PATCH 09/22] Commented out optional configuration option --- config/config.template.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.template.ini b/config/config.template.ini index 8f22956..c70e8f9 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -128,7 +128,7 @@ geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat # Analyze message and associate data to named fields -schemaRegex = ^(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?)$ +#schemaRegex = ^(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?)$ [multicastAlarm] From 1fc2980f6d8a0b6214325c9cfde9c899aad33873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 13:39:40 +0100 Subject: [PATCH 10/22] Make configuration section optional --- config/config.template.ini | 2 +- includes/locationCoordinates.py | 41 +++++++++++++++++---------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/config/config.template.ini b/config/config.template.ini index c097993..8c4ed23 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -127,7 +127,7 @@ geo_enable = 0 geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat -[LocationCoordinates] +#[LocationCoordinates] # Regex Coordinate replacement (only for POC) # All fields in data structure can be used, also dynamically added fields that have been evaluated in "schemaPOCMsg". # Multiple search criteria can be given, then all of them must be hit (AND-condition). diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py index ef95ecd..f5a18c9 100644 --- a/includes/locationCoordinates.py +++ b/includes/locationCoordinates.py @@ -20,27 +20,28 @@ filterList = [] def loadFilters(): try: - logging.debug("Loading location coordinates") - - for key,val in globalVars.config.items("LocationCoordinates"): - logging.debug(" - %s = %s", key, val) - filterData = val.split(";") - - # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items - if len(filterData) < 3 and len(filterData) % 2 == 0: - logging.debug("Invalid argument count; skipping") - else: - # first store all regular expressions in list - filterItem = [] - i = 0 - - while i < len(filterData) - 2: - filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) + if globalVars.config.has_section("LocationCoordinates"): + logging.debug("Loading location coordinates") + + for key,val in globalVars.config.items("LocationCoordinates"): + logging.debug(" - %s = %s", key, val) + filterData = val.split(";") + + # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items + if len(filterData) < 3 and len(filterData) % 2 == 0: + logging.debug("Invalid argument count; skipping") + else: + # first store all regular expressions in list + filterItem = [] + i = 0 - # step to next field - i += 2 - # then transfer to filterList; include coordinates - filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) + while i < len(filterData) - 2: + filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) + + # step to next field + i += 2 + # then transfer to filterList; include coordinates + filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) except: logging.error("cannot read config file") logging.debug("cannot read config file", exc_info=True) From d9b142c59fe83c00ec09c37d322187d4ac4251f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 14:40:40 +0100 Subject: [PATCH 11/22] Added some more description --- config/config.template.ini | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/config/config.template.ini b/config/config.template.ini index c70e8f9..533d840 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -128,6 +128,19 @@ geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat # Analyze message and associate data to named fields +# If the regular expression matches the POC message, the "named groups" defined in schemaRegex will be added to the data structure. +# You can check this by looking at the value in data["has_schema_fields"]. If it is True, then the regular expression has been hit and the additional fields are available. +# E.g. POC message is "TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz" +# +# Without the schemaRegex, data will look like this (RIC, SubRIC and Bitrate are random): +# {'function': '1', 'has_geo': False, 'description': '1234567', 'has_schema_fields': False, 'msg': 'TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz', 'bitrate': 1200, 'ric': '1234567'} +# +# When using the schemaRegex from below, data will look like this: +# {'function': '1', 'has_geo': False, 'description': '1234567', 'has_schema_fields': True, 'msg': 'TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz', 'bitrate': 1200, 'ric': '1234567, 'Objekt': 'VS Winzerheim', 'StichwortLang': 'TESTALARM Person in Zwangslage', 'Ort': 'Neustadt', 'Bemerkung1': '', 'Adresse': 'Weinstraße 1', 'Bemerkung2': 'Dortige Baustelle..Treppensturz', 'StichwortKurz': 'H2.04', 'Ortsteil': 'Königsbach' +# +# Attention: If you define named groups with names of fields that that are already present in POC decoder's standard data-structure, the content of the respective fields will be overwritten +# with data evaluated by this regex (if the regex finds a match). E.g. if you define a named group like "...(?P.*)..." and the regex will be hit, +# then data["ric"] will get filled with the content evaluated by the regex. #schemaRegex = ^(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?);(?P.*?)$ From 43697da228698014c0a658aafad52e62e53c3639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 14:43:07 +0100 Subject: [PATCH 12/22] Typo --- config/config.template.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.template.ini b/config/config.template.ini index 533d840..e49cbea 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -136,7 +136,7 @@ geo_order = LON, lon, LAT, lat # {'function': '1', 'has_geo': False, 'description': '1234567', 'has_schema_fields': False, 'msg': 'TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz', 'bitrate': 1200, 'ric': '1234567'} # # When using the schemaRegex from below, data will look like this: -# {'function': '1', 'has_geo': False, 'description': '1234567', 'has_schema_fields': True, 'msg': 'TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz', 'bitrate': 1200, 'ric': '1234567, 'Objekt': 'VS Winzerheim', 'StichwortLang': 'TESTALARM Person in Zwangslage', 'Ort': 'Neustadt', 'Bemerkung1': '', 'Adresse': 'Weinstraße 1', 'Bemerkung2': 'Dortige Baustelle..Treppensturz', 'StichwortKurz': 'H2.04', 'Ortsteil': 'Königsbach' +# {'function': '1', 'has_geo': False, 'description': '1234567', 'has_schema_fields': True, 'msg': 'TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz', 'bitrate': 1200, 'ric': '1234567, 'Objekt': 'VS Winzerheim', 'StichwortLang': 'TESTALARM Person in Zwangslage', 'Ort': 'Neustadt', 'Bemerkung1': '', 'Adresse': 'Weinstraße 1', 'Bemerkung2': 'Dortige Baustelle..Treppensturz', 'StichwortKurz': 'H2.04', 'Ortsteil': 'Königsbach'} # # Attention: If you define named groups with names of fields that that are already present in POC decoder's standard data-structure, the content of the respective fields will be overwritten # with data evaluated by this regex (if the regex finds a match). E.g. if you define a named group like "...(?P.*)..." and the regex will be hit, From a76e54211f14863a2e04bfe192dd7236a4e1656f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 16:14:57 +0100 Subject: [PATCH 13/22] Dedicated on/off switch for location coordinate finding functionality --- boswatch.py | 5 ++-- config/config.template.ini | 17 ++++++++++---- includes/decoders/poc.py | 3 ++- includes/locationCoordinates.py | 41 ++++++++++++++++----------------- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/boswatch.py b/boswatch.py index 76d8f99..0c2457f 100755 --- a/boswatch.py +++ b/boswatch.py @@ -291,8 +291,9 @@ try: # Load location RegEx # try: - from includes import locationCoordinates - locationCoordinates.loadFilters() + if globalVars.config.getboolean("LocationCoordinates", "locationCoordinates"): + from includes import locationCoordinates + locationCoordinates.loadFilters() except: # It's an error, but we could work without that stuff... logging.error("cannot load location regex") diff --git a/config/config.template.ini b/config/config.template.ini index 8c4ed23..b468c37 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -127,22 +127,29 @@ geo_enable = 0 geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat -#[LocationCoordinates] +[LocationCoordinates] # Regex Coordinate replacement (only for POC) # All fields in data structure can be used, also dynamically added fields that have been evaluated in "schemaPOCMsg". # Multiple search criteria can be given, then all of them must be hit (AND-condition). # Coordinates must be the last field, consisting of latitude and longitude (order is important), split by comma. # First match has priority; search will not proceed as soon as one hit is found. # Important: Semicolon and comma must not be part of a field or regex, as they are used internally for splitting up the config value correctly. -#LocationName = field1;regex1;...;lat, lon + +# Do you want to enable this feature? (0 - off | 1 - on) +locationCoordinates = 0 + +# LocationName = field1;regex1;...;lat, lon # Examples: + # msg starting with "BOSWatch-Test" -Location1 = msg;^BOSWatch-Test;49.344394413024084, 8.167496841047555 +#Location1 = msg;^BOSWatch-Test;49.344394413024084, 8.167496841047555 + # Objekt containing "VS Wachtenburg" -Location2 = Objekt;VS Wachtenburg;49.437673, 8.173793 +#Location2 = Objekt;VS Wachtenburg;49.437673, 8.173793 + # Ort starting with "B9 ", Ortsteil starting with "16 AK " -B9_16 = Ort;^B9 .*$;Ortsteil;^16 AK .*$;49.428685, 8.408548 +#B9_16 = Ort;^B9 .*$;Ortsteil;^16 AK .*$;49.428685, 8.408548 [multicastAlarm] diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index d51e7d0..898085c 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -163,7 +163,8 @@ def decode(freq, decoded): data["lon"] = lon data["lat"] = lat else: - locationCoordinates.findCoordinates(data) + if globalVars.config.getboolean("LocationCoordinates", "locationCoordinates"): + locationCoordinates.findCoordinates(data) # Add function as character a-d to dataset data["functionChar"] = data["function"].replace("1", "a").replace("2", "b").replace("3", "c").replace("4", "d") diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py index f5a18c9..ef95ecd 100644 --- a/includes/locationCoordinates.py +++ b/includes/locationCoordinates.py @@ -20,28 +20,27 @@ filterList = [] def loadFilters(): try: - if globalVars.config.has_section("LocationCoordinates"): - logging.debug("Loading location coordinates") - - for key,val in globalVars.config.items("LocationCoordinates"): - logging.debug(" - %s = %s", key, val) - filterData = val.split(";") - - # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items - if len(filterData) < 3 and len(filterData) % 2 == 0: - logging.debug("Invalid argument count; skipping") - else: - # first store all regular expressions in list - filterItem = [] - i = 0 + logging.debug("Loading location coordinates") + + for key,val in globalVars.config.items("LocationCoordinates"): + logging.debug(" - %s = %s", key, val) + filterData = val.split(";") + + # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items + if len(filterData) < 3 and len(filterData) % 2 == 0: + logging.debug("Invalid argument count; skipping") + else: + # first store all regular expressions in list + filterItem = [] + i = 0 + + while i < len(filterData) - 2: + filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) - while i < len(filterData) - 2: - filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) - - # step to next field - i += 2 - # then transfer to filterList; include coordinates - filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) + # step to next field + i += 2 + # then transfer to filterList; include coordinates + filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) except: logging.error("cannot read config file") logging.debug("cannot read config file", exc_info=True) From 0cdde7808b35fff2c3e643f090e186ca3a2acc2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 17:43:09 +0100 Subject: [PATCH 14/22] reduce loglevel for config option disovery notification --- includes/decoders/poc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index 1a0bc45..aafe44e 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -161,7 +161,7 @@ def decode(freq, decoded): # if a schema is defined, analyze and associate if globalVars.config.has_option("POC", "schemaRegex"): - logging.info("schemaRegex found") + logging.debug("schemaRegex found") m = re.match(globalVars.config.get("POC", "schemaRegex"), poc_text) if m: logging.info("POC Schema match") From 567303ee9924400b863a04c858cbfc193518a6b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 19:18:18 +0100 Subject: [PATCH 15/22] Decrease two more logging occurrences --- includes/decoders/poc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index aafe44e..7c5fec5 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -164,12 +164,12 @@ def decode(freq, decoded): logging.debug("schemaRegex found") m = re.match(globalVars.config.get("POC", "schemaRegex"), poc_text) if m: - logging.info("POC Schema match") + logging.debug("POC Schema match") # enrich data structure by regex groups data.update(m.groupdict()) data["has_schema_fields"] = True else: - logging.info("No POC Schema match") + logging.debug("No POC Schema match") if has_geo == True: data["lon"] = lon From 36ebaa80ca1016d37590112f2e7b9df9f1ed0e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Thu, 28 Jan 2021 20:54:55 +0100 Subject: [PATCH 16/22] Enhanced changelog; decreased loglevel --- CHANGELOG.md | 2 +- includes/locationCoordinates.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dae948b..9bfe776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added -- GPS-coordinates configuration based on POC message elements. [#510](https://github.com/Schrolli91/BOSWatch/pull/510) +- Functionality to fill coordinate values in POC data structure (lat, lon) based on configured locations that match a regular expression in POC message [#510](https://github.com/Schrolli91/BOSWatch/pull/510) ##### Changed ##### Deprecated ##### Removed diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py index ef95ecd..860aa0f 100644 --- a/includes/locationCoordinates.py +++ b/includes/locationCoordinates.py @@ -71,7 +71,7 @@ def findCoordinates(data): data["lat"] = coordinatesList[0] data["lon"] = coordinatesList[1] data["has_geo"] = True - logging.info("Coordinates found!") + logging.debug("Coordinates found!") break except: logging.error("cannot read config file") From d1d7bb3be1da96fabeddda5fad9ab0f86990a23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Thu, 28 Jan 2021 21:22:09 +0100 Subject: [PATCH 17/22] Import locationCoordinates module only when activated --- includes/decoders/poc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index 03e90f3..9166d98 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -15,7 +15,6 @@ import re # Regex for validation from includes import globalVars # Global variables from includes import doubleFilter # double alarm filter -from includes import locationCoordinates ## # @@ -177,6 +176,7 @@ def decode(freq, decoded): data["lat"] = lat else: if globalVars.config.getboolean("LocationCoordinates", "locationCoordinates"): + from includes import locationCoordinates locationCoordinates.findCoordinates(data) # Add function as character a-d to dataset From 0747a38ac0426c7cbd62ce70ed141308da98d46c Mon Sep 17 00:00:00 2001 From: Lars Gremme Date: Sat, 11 Dec 2021 10:59:14 +0100 Subject: [PATCH 18/22] Update escaping --- plugins/MySQL/boswatch.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/MySQL/boswatch.sql b/plugins/MySQL/boswatch.sql index cd044a8..8eb0dd0 100644 --- a/plugins/MySQL/boswatch.sql +++ b/plugins/MySQL/boswatch.sql @@ -25,8 +25,8 @@ SET time_zone = "+00:00"; -- Datenbank anlegen `boswatch` -- -CREATE DATABASE IF NOT EXISTS 'boswatch' DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; -USE 'boswatch'; +CREATE DATABASE IF NOT EXISTS `boswatch` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +USE `boswatch`; -- -------------------------------------------------------- From 5060c79ac7af7a3f5bffbd0bb05fbdae2d8d3d39 Mon Sep 17 00:00:00 2001 From: Lars Gremme Date: Sat, 11 Dec 2021 11:08:21 +0100 Subject: [PATCH 19/22] Use running python-mysql-connector --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 0335aee..365ec15 100755 --- a/install.sh +++ b/install.sh @@ -187,7 +187,7 @@ echo "[ 8/9] [########-]" tput cup 15 5 echo "-> Download & Install MySQL connector for Python." cd $boswatch_install_path -pip install mysql-connector-python >> $boswatch_install_path/setup_log.txt 2>&1 +pip install mysql-connector-python==8.0.19 >> $boswatch_install_path/setup_log.txt 2>&1 exitcodefunction $? install mysql-connector # Blacklist DVB-Drivers From 09b8af0573835acf8b51f70222dc4fbfbe8883d2 Mon Sep 17 00:00:00 2001 From: Lars Gremme Date: Sat, 11 Dec 2021 11:27:11 +0100 Subject: [PATCH 20/22] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3107002..f0e9434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ ##### Deprecated ##### Removed ##### Fixed + - Use specific mysql-connector-python version 8.0.19 [#533](https://github.com/Schrolli91/BOSWatch/pull/533) + - Correct Syntax-Errors in MySQL Statements [#533](https://github.com/Schrolli91/BOSWatch/pull/533) ##### Security From 14ab195061cec7dabe7797b0a50533d4f41248c3 Mon Sep 17 00:00:00 2001 From: Jonas Arnold Date: Mon, 14 Mar 2022 08:52:09 +0100 Subject: [PATCH 21/22] added possibility to configure login data for mqtt broker --- config/config.template.ini | 3 +++ plugins/mqtt/mqtt.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/config/config.template.ini b/config/config.template.ini index 4469824..e4f60ba 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -596,6 +596,9 @@ commandPOC = #Adress from MQTT-Broker brokeraddress = 192.168.178.27 topic = alarm/data +# username and password for the broker. leave username empty to use anonymous login +brokerusername = +brokerpassword = ##################### ##### Not ready yet # diff --git a/plugins/mqtt/mqtt.py b/plugins/mqtt/mqtt.py index 683d7d6..3abcf53 100644 --- a/plugins/mqtt/mqtt.py +++ b/plugins/mqtt/mqtt.py @@ -77,7 +77,11 @@ def run(typ,freq,data): ########## User Plugin CODE ########## broker_address = globalVars.config.get("mqtt", "brokeraddress") topic = globalVars.config.get("mqtt", "topic") - mqttClient = mqtt.Client() + + broker_username = globalVars.config.get("mqtt", "brokerusername") + broker_password = globalVars.config.get("mqtt", "brokerpassword") + + mqttClient = mqtt.Client() if typ == "FMS": x = { @@ -112,6 +116,12 @@ def run(typ,freq,data): logging.warning("Invalid Typ: %s", typ) y = json.dumps(x) + + + ## only login if there is a username given + if( len(broker_username) > 0 ): + mqttClient.username_pw_set(broker_username, broker_password) + mqttClient.connect(broker_address) mqttClient.publish(topic,y) ########## User Plugin CODE ########## From 278b5361c5456c81f70c4f6dffaf88c2512ad3a7 Mon Sep 17 00:00:00 2001 From: Jonas Arnold Date: Mon, 14 Mar 2022 10:28:10 +0100 Subject: [PATCH 22/22] add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38141dd..bbafb05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ##### Added - Functionality to fill coordinate values in POC data structure (lat, lon) based on configured locations that match a regular expression in POC message [#510](https://github.com/Schrolli91/BOSWatch/pull/510) - Extending POC data-structure by Regex named groups matching. [#508](https://github.com/Schrolli91/BOSWatch/pull/508) +- MQTT Plugin: Added possibility to configure login (user and password) for mqtt broker. [#539](https://github.com/Schrolli91/BOSWatch/pull/539) ##### Changed ##### Deprecated ##### Removed