From fed8c4638262cb6cc09c898995e22165f1213c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Mon, 25 Jan 2021 17:53:51 +0100 Subject: [PATCH 1/9] Location configuration based on POC message elements --- boswatch.py | 11 +++++ config/config.template.ini | 17 ++++++++ includes/decoders/poc.py | 4 ++ includes/locationCoordinates.py | 77 +++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 includes/locationCoordinates.py diff --git a/boswatch.py b/boswatch.py index 8d316b6..76d8f99 100755 --- a/boswatch.py +++ b/boswatch.py @@ -287,6 +287,17 @@ try: logging.error("cannot load description lists") logging.debug("cannot load description lists", exc_info=True) + # + # Load location RegEx + # + try: + from includes import locationCoordinates + locationCoordinates.loadFilters() + except: + # It's an error, but we could work without that stuff... + logging.error("cannot load location regex") + logging.debug("cannot load location regex", exc_info=True) + # # Start rtl_fm # diff --git a/config/config.template.ini b/config/config.template.ini index e20a7eb..c097993 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -127,6 +127,23 @@ geo_enable = 0 geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat +[LocationCoordinates] +# Regex Coordinate replacement (only for POC) +# All fields in data structure can be used, also dynamically added fields that have been evaluated in "schemaPOCMsg". +# Multiple search criteria can be given, then all of them must be hit (AND-condition). +# Coordinates must be the last field, consisting of latitude and longitude (order is important), split by comma. +# First match has priority; search will not proceed as soon as one hit is found. +# Important: Semicolon and comma must not be part of a field or regex, as they are used internally for splitting up the config value correctly. +#LocationName = field1;regex1;...;lat, lon + +# Examples: +# msg starting with "BOSWatch-Test" +Location1 = msg;^BOSWatch-Test;49.344394413024084, 8.167496841047555 +# Objekt containing "VS Wachtenburg" +Location2 = Objekt;VS Wachtenburg;49.437673, 8.173793 +# Ort starting with "B9 ", Ortsteil starting with "16 AK " +B9_16 = Ort;^B9 .*$;Ortsteil;^16 AK .*$;49.428685, 8.408548 + [multicastAlarm] # Configure multicastAlarm if your POCSAG network uses an optimized transmission scheme for alarms with more than one RIC (often found in Swissphone networks). diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index bf906fd..d51e7d0 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -15,6 +15,7 @@ import re # Regex for validation from includes import globalVars # Global variables from includes import doubleFilter # double alarm filter +from includes import locationCoordinates ## # @@ -161,6 +162,9 @@ def decode(freq, decoded): if has_geo == True: data["lon"] = lon data["lat"] = lat + else: + locationCoordinates.findCoordinates(data) + # Add function as character a-d to dataset data["functionChar"] = data["function"].replace("1", "a").replace("2", "b").replace("3", "c").replace("4", "d") data["ricFuncChar"] = data["ric"] + data["functionChar"] diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py new file mode 100644 index 0000000..d9c6e52 --- /dev/null +++ b/includes/locationCoordinates.py @@ -0,0 +1,77 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +""" +Functions for the location RegEX + +@author: Marco Schotthöfer + +@requires: Configuration has to be set in the config.ini +""" + +import logging # Global logger +import re #Regex for Filter Check + +from includes import globalVars # Global variables + +# local variables +filterList = [] + + +def loadFilters(): + try: + logging.debug("Loading Location Coordinates") + + for key,val in globalVars.config.items("LocationCoordinates"): + logging.debug(" - %s = %s", key, val) + filterData = val.split(";") + + # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items + if len(filterData) < 3 and len(filterData) % 2 == 0: + logging.debug("Invalid argument count; skipping") + else: + # first store all regular expressions in list + filterItem = [] + i = 0 + + while i < len(filterData) - 2: + filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) + + # step to next field + i += 2 + # then transfer to filterList; include coordinates + filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) + except: + logging.error("cannot read config file") + logging.debug("cannot read config file", exc_info=True) + return + +def findCoordinates(data): + try: + logging.debug("Find coordinates") + for i in filterList: + logging.debug("Filter: " + str(i)) + regexMatch = True + for k in i["filterItem"]: + logging.debug("Pattern : " + str(k)) + if k["field"] not in data.keys(): + logging.debug("field " + k["field"] + " not in data structure, hence no match") + regexMatch = False + break + else: + if (k["field"] and not re.search(k["pattern"], data.get(k["field"]))): + logging.debug("No match") + regexMatch = False + if regexMatch: + coordinatesString = i["coordinates"] + logging.debug("Coordinate String: " + coordinatesString) + coordinatesList = coordinatesString.replace(" ", "").split(",") + if len(coordinatesList) == 2: + data["lat"] = coordinatesList[0] + data["lon"] = coordinatesList[1] + data["has_geo"] = True + logging.info("Coordinates found!") + break + except: + logging.error("cannot read config file") + logging.debug("cannot read config file", exc_info=True) From 6b2b34b3bbdc34d0576dd4858900498d778903f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 09:55:05 +0100 Subject: [PATCH 2/9] Comments --- includes/locationCoordinates.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py index d9c6e52..ef95ecd 100644 --- a/includes/locationCoordinates.py +++ b/includes/locationCoordinates.py @@ -20,7 +20,7 @@ filterList = [] def loadFilters(): try: - logging.debug("Loading Location Coordinates") + logging.debug("Loading location coordinates") for key,val in globalVars.config.items("LocationCoordinates"): logging.debug(" - %s = %s", key, val) @@ -55,16 +55,17 @@ def findCoordinates(data): for k in i["filterItem"]: logging.debug("Pattern : " + str(k)) if k["field"] not in data.keys(): - logging.debug("field " + k["field"] + " not in data structure, hence no match") + logging.debug("Field " + k["field"] + " not in data structure, hence no match") regexMatch = False break else: - if (k["field"] and not re.search(k["pattern"], data.get(k["field"]))): + if not re.search(k["pattern"], data.get(k["field"])): logging.debug("No match") regexMatch = False + break if regexMatch: coordinatesString = i["coordinates"] - logging.debug("Coordinate String: " + coordinatesString) + logging.debug("Coordinates string: " + coordinatesString) coordinatesList = coordinatesString.replace(" ", "").split(",") if len(coordinatesList) == 2: data["lat"] = coordinatesList[0] From 3f9bb7ceb2b6a016c18e3420add16cb449849d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 11:22:40 +0100 Subject: [PATCH 3/9] Added changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3107002..75759ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added +- GPS-coordinates configuration based on POC message elements ##### Changed ##### Deprecated ##### Removed From ec2e5bd99b2f9a9ba90cbe340531e30df05483f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 11:54:29 +0100 Subject: [PATCH 4/9] Add pull request link to changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75759ef..8e950cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added -- GPS-coordinates configuration based on POC message elements +- GPS-coordinates configuration based on POC message elements. [#509](https://github.com/Schrolli91/BOSWatch/pull/509) ##### Changed ##### Deprecated ##### Removed From 13d76ceab998ed66cc5739896826168d9663447e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 12:00:35 +0100 Subject: [PATCH 5/9] Corrected pull request link in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e950cb..dae948b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added -- GPS-coordinates configuration based on POC message elements. [#509](https://github.com/Schrolli91/BOSWatch/pull/509) +- GPS-coordinates configuration based on POC message elements. [#510](https://github.com/Schrolli91/BOSWatch/pull/510) ##### Changed ##### Deprecated ##### Removed From 1fc2980f6d8a0b6214325c9cfde9c899aad33873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 13:39:40 +0100 Subject: [PATCH 6/9] Make configuration section optional --- config/config.template.ini | 2 +- includes/locationCoordinates.py | 41 +++++++++++++++++---------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/config/config.template.ini b/config/config.template.ini index c097993..8c4ed23 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -127,7 +127,7 @@ geo_enable = 0 geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat -[LocationCoordinates] +#[LocationCoordinates] # Regex Coordinate replacement (only for POC) # All fields in data structure can be used, also dynamically added fields that have been evaluated in "schemaPOCMsg". # Multiple search criteria can be given, then all of them must be hit (AND-condition). diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py index ef95ecd..f5a18c9 100644 --- a/includes/locationCoordinates.py +++ b/includes/locationCoordinates.py @@ -20,27 +20,28 @@ filterList = [] def loadFilters(): try: - logging.debug("Loading location coordinates") - - for key,val in globalVars.config.items("LocationCoordinates"): - logging.debug(" - %s = %s", key, val) - filterData = val.split(";") - - # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items - if len(filterData) < 3 and len(filterData) % 2 == 0: - logging.debug("Invalid argument count; skipping") - else: - # first store all regular expressions in list - filterItem = [] - i = 0 - - while i < len(filterData) - 2: - filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) + if globalVars.config.has_section("LocationCoordinates"): + logging.debug("Loading location coordinates") + + for key,val in globalVars.config.items("LocationCoordinates"): + logging.debug(" - %s = %s", key, val) + filterData = val.split(";") + + # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items + if len(filterData) < 3 and len(filterData) % 2 == 0: + logging.debug("Invalid argument count; skipping") + else: + # first store all regular expressions in list + filterItem = [] + i = 0 - # step to next field - i += 2 - # then transfer to filterList; include coordinates - filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) + while i < len(filterData) - 2: + filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) + + # step to next field + i += 2 + # then transfer to filterList; include coordinates + filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) except: logging.error("cannot read config file") logging.debug("cannot read config file", exc_info=True) From a76e54211f14863a2e04bfe192dd7236a4e1656f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Tue, 26 Jan 2021 16:14:57 +0100 Subject: [PATCH 7/9] Dedicated on/off switch for location coordinate finding functionality --- boswatch.py | 5 ++-- config/config.template.ini | 17 ++++++++++---- includes/decoders/poc.py | 3 ++- includes/locationCoordinates.py | 41 ++++++++++++++++----------------- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/boswatch.py b/boswatch.py index 76d8f99..0c2457f 100755 --- a/boswatch.py +++ b/boswatch.py @@ -291,8 +291,9 @@ try: # Load location RegEx # try: - from includes import locationCoordinates - locationCoordinates.loadFilters() + if globalVars.config.getboolean("LocationCoordinates", "locationCoordinates"): + from includes import locationCoordinates + locationCoordinates.loadFilters() except: # It's an error, but we could work without that stuff... logging.error("cannot load location regex") diff --git a/config/config.template.ini b/config/config.template.ini index 8c4ed23..b468c37 100644 --- a/config/config.template.ini +++ b/config/config.template.ini @@ -127,22 +127,29 @@ geo_enable = 0 geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})# geo_order = LON, lon, LAT, lat -#[LocationCoordinates] +[LocationCoordinates] # Regex Coordinate replacement (only for POC) # All fields in data structure can be used, also dynamically added fields that have been evaluated in "schemaPOCMsg". # Multiple search criteria can be given, then all of them must be hit (AND-condition). # Coordinates must be the last field, consisting of latitude and longitude (order is important), split by comma. # First match has priority; search will not proceed as soon as one hit is found. # Important: Semicolon and comma must not be part of a field or regex, as they are used internally for splitting up the config value correctly. -#LocationName = field1;regex1;...;lat, lon + +# Do you want to enable this feature? (0 - off | 1 - on) +locationCoordinates = 0 + +# LocationName = field1;regex1;...;lat, lon # Examples: + # msg starting with "BOSWatch-Test" -Location1 = msg;^BOSWatch-Test;49.344394413024084, 8.167496841047555 +#Location1 = msg;^BOSWatch-Test;49.344394413024084, 8.167496841047555 + # Objekt containing "VS Wachtenburg" -Location2 = Objekt;VS Wachtenburg;49.437673, 8.173793 +#Location2 = Objekt;VS Wachtenburg;49.437673, 8.173793 + # Ort starting with "B9 ", Ortsteil starting with "16 AK " -B9_16 = Ort;^B9 .*$;Ortsteil;^16 AK .*$;49.428685, 8.408548 +#B9_16 = Ort;^B9 .*$;Ortsteil;^16 AK .*$;49.428685, 8.408548 [multicastAlarm] diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index d51e7d0..898085c 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -163,7 +163,8 @@ def decode(freq, decoded): data["lon"] = lon data["lat"] = lat else: - locationCoordinates.findCoordinates(data) + if globalVars.config.getboolean("LocationCoordinates", "locationCoordinates"): + locationCoordinates.findCoordinates(data) # Add function as character a-d to dataset data["functionChar"] = data["function"].replace("1", "a").replace("2", "b").replace("3", "c").replace("4", "d") diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py index f5a18c9..ef95ecd 100644 --- a/includes/locationCoordinates.py +++ b/includes/locationCoordinates.py @@ -20,28 +20,27 @@ filterList = [] def loadFilters(): try: - if globalVars.config.has_section("LocationCoordinates"): - logging.debug("Loading location coordinates") - - for key,val in globalVars.config.items("LocationCoordinates"): - logging.debug(" - %s = %s", key, val) - filterData = val.split(";") - - # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items - if len(filterData) < 3 and len(filterData) % 2 == 0: - logging.debug("Invalid argument count; skipping") - else: - # first store all regular expressions in list - filterItem = [] - i = 0 + logging.debug("Loading location coordinates") + + for key,val in globalVars.config.items("LocationCoordinates"): + logging.debug(" - %s = %s", key, val) + filterData = val.split(";") + + # at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items + if len(filterData) < 3 and len(filterData) % 2 == 0: + logging.debug("Invalid argument count; skipping") + else: + # first store all regular expressions in list + filterItem = [] + i = 0 + + while i < len(filterData) - 2: + filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) - while i < len(filterData) - 2: - filterItem.append({"field": filterData[i], "pattern": filterData[i+1]}) - - # step to next field - i += 2 - # then transfer to filterList; include coordinates - filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) + # step to next field + i += 2 + # then transfer to filterList; include coordinates + filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]}) except: logging.error("cannot read config file") logging.debug("cannot read config file", exc_info=True) From 36ebaa80ca1016d37590112f2e7b9df9f1ed0e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Thu, 28 Jan 2021 20:54:55 +0100 Subject: [PATCH 8/9] Enhanced changelog; decreased loglevel --- CHANGELOG.md | 2 +- includes/locationCoordinates.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dae948b..9bfe776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### __[v2.5.3]__ - unreleased ##### Added -- GPS-coordinates configuration based on POC message elements. [#510](https://github.com/Schrolli91/BOSWatch/pull/510) +- Functionality to fill coordinate values in POC data structure (lat, lon) based on configured locations that match a regular expression in POC message [#510](https://github.com/Schrolli91/BOSWatch/pull/510) ##### Changed ##### Deprecated ##### Removed diff --git a/includes/locationCoordinates.py b/includes/locationCoordinates.py index ef95ecd..860aa0f 100644 --- a/includes/locationCoordinates.py +++ b/includes/locationCoordinates.py @@ -71,7 +71,7 @@ def findCoordinates(data): data["lat"] = coordinatesList[0] data["lon"] = coordinatesList[1] data["has_geo"] = True - logging.info("Coordinates found!") + logging.debug("Coordinates found!") break except: logging.error("cannot read config file") From d1d7bb3be1da96fabeddda5fad9ab0f86990a23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Schotth=C3=B6fer?= Date: Thu, 28 Jan 2021 21:22:09 +0100 Subject: [PATCH 9/9] Import locationCoordinates module only when activated --- includes/decoders/poc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/decoders/poc.py b/includes/decoders/poc.py index 03e90f3..9166d98 100644 --- a/includes/decoders/poc.py +++ b/includes/decoders/poc.py @@ -15,7 +15,6 @@ import re # Regex for validation from includes import globalVars # Global variables from includes import doubleFilter # double alarm filter -from includes import locationCoordinates ## # @@ -177,6 +176,7 @@ def decode(freq, decoded): data["lat"] = lat else: if globalVars.config.getboolean("LocationCoordinates", "locationCoordinates"): + from includes import locationCoordinates locationCoordinates.findCoordinates(data) # Add function as character a-d to dataset