Merge branch 'develop' into new_branch

CHANGELOG.md

@@ -1,6 +1,20 @@
 # Changelog
 
 
+### __[v2.5.3]__ - unreleased
+##### Added
+- Functionality to fill coordinate values in POC data structure (lat, lon) based on configured locations that match a regular expression in POC message [#510](https://github.com/Schrolli91/BOSWatch/pull/510)
+- Extending POC data-structure by Regex named groups matching. [#508](https://github.com/Schrolli91/BOSWatch/pull/508)
+- MQTT Plugin: Added possibility to configure login (user and password) for mqtt broker. [#539](https://github.com/Schrolli91/BOSWatch/pull/539)
+##### Changed
+##### Deprecated
+##### Removed
+##### Fixed
+ - Use specific mysql-connector-python version 8.0.19 [#533](https://github.com/Schrolli91/BOSWatch/pull/533)
+ - Correct Syntax-Errors in MySQL Statements [#533](https://github.com/Schrolli91/BOSWatch/pull/533)
+##### Security
+
+
 ### __[v2.5.2]__ - 08.01.2021
 ##### Added
 - fhemCmd-Plugin: New plugin fhemCmd to execute commands in FHEM home automation. [#457](https://github.com/Schrolli91/BOSWatch/pull/457)

boswatch.py

@@ -287,6 +287,18 @@ try:
 		logging.error("cannot load description lists")
 		logging.debug("cannot load description lists", exc_info=True)
 
+	#
+	# Load location RegEx
+	#
+	try:
+		if globalVars.config.getboolean("LocationCoordinates", "locationCoordinates"):
+			from includes import locationCoordinates
+			locationCoordinates.loadFilters()
+	except:
+		# It's an error, but we could work without that stuff...
+		logging.error("cannot load location regex")
+		logging.debug("cannot load location regex", exc_info=True)
+
 	#
 	# Start rtl_fm
 	#

config/config.template.ini

@@ -127,6 +127,47 @@ geo_enable = 0
 geo_format = #C(\d{2})(\d{5}),(\d{2})(\d{5})#
 geo_order = LON, lon, LAT, lat
 
+# Analyze message and associate data to named fields
+# If the regular expression matches the POC message, the "named groups" defined in schemaRegex will be added to the data structure.
+# You can check this by looking at the value in data["has_schema_fields"]. If it is True, then the regular expression has been hit and the additional fields are available.
+# E.g. POC message is "TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz"
+#
+# Without the schemaRegex, data will look like this (RIC, SubRIC and Bitrate are random):
+# {'function': '1', 'has_geo': False, 'description': '1234567', 'has_schema_fields': False, 'msg': 'TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz', 'bitrate': 1200, 'ric': '1234567'}
+#
+# When using the schemaRegex from below, data will look like this:
+# {'function': '1', 'has_geo': False, 'description': '1234567', 'has_schema_fields': True, 'msg': 'TESTALARM Person in Zwangslage;H2.04;Neustadt;Königsbach;Weinstraße 1;VS Winzerheim;;Dortige Baustelle..Treppensturz', 'bitrate': 1200, 'ric': '1234567, 'Objekt': 'VS Winzerheim', 'StichwortLang': 'TESTALARM Person in Zwangslage', 'Ort': 'Neustadt', 'Bemerkung1': '', 'Adresse': 'Weinstraße 1', 'Bemerkung2': 'Dortige Baustelle..Treppensturz', 'StichwortKurz': 'H2.04', 'Ortsteil': 'Königsbach'}
+#
+# Attention: If you define named groups with names of fields that that are already present in POC decoder's standard data-structure, the content of the respective fields will be overwritten
+# with data evaluated by this regex (if the regex finds a match). E.g. if you define a named group like "...(?P<ric>.*)..." and the regex will be hit,
+# then data["ric"] will get filled with the content evaluated by the regex.
+#schemaRegex = ^(?P<StichwortLang>.*?);(?P<StichwortKurz>.*?);(?P<Ort>.*?);(?P<Ortsteil>.*?);(?P<Adresse>.*?);(?P<Objekt>.*?);(?P<Bemerkung1>.*?);(?P<Bemerkung2>.*?)$
+
+
+[LocationCoordinates]
+# Regex Coordinate replacement (only for POC)
+# All fields in data structure can be used, also dynamically added fields that have been evaluated in "schemaPOCMsg".
+# Multiple search criteria can be given, then all of them must be hit (AND-condition).
+# Coordinates must be the last field, consisting of latitude and longitude (order is important), split by comma.
+# First match has priority; search will not proceed as soon as one hit is found.
+# Important: Semicolon and comma must not be part of a field or regex, as they are used internally for splitting up the config value correctly.
+
+# Do you want to enable this feature? (0 - off | 1 - on)
+locationCoordinates = 0
+
+# LocationName = field1;regex1;...;lat, lon
+
+# Examples:
+
+# msg starting with "BOSWatch-Test"
+#Location1 = msg;^BOSWatch-Test;49.344394413024084, 8.167496841047555
+
+# Objekt containing "VS Wachtenburg"
+#Location2 = Objekt;VS Wachtenburg;49.437673, 8.173793
+
+# Ort starting with "B9 ", Ortsteil starting with "16 AK "
+#B9_16 = Ort;^B9 .*$;Ortsteil;^16 AK .*$;49.428685, 8.408548
+
 
 [multicastAlarm]
 # Configure multicastAlarm if your POCSAG network uses an optimized transmission scheme for alarms with more than one RIC (often found in Swissphone networks).
@@ -555,6 +596,9 @@ commandPOC =
 #Adress from MQTT-Broker
 brokeraddress = 192.168.178.27
 topic = alarm/data
+# username and password for the broker. leave username empty to use anonymous login
+brokerusername = 
+brokerpassword = 
 
 #####################
 ##### Not ready yet #

includes/decoders/poc.py

@@ -157,10 +157,28 @@ def decode(freq, decoded):
 
 					# check for double alarm
 					if doubleFilter.checkID("POC", poc_id+poc_sub, poc_text):
-						data = {"ric":poc_id, "function":poc_sub, "msg":poc_text, "bitrate":bitrate, "description":poc_id, "has_geo":has_geo}
+						data = {"ric":poc_id, "function":poc_sub, "msg":poc_text, "bitrate":bitrate, "description":poc_id, "has_geo":has_geo, "has_schema_fields":False}
+						
+						# if a schema is defined, analyze and associate
+						if globalVars.config.has_option("POC", "schemaRegex"):
+							logging.debug("schemaRegex found")
+							m = re.match(globalVars.config.get("POC", "schemaRegex"), poc_text)
+							if m:
+								logging.debug("POC Schema match")
+								# enrich data structure by regex groups
+								data.update(m.groupdict())
+								data["has_schema_fields"] = True
+							else:
+								logging.debug("No POC Schema match")
+						
 						if has_geo == True:
 							data["lon"] = lon
 							data["lat"] = lat
+						else:
+							if globalVars.config.getboolean("LocationCoordinates", "locationCoordinates"):
+								from includes import locationCoordinates
+								locationCoordinates.findCoordinates(data)
+							
 						# Add function as character a-d to dataset
 						data["functionChar"] = data["function"].replace("1", "a").replace("2", "b").replace("3", "c").replace("4", "d")
 						data["ricFuncChar"] = data["ric"] + data["functionChar"]
@@ -177,7 +195,7 @@ def decode(freq, decoded):
 							logging.debug(" - multicastAlarm without msg")
 							from includes import multicastAlarm
 							multicastAlarm.newEntrymultiList(data)
-
+						
 						# multicastAlarm processing if enabled and alarm message has been received
 						elif globalVars.config.getint("multicastAlarm", "multicastAlarm") and data["msg"] != "" and data["ric"] in globalVars.config.get("multicastAlarm", "multicastAlarm_ric"):
 							logging.debug(" - multicastAlarm with message")

includes/locationCoordinates.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+"""
+Functions for the location RegEX
+
+@author: Marco Schotthöfer
+
+@requires: Configuration has to be set in the config.ini
+"""
+
+import logging # Global logger
+import re #Regex for Filter Check
+
+from includes import globalVars  # Global variables
+
+# local variables
+filterList = []
+
+
+def loadFilters():
+	try:
+		logging.debug("Loading location coordinates")
+		
+		for key,val in globalVars.config.items("LocationCoordinates"):
+			logging.debug(" - %s = %s", key, val)
+			filterData = val.split(";")
+
+			# at least 3 items needed (field1;pattern1;lat,lon), and in any case an uneven count of items
+			if len(filterData) < 3 and len(filterData) % 2 == 0:
+				logging.debug("Invalid argument count; skipping")
+			else:
+				# first store all regular expressions in list
+				filterItem = []
+				i = 0
+				
+				while i < len(filterData) - 2:
+					filterItem.append({"field": filterData[i], "pattern": filterData[i+1]})
+					
+					# step to next field
+					i += 2
+			# then transfer to filterList; include coordinates
+			filterList.append({"name": key, "filterItem": filterItem, "coordinates": filterData[len(filterData) - 1]})
+	except:
+		logging.error("cannot read config file")
+		logging.debug("cannot read config file", exc_info=True)
+		return
+
+def findCoordinates(data):
+	try:
+		logging.debug("Find coordinates")
+		for i in filterList:
+			logging.debug("Filter: " + str(i))
+			regexMatch = True
+			for k in i["filterItem"]:
+				logging.debug("Pattern : " + str(k))
+				if k["field"] not in data.keys():
+					logging.debug("Field " + k["field"] + " not in data structure, hence no match")
+					regexMatch = False
+					break
+				else:
+					if not re.search(k["pattern"], data.get(k["field"])):
+						logging.debug("No match")
+						regexMatch = False
+						break
+			if regexMatch:
+				coordinatesString = i["coordinates"]
+				logging.debug("Coordinates string: " + coordinatesString)
+				coordinatesList = coordinatesString.replace(" ", "").split(",")
+				if len(coordinatesList) == 2:
+					data["lat"] = coordinatesList[0]
+					data["lon"] = coordinatesList[1]
+					data["has_geo"] = True
+					logging.debug("Coordinates found!")
+					break
+	except:
+		logging.error("cannot read config file")
+		logging.debug("cannot read config file", exc_info=True)

install.sh

@@ -187,7 +187,7 @@ echo "[ 8/9] [########-]"
 tput cup 15 5
 echo "-> Download & Install MySQL connector for Python."
 cd $boswatch_install_path
-pip install mysql-connector-python >> $boswatch_install_path/setup_log.txt 2>&1
+pip install mysql-connector-python==8.0.19 >> $boswatch_install_path/setup_log.txt 2>&1
 exitcodefunction $? install mysql-connector
 
 # Blacklist DVB-Drivers

plugins/MySQL/boswatch.sql

@@ -25,8 +25,8 @@ SET time_zone = "+00:00";
 -- Datenbank anlegen `boswatch`
 --
 
-CREATE DATABASE IF NOT EXISTS 'boswatch' DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-USE 'boswatch';
+CREATE DATABASE IF NOT EXISTS `boswatch` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+USE `boswatch`;
 
 -- --------------------------------------------------------
 

plugins/mqtt/mqtt.py

@@ -77,7 +77,11 @@ def run(typ,freq,data):
 			########## User Plugin CODE ##########
 			broker_address = globalVars.config.get("mqtt", "brokeraddress")
 			topic = globalVars.config.get("mqtt", "topic")
-			mqttClient = mqtt.Client()
+
+            broker_username = globalVars.config.get("mqtt", "brokerusername")
+            broker_password = globalVars.config.get("mqtt", "brokerpassword")
+
+            mqttClient = mqtt.Client()
 
 			if typ == "FMS":
 				x = {
@@ -112,6 +116,12 @@ def run(typ,freq,data):
 				logging.warning("Invalid Typ: %s", typ)
 			
 			y = json.dumps(x)
+
+
+            ## only login if there is a username given
+            if( len(broker_username) > 0 ):
+                mqttClient.username_pw_set(broker_username, broker_password)
+
 			mqttClient.connect(broker_address)
 			mqttClient.publish(topic,y)
 			########## User Plugin CODE ##########